Add default/fallback behavior for json.dumps (#47)

* Add test case

* Add default/fallback behavior for json.dumps

* CHANGELOG

* Collapse duplicate json.dumps kwargs

* Update precommit

* Remove whitespace

* Add more pre-commit (black, flake8, mypy)

* Fix mypy

* Update pre-commit

Author: basepi

.pre-commit-config.yaml

@@ -1,14 +1,28 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.2.3
+    rev: v4.0.1
     hooks:
     -   id: check-case-conflict
     -   id: check-executables-have-shebangs
     -   id: check-merge-conflict
-
 -   repo: [email protected]:elastic/apm-pipeline-library
     rev: current
     hooks:
     -   id: check-bash-syntax
     -   id: check-jenkins-pipelines
     -   id: check-jjbb
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910
+    hooks:
+    -   id: mypy
+        args: [--strict]
+-   repo: https://github.com/ambv/black
+    rev: 21.6b0
+    hooks:
+    -   id: black
+        language_version: python3
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    -   id: flake8
+        exclude: tests|conftest.py|setup.py

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Fixed an issue in `StructlogFormatter` caused by a conflict with `event`
   (used for the log `message`) and `event.dataset` (a field provided by the
   `elasticapm` integration) ([#46](https://github.com/elastic/ecs-logging-python/pull/46))
+* Add default/fallback handling for json.dumps ([#47](https://github.com/elastic/ecs-logging-python/pull/47))
 
 ## 1.0.0 (2021-02-08)
 

ecs_logging/_utils.py

@@ -16,6 +16,7 @@
 # under the License.
 
 import json
+import functools
 
 try:
     import typing
@@ -152,24 +153,45 @@ def json_dumps(value):
     except KeyError:
         pass
 
+    json_dumps = functools.partial(
+        json.dumps, sort_keys=True, separators=(",", ":"), default=_json_dumps_fallback
+    )
+
     # Because we want to use 'sorted_keys=True' we manually build
     # the first three keys and then build the rest with json.dumps()
     if ordered_fields:
         # Need to call json.dumps() on values just in
         # case the given values aren't strings (even though
         # they should be according to the spec)
         ordered_json = ",".join(
-            '"%s":%s' % (k, json.dumps(v, sort_keys=True, separators=(",", ":")))
+            '"%s":%s'
+            % (
+                k,
+                json_dumps(v),
+            )
             for k, v in ordered_fields
         )
         if value:
             return "{%s,%s" % (
                 ordered_json,
-                json.dumps(value, sort_keys=True, separators=(",", ":"))[1:],
+                json_dumps(value)[1:],
             )
         else:
             return "{%s}" % ordered_json
     # If there are no fields with ordering requirements we
     # pass everything into json.dumps()
     else:
-        return json.dumps(value, sort_keys=True, separators=(",", ":"))
+        return json_dumps(value)
+
+
+def _json_dumps_fallback(value):
+    # type: (Any) -> Any
+    """
+    Fallback handler for json.dumps to handle objects json doesn't know how to
+    serialize.
+    """
+    try:
+        # This is what structlog's json fallback does
+        return value.__structlog__()
+    except AttributeError:
+        return repr(value)

tests/test_structlog_formatter.py

@@ -23,12 +23,18 @@
 import pytest
 
 
+class NotSerializable:
+    def __repr__(self):
+        return "<NotSerializable>"
+
+
 def make_event_dict():
     return {
         "event": "test message",
         "event.dataset": "agent.log",
         "log.logger": "logger-name",
         "foo": "bar",
+        "baz": NotSerializable(),
     }
 
 
@@ -47,7 +53,9 @@ def test_event_dict_formatted(time, spec_validator):
     formatter = ecs_logging.StructlogFormatter()
     assert spec_validator(formatter(None, "debug", make_event_dict())) == (
         '{"@timestamp":"2020-03-20T16:16:37.187Z","log.level":"debug",'
-        '"message":"test message","ecs":{"version":"1.6.0"},'
+        '"message":"test message",'
+        '"baz":"<NotSerializable>",'
+        '"ecs":{"version":"1.6.0"},'
         '"event":{"dataset":"agent.log"},'
         '"foo":"bar",'
         '"log":{"logger":"logger-name"}}'