From 9ef49c955e89ff7e5f9197d97ad3d931194f4f6e Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Thu, 24 Oct 2024 08:53:22 -0700 Subject: [PATCH 1/2] mypy checks for unused ignores --- ecs_logging/_stdlib.py | 24 +++++++++++++----------- noxfile.py | 1 - 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ecs_logging/_stdlib.py b/ecs_logging/_stdlib.py index 7839a7d..f1cbac5 100644 --- a/ecs_logging/_stdlib.py +++ b/ecs_logging/_stdlib.py @@ -32,10 +32,10 @@ from typing import Any, Callable, Dict, Optional, Sequence, Union -try: - from typing import Literal # type: ignore -except ImportError: - from typing_extensions import Literal # type: ignore +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal # Load the attributes of a LogRecord so if some are @@ -105,13 +105,15 @@ def __init__( exclude_keys=["error"] """ - _kwargs = {} - if validate is not None: - # validate was introduced in py3.8 so we need to only provide it if the user provided it - _kwargs["validate"] = validate - super().__init__( # type: ignore[call-arg] - fmt=fmt, datefmt=datefmt, style=style, **_kwargs # type: ignore[arg-type] - ) + # validate was introduced in py3.8 so we need to only provide it if the user provided it + if sys.version_info >= (3, 8) and validate is not None: + super().__init__( + fmt=fmt, datefmt=datefmt, style=style, validate=validate, + ) + else: + super().__init__( + fmt=fmt, datefmt=datefmt, style=style, + ) if stack_trace_limit is not None: if not isinstance(stack_trace_limit, int): diff --git a/noxfile.py b/noxfile.py index 3d4925f..ec6c0bf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -58,6 +58,5 @@ def lint(session): "mypy", "--strict", "--show-error-codes", - "--no-warn-unused-ignores", "ecs_logging/", ) From f44ebffa2b9e19e4d83a3911bc1ef28335cc95aa Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Thu, 24 Oct 2024 09:19:38 -0700 Subject: [PATCH 2/2] StructlogFormatter conforms to structlog.typing.Processor Closes #146 --- ecs_logging/_structlog.py | 16 ++++++++++++---- ecs_logging/_utils.py | 16 ++++++++++++---- pyproject.toml | 1 + tests/test_typing.py | 14 ++++++++++++++ tests/typing/structlog_usage.py | 34 +++++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 tests/test_typing.py create mode 100644 tests/typing/structlog_usage.py diff --git a/ecs_logging/_structlog.py b/ecs_logging/_structlog.py index 5bc65e5..79533da 100644 --- a/ecs_logging/_structlog.py +++ b/ecs_logging/_structlog.py @@ -17,7 +17,13 @@ import time import datetime -from typing import Any, Dict +import sys +from typing import Any + +if sys.version_info >= (3, 9): + from collections.abc import MutableMapping +else: + from typing import MutableMapping from ._meta import ECS_VERSION from ._utils import json_dumps, normalize_dict @@ -26,7 +32,7 @@ class StructlogFormatter: """ECS formatter for the ``structlog`` module""" - def __call__(self, _: Any, name: str, event_dict: Dict[str, Any]) -> str: + def __call__(self, _: Any, name: str, event_dict: MutableMapping[str, Any]) -> str: # Handle event -> message now so that stuff like `event.dataset` doesn't # cause problems down the line @@ -36,7 +42,9 @@ def __call__(self, _: Any, name: str, event_dict: Dict[str, Any]) -> str: event_dict = self.format_to_ecs(event_dict) return self._json_dumps(event_dict) - def format_to_ecs(self, event_dict: Dict[str, Any]) -> Dict[str, Any]: + def format_to_ecs( + self, event_dict: MutableMapping[str, Any] + ) -> MutableMapping[str, Any]: if "@timestamp" not in event_dict: event_dict["@timestamp"] = ( datetime.datetime.fromtimestamp( @@ -55,5 +63,5 @@ def format_to_ecs(self, event_dict: Dict[str, Any]) -> Dict[str, Any]: event_dict.setdefault("ecs.version", ECS_VERSION) return event_dict - def _json_dumps(self, value: Dict[str, Any]) -> str: + def _json_dumps(self, value: MutableMapping[str, Any]) -> str: return json_dumps(value=value) diff --git a/ecs_logging/_utils.py b/ecs_logging/_utils.py index ee5dc6b..0fae1ce 100644 --- a/ecs_logging/_utils.py +++ b/ecs_logging/_utils.py @@ -18,8 +18,14 @@ import collections.abc import json import functools +import sys from typing import Any, Dict, Mapping +if sys.version_info >= (3, 9): + from collections.abc import MutableMapping +else: + from typing import MutableMapping + __all__ = [ "normalize_dict", @@ -53,9 +59,9 @@ def flatten_dict(value: Mapping[str, Any]) -> Dict[str, Any]: return top_level -def normalize_dict(value: Dict[str, Any]) -> Dict[str, Any]: +def normalize_dict(value: MutableMapping[str, Any]) -> MutableMapping[str, Any]: """Expands all dotted names to nested dictionaries""" - if not isinstance(value, dict): + if not isinstance(value, MutableMapping): return value keys = list(value.keys()) for key in keys: @@ -78,7 +84,9 @@ def de_dot(dot_string: str, msg: Any) -> Dict[str, Any]: return ret -def merge_dicts(from_: Dict[Any, Any], into: Dict[Any, Any]) -> Dict[Any, Any]: +def merge_dicts( + from_: Mapping[Any, Any], into: MutableMapping[Any, Any] +) -> MutableMapping[Any, Any]: """Merge deeply nested dictionary structures. When called has side-effects within 'destination'. """ @@ -98,7 +106,7 @@ def merge_dicts(from_: Dict[Any, Any], into: Dict[Any, Any]) -> Dict[Any, Any]: return into -def json_dumps(value: Dict[str, Any]) -> str: +def json_dumps(value: MutableMapping[str, Any]) -> str: # Ensure that the first three fields are '@timestamp', # 'log.level', and 'message' per ECS spec diff --git a/pyproject.toml b/pyproject.toml index 19f0218..4ac503a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ develop = [ "mock", "structlog", "elastic-apm", + "mypy", ] [tool.flit.metadata.urls] diff --git a/tests/test_typing.py b/tests/test_typing.py new file mode 100644 index 0000000..09274ca --- /dev/null +++ b/tests/test_typing.py @@ -0,0 +1,14 @@ +import pathlib +import subprocess +import sys + +import pytest + +_THIS_DIR = pathlib.Path(__file__).parent + + +@pytest.mark.parametrize("file", (_THIS_DIR / "typing").glob("*.py")) +def test_type_check(file): + subprocess.check_call( + [sys.executable, "-m", "mypy", "--strict", "--config-file=", file] + ) diff --git a/tests/typing/structlog_usage.py b/tests/typing/structlog_usage.py new file mode 100644 index 0000000..0824571 --- /dev/null +++ b/tests/typing/structlog_usage.py @@ -0,0 +1,34 @@ +import sys +from typing import List, Tuple + +import ecs_logging +import structlog +from structlog.typing import Processor + + +shared_processors: Tuple[Processor, ...] = ( + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.StackInfoRenderer(), + structlog.dev.set_exc_info, + structlog.processors.TimeStamper(fmt="iso", utc=True), +) + +processors: List[Processor] +if sys.stderr.isatty(): + processors = [ + *shared_processors, + structlog.dev.ConsoleRenderer(), + ] +else: + processors = [ + *shared_processors, + structlog.processors.dict_tracebacks, + ecs_logging.StructlogFormatter(), + ] + +structlog.configure( + processors=processors, + logger_factory=structlog.PrintLoggerFactory(), + cache_logger_on_first_use=True, +)