diff --git a/aws_lambda_powertools/logging/formatter.py b/aws_lambda_powertools/logging/formatter.py index 93e91e0dc8d..db80876c798 100644 --- a/aws_lambda_powertools/logging/formatter.py +++ b/aws_lambda_powertools/logging/formatter.py @@ -139,11 +139,11 @@ def __init__( if self.utc: self.converter = time.gmtime - super(LambdaPowertoolsFormatter, self).__init__(datefmt=self.datefmt) - self.keys_combined = {**self._build_default_keys(), **kwargs} self.log_format.update(**self.keys_combined) + super().__init__(datefmt=self.datefmt) + def serialize(self, log: Dict) -> str: """Serialize structured log dict to JSON str""" return self.json_serializer(log) diff --git a/aws_lambda_powertools/logging/formatters/__init__.py b/aws_lambda_powertools/logging/formatters/__init__.py new file mode 100644 index 00000000000..b6974414f4c --- /dev/null +++ b/aws_lambda_powertools/logging/formatters/__init__.py @@ -0,0 +1,5 @@ +"""Built-in Logger formatters for Observability Providers that require custom config.""" + +# NOTE: we don't expose formatters directly (barrel import) +# as we cannot know if they'll need additional dependencies in the future +# so we isolate to avoid a performance hit and workarounds like lazy imports diff --git a/aws_lambda_powertools/logging/formatters/datadog.py b/aws_lambda_powertools/logging/formatters/datadog.py new file mode 100644 index 00000000000..fa92bf74598 --- /dev/null +++ b/aws_lambda_powertools/logging/formatters/datadog.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from typing import Any, Callable + +from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter + + +class DatadogLogFormatter(LambdaPowertoolsFormatter): + def __init__( + self, + json_serializer: Callable[[dict], str] | None = None, + json_deserializer: Callable[[dict | str | bool | int | float], str] | None = None, + json_default: Callable[[Any], Any] | None = None, + datefmt: str | None = None, + use_datetime_directive: bool = False, + log_record_order: list[str] | None = None, + utc: bool = False, + use_rfc3339: bool = True, # NOTE: The only change from our base formatter + **kwargs, + ): + """Datadog formatter to comply with Datadog log parsing + + Changes compared to the default Logger Formatter: + + - timestamp format to use RFC3339 e.g., "2023-05-01T15:34:26.841+0200" + + + Parameters + ---------- + log_record_order : list[str] | None, optional + _description_, by default None + + Parameters + ---------- + json_serializer : Callable, optional + function to serialize `obj` to a JSON formatted `str`, by default json.dumps + json_deserializer : Callable, optional + function to deserialize `str`, `bytes`, bytearray` containing a JSON document to a Python `obj`, + by default json.loads + json_default : Callable, optional + function to coerce unserializable values, by default str + + Only used when no custom JSON encoder is set + + datefmt : str, optional + String directives (strftime) to format log timestamp. + + See https://docs.python.org/3/library/time.html#time.strftime or + use_datetime_directive: str, optional + Interpret `datefmt` as a format string for `datetime.datetime.strftime`, rather than + `time.strftime` - Only useful when used alongside `datefmt`. + + See https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior . This + also supports a custom %F directive for milliseconds. + + log_record_order : list, optional + set order of log keys when logging, by default ["level", "location", "message", "timestamp"] + + utc : bool, optional + set logging timestamp to UTC, by default False to continue to use local time as per stdlib + use_rfc3339: bool, optional + Whether to use a popular dateformat that complies with both RFC3339 and ISO8601. + e.g., 2022-10-27T16:27:43.738+02:00. + kwargs + Key-value to persist in all log messages + """ + super().__init__( + json_serializer=json_serializer, + json_deserializer=json_deserializer, + json_default=json_default, + datefmt=datefmt, + use_datetime_directive=use_datetime_directive, + log_record_order=log_record_order, + utc=utc, + use_rfc3339=use_rfc3339, + **kwargs, + ) diff --git a/docs/core/logger.md b/docs/core/logger.md index 8bccdafeec3..2f0472368c3 100644 --- a/docs/core/logger.md +++ b/docs/core/logger.md @@ -445,6 +445,26 @@ If you prefer configuring it separately, or you'd want to bring this JSON Format --8<-- "examples/logger/src/powertools_formatter_setup.py" ``` +### Observability providers + +!!! note "In this context, an observability provider is an [AWS Lambda Partner](https://go.aws/3HtU6CZ){target="_blank"} offering a platform for logging, metrics, traces, etc." + +You can send logs to the observability provider of your choice via [Lambda Extensions](https://aws.amazon.com/blogs/compute/using-aws-lambda-extensions-to-send-logs-to-custom-destinations/){target="_blank"}. In most cases, you shouldn't need any custom Logger configuration, and logs will be shipped async without any performance impact. + +#### Built-in formatters + +In rare circumstances where JSON logs are not parsed correctly by your provider, we offer built-in formatters to make this transition easier. + +| Provider | Formatter | Notes | +| -------- | --------------------- | ---------------------------------------------------- | +| Datadog | `DatadogLogFormatter` | Modifies default timestamp to use RFC3339 by default | + +You can use import and use them as any other Logger formatter via `logger_formatter` parameter: + +```python hl_lines="2 4" title="Using built-in Logger Formatters" +--8<-- "examples/logger/src/observability_provider_builtin_formatters.py" +``` + ### Migrating from other Loggers If you're migrating from other Loggers, there are few key points to be aware of: [Service parameter](#the-service-parameter), [Inheriting Loggers](#inheriting-loggers), [Overriding Log records](#overriding-log-records), and [Logging exceptions](#logging-exceptions). diff --git a/examples/logger/src/observability_provider_builtin_formatters.py b/examples/logger/src/observability_provider_builtin_formatters.py new file mode 100644 index 00000000000..3817f1f1b55 --- /dev/null +++ b/examples/logger/src/observability_provider_builtin_formatters.py @@ -0,0 +1,5 @@ +from aws_lambda_powertools import Logger +from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter + +logger = Logger(service="payment", logger_formatter=DatadogLogFormatter()) +logger.info("hello") diff --git a/tests/functional/test_logger_powertools_formatter.py b/tests/functional/test_logger_powertools_formatter.py index 7276f49d487..8b874894e27 100644 --- a/tests/functional/test_logger_powertools_formatter.py +++ b/tests/functional/test_logger_powertools_formatter.py @@ -3,12 +3,14 @@ import json import os import random +import re import string import time import pytest from aws_lambda_powertools import Logger +from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter @pytest.fixture @@ -22,6 +24,10 @@ def service_name(): return "".join(random.SystemRandom().choice(chars) for _ in range(15)) +def capture_logging_output(stdout): + return json.loads(stdout.getvalue().strip()) + + @pytest.mark.parametrize("level", ["DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL"]) def test_setup_with_valid_log_levels(stdout, level, service_name): logger = Logger(service=service_name, level=level, stream=stdout, request_id="request id!", another="value") @@ -309,3 +315,17 @@ def test_log_json_pretty_indent(stdout, service_name, monkeypatch): # THEN the json should contain more than line new_lines = stdout.getvalue().count(os.linesep) assert new_lines > 1 + + +def test_datadog_formatter_use_rfc3339_date(stdout, service_name): + # GIVEN Datadog Log Formatter is used + logger = Logger(service=service_name, stream=stdout, logger_formatter=DatadogLogFormatter()) + RFC3339_REGEX = r"^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)$" + + # WHEN a log statement happens + logger.info({}) + + # THEN the timestamp uses RFC3339 by default + log = capture_logging_output(stdout) + + assert re.fullmatch(RFC3339_REGEX, log["timestamp"]) # "2022-10-27T17:42:26.841+0200"