Merge pull request #43 from DataDog/stephenf/log-enhanced-metrics-by-default

Enable enhanced metrics by default and write enhanced metrics to logs

Author: sfirrin

CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+# Version 12 / 2020-02-03
+
+- Defaults the DD_ENHANCED_METRICS option to `true`
+- If DD_ENHANCED_METRICS is enabled, always writes enhanced metrics to stdout
+- Breaking change: if you previously set the env var DD_ENHANCED_METRICS=true and did not set DD_FLUSH_TO_LOG=true, the enhanced metrics will no longer be submitted to Datadog synchronously; the metrics will now be written to logs. If you already have a Datadog Forwarder Lambda configured, that will read the enhanced metrics logs and submit the metrics asynchronously. If you do not have a Datadog Forwarder set up, you'll need to create one to get enhanced metrics into your Datadog account. See [Datadog Forwarder Lambda setup instructions](https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring).
+- Because of the breaking change above, we've bumped the major package version so that this release is version 1.12.0. We've set the minor version to 12 so that the minor version of our package stays in alignment with the version of the layer we're releasing.
+
 # Version 11 / 2019-12-06
 
 - Add python 3.8 support

datadog_lambda/__init__.py

@@ -1,9 +1,10 @@
 # The minor version corresponds to the Lambda layer version.
 # E.g.,, version 0.5.0 gets packaged into layer version 5.
-__version__ = '0.11.0'
+__version__ = "1.12.0"
 
 
 import os
 import logging
+
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.getLevelName(os.environ.get('DD_LOG_LEVEL', 'INFO').upper()))
+logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper()))

datadog_lambda/metric.py

@@ -4,7 +4,6 @@
 # Copyright 2019 Datadog, Inc.
 
 import os
-import sys
 import json
 import time
 import base64
@@ -13,8 +12,7 @@
 import boto3
 from datadog import api
 from datadog.threadstats import ThreadStats
-from datadog_lambda import __version__
-from datadog_lambda.tags import get_enhanced_metrics_tags
+from datadog_lambda.tags import get_enhanced_metrics_tags, tag_dd_lambda_layer
 
 
 ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced"
@@ -25,25 +23,6 @@
 lambda_stats.start()
 
 
-def _format_dd_lambda_layer_tag():
-    """
-    Formats the dd_lambda_layer tag, e.g., 'dd_lambda_layer:datadog-python27_1'
-    """
-    runtime = "python{}{}".format(sys.version_info[0], sys.version_info[1])
-    return "dd_lambda_layer:datadog-{}_{}".format(runtime, __version__)
-
-
-def _tag_dd_lambda_layer(tags):
-    """
-    Used by lambda_metric to insert the dd_lambda_layer tag
-    """
-    dd_lambda_layer_tag = _format_dd_lambda_layer_tag()
-    if tags:
-        return tags + [dd_lambda_layer_tag]
-    else:
-        return [dd_lambda_layer_tag]
-
-
 def lambda_metric(metric_name, value, timestamp=None, tags=None):
     """
     Submit a data point to Datadog distribution metrics.
@@ -57,54 +36,80 @@ def lambda_metric(metric_name, value, timestamp=None, tags=None):
     periodically and at the end of the function execution in a
     background thread.
     """
-    tags = _tag_dd_lambda_layer(tags)
+    tags = tag_dd_lambda_layer(tags)
     if os.environ.get("DD_FLUSH_TO_LOG", "").lower() == "true":
-        logger.debug("Sending metric %s to Datadog via log forwarder", metric_name)
-        print(
-            json.dumps(
-                {
-                    "m": metric_name,
-                    "v": value,
-                    "e": timestamp or int(time.time()),
-                    "t": tags,
-                }
-            )
-        )
+        write_metric_point_to_stdout(metric_name, value, timestamp, tags)
     else:
         logger.debug("Sending metric %s to Datadog via lambda layer", metric_name)
         lambda_stats.distribution(metric_name, value, timestamp=timestamp, tags=tags)
 
 
+def write_metric_point_to_stdout(metric_name, value, timestamp=None, tags=[]):
+    """Writes the specified metric point to standard output
+    """
+    logger.debug(
+        "Sending metric %s value %s to Datadog via log forwarder", metric_name, value
+    )
+    print(
+        json.dumps(
+            {
+                "m": metric_name,
+                "v": value,
+                "e": timestamp or int(time.time()),
+                "t": tags,
+            }
+        )
+    )
+
+
 def are_enhanced_metrics_enabled():
     """Check env var to find if enhanced metrics should be submitted
+
+    Returns:
+        boolean for whether enhanced metrics are enabled
     """
-    return os.environ.get("DD_ENHANCED_METRICS", "false").lower() == "true"
+    # DD_ENHANCED_METRICS defaults to true
+    return os.environ.get("DD_ENHANCED_METRICS", "true").lower() == "true"
 
 
-def submit_invocations_metric(lambda_context):
-    """Increment aws.lambda.enhanced.invocations by 1
+def submit_enhanced_metric(metric_name, lambda_context):
+    """Submits the enhanced metric with the given name
+
+    Args:
+        metric_name (str): metric name w/o enhanced prefix i.e. "invocations" or "errors"
+        lambda_context (dict): Lambda context dict passed to the function by AWS
     """
     if not are_enhanced_metrics_enabled():
+        logger.debug(
+            "Not submitting enhanced metric %s because enhanced metrics are disabled",
+            metric_name,
+        )
         return
 
-    lambda_metric(
-        "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX),
+    # Enhanced metrics are always written to logs
+    write_metric_point_to_stdout(
+        "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, metric_name),
         1,
         tags=get_enhanced_metrics_tags(lambda_context),
     )
 
 
-def submit_errors_metric(lambda_context):
-    """Increment aws.lambda.enhanced.errors by 1
+def submit_invocations_metric(lambda_context):
+    """Increment aws.lambda.enhanced.invocations by 1, applying runtime, layer, and cold_start tags
+
+    Args:
+        lambda_context (dict): Lambda context dict passed to the function by AWS
     """
-    if not are_enhanced_metrics_enabled():
-        return
+    submit_enhanced_metric("invocations", lambda_context)
 
-    lambda_metric(
-        "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX),
-        1,
-        tags=get_enhanced_metrics_tags(lambda_context),
-    )
+
+def submit_errors_metric(lambda_context):
+    """Increment aws.lambda.enhanced.errors by 1, applying runtime, layer, and cold_start tags
+
+    Args:
+        lambda_context (dict): Lambda context dict passed to the function by AWS
+    """
+    submit_enhanced_metric("errors", lambda_context)
 
 
 # Set API Key and Host in the module, so they only set once per container

datadog_lambda/tags.py

@@ -1,8 +1,30 @@
+import sys
+
 from platform import python_version_tuple
 
+from datadog_lambda import __version__
 from datadog_lambda.cold_start import get_cold_start_tag
 
 
+def _format_dd_lambda_layer_tag():
+    """
+    Formats the dd_lambda_layer tag, e.g., 'dd_lambda_layer:datadog-python27_1'
+    """
+    runtime = "python{}{}".format(sys.version_info[0], sys.version_info[1])
+    return "dd_lambda_layer:datadog-{}_{}".format(runtime, __version__)
+
+
+def tag_dd_lambda_layer(tags):
+    """
+    Used by lambda_metric to insert the dd_lambda_layer tag
+    """
+    dd_lambda_layer_tag = _format_dd_lambda_layer_tag()
+    if tags:
+        return tags + [dd_lambda_layer_tag]
+    else:
+        return [dd_lambda_layer_tag]
+
+
 def parse_lambda_tags_from_arn(arn):
     """Generate the list of lambda tags based on the data in the arn
     Args:
@@ -42,4 +64,5 @@ def get_enhanced_metrics_tags(lambda_context):
         get_cold_start_tag(),
         "memorysize:{}".format(lambda_context.memory_limit_in_mb),
         get_runtime_tag(),
+        _format_dd_lambda_layer_tag(),
     ]

tests/test_metric.py

@@ -1,38 +1,38 @@
 import os
 import unittest
+
 try:
     from unittest.mock import patch, call
 except ImportError:
     from mock import patch, call
 
-from datadog_lambda.metric import (
-    lambda_metric,
-    _format_dd_lambda_layer_tag,
-)
+from datadog_lambda.metric import lambda_metric
+from datadog_lambda.tags import _format_dd_lambda_layer_tag
 
 
 class TestLambdaMetric(unittest.TestCase):
-
     def setUp(self):
-        patcher = patch('datadog_lambda.metric.lambda_stats')
+        patcher = patch("datadog_lambda.metric.lambda_stats")
         self.mock_metric_lambda_stats = patcher.start()
         self.addCleanup(patcher.stop)
 
     def test_lambda_metric_tagged_with_dd_lambda_layer(self):
-        lambda_metric('test', 1)
-        lambda_metric('test', 1, 123, [])
-        lambda_metric('test', 1, tags=['tag1:test'])
+        lambda_metric("test", 1)
+        lambda_metric("test", 1, 123, [])
+        lambda_metric("test", 1, tags=["tag1:test"])
         expected_tag = _format_dd_lambda_layer_tag()
-        self.mock_metric_lambda_stats.distribution.assert_has_calls([
-            call('test', 1, timestamp=None, tags=[expected_tag]),
-            call('test', 1, timestamp=123, tags=[expected_tag]),
-            call('test', 1, timestamp=None, tags=['tag1:test', expected_tag]),
-        ])
+        self.mock_metric_lambda_stats.distribution.assert_has_calls(
+            [
+                call("test", 1, timestamp=None, tags=[expected_tag]),
+                call("test", 1, timestamp=123, tags=[expected_tag]),
+                call("test", 1, timestamp=None, tags=["tag1:test", expected_tag]),
+            ]
+        )
 
     def test_lambda_metric_flush_to_log(self):
-        os.environ["DD_FLUSH_TO_LOG"] = 'True'
+        os.environ["DD_FLUSH_TO_LOG"] = "True"
 
-        lambda_metric('test', 1)
+        lambda_metric("test", 1)
         self.mock_metric_lambda_stats.distribution.assert_not_called()
 
         del os.environ["DD_FLUSH_TO_LOG"]