From 4bc83666e9165d23393235a8c89bf3fa61979a4f Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Wed, 2 Oct 2019 17:00:55 -0400 Subject: [PATCH 1/4] Submit metrics for invocations and errors --- datadog_lambda/tags.py | 40 ++++++++++ datadog_lambda/wrapper.py | 32 +++++++- tests/test_tags.py | 60 +++++++++++++++ tests/test_wrapper.py | 158 +++++++++++++++++++++++++++++++++----- 4 files changed, 265 insertions(+), 25 deletions(-) create mode 100644 datadog_lambda/tags.py create mode 100644 tests/test_tags.py diff --git a/datadog_lambda/tags.py b/datadog_lambda/tags.py new file mode 100644 index 00000000..c3e5fc03 --- /dev/null +++ b/datadog_lambda/tags.py @@ -0,0 +1,40 @@ +def parse_lambda_tags_from_arn(arn): + """Generate the list of lambda tags based on the data in the arn + Args: + arn (str): Lambda ARN. + ex: arn:aws:lambda:us-east-1:123597598159:function:my-lambda[:optional-version] + """ + # Cap the number of times to split + split_arn = arn.split(":") + + # If ARN includes version / alias at the end, drop it + if len(split_arn) > 7: + split_arn = split_arn[:7] + + _, _, _, region, account_id, _, function_name = split_arn + + return [ + "region:{}".format(region), + "account_id:{}".format(account_id), + "functionname:{}".format(function_name), + ] + + +def get_tags_from_context(context, cold_start_request_id): + """Uses properties of the Lambda context to create the list of tags + + Args: + context (dict): context this Lambda was invoked with + cold_start_request_id (str): the first request ID to execute in this container + + Returns: + tag list (str[]): list of string tags in key:value format + """ + tags = parse_lambda_tags_from_arn(context.invoked_function_arn) + tags.append("memorysize:{}".format(context.memory_limit_in_mb)) + + did_request_cold_start = cold_start_request_id == context.aws_request_id + cold_start_tag = "cold_start:{}".format(str(did_request_cold_start).lower()) + tags.append(cold_start_tag) + + return tags diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index ab43115a..9264a2f9 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -7,14 +7,17 @@ import logging import traceback -from datadog_lambda.metric import lambda_stats +from datadog_lambda.metric import lambda_stats, lambda_metric from datadog_lambda.patch import patch_all +from datadog_lambda.tags import get_tags_from_context from datadog_lambda.tracing import ( extract_dd_trace_context, set_correlation_ids, inject_correlation_ids, ) +ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" + logger = logging.getLogger(__name__) @@ -31,6 +34,10 @@ def my_lambda_handle(event, context): requests.get("https://www.datadoghq.com") """ +# On the first run of this Lambda container this variable is set +# to the str value of the request ID, taken from the Lambda context +cold_start_request_id = None + class _LambdaDecorator(object): """ @@ -40,8 +47,8 @@ class _LambdaDecorator(object): def __init__(self, func): self.func = func - self.flush_to_log = os.environ.get('DD_FLUSH_TO_LOG', '').lower() == 'true' - self.logs_injection = os.environ.get('DD_LOGS_INJECTION', '').lower() == 'true' + self.flush_to_log = os.environ.get("DD_FLUSH_TO_LOG", "").lower() == "true" + self.logs_injection = os.environ.get("DD_LOGS_INJECTION", "").lower() == "true" # Inject trace correlation ids to logs if self.logs_injection: @@ -49,10 +56,20 @@ def __init__(self, func): # Patch HTTP clients to propagate Datadog trace context patch_all() - logger.debug('datadog_lambda_wrapper initialized') + logger.debug("datadog_lambda_wrapper initialized") def _before(self, event, context): + global cold_start_request_id + # Assign this request ID as the cold start if there is no value yet + if cold_start_request_id is None: + cold_start_request_id = context.aws_request_id + try: + lambda_metric( + "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX), + 1, + tags=get_tags_from_context(context, cold_start_request_id), + ) # Extract Datadog trace context from incoming requests extract_dd_trace_context(event) @@ -72,6 +89,13 @@ def __call__(self, event, context): self._before(event, context) try: return self.func(event, context) + except Exception: + lambda_metric( + "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX), + 1, + tags=get_tags_from_context(context, cold_start_request_id), + ) + raise finally: self._after(event, context) diff --git a/tests/test_tags.py b/tests/test_tags.py new file mode 100644 index 00000000..a1271ff8 --- /dev/null +++ b/tests/test_tags.py @@ -0,0 +1,60 @@ +import unittest + +from datadog_lambda.tags import parse_lambda_tags_from_arn, get_tags_from_context +from tests.test_wrapper import get_mock_context + + +class TestMetricTags(unittest.TestCase): + def test_parse_lambda_tags_from_arn(self): + self.assertListEqual( + parse_lambda_tags_from_arn( + "arn:aws:lambda:us-east-1:1234597598159:function:swf-hello-test" + ), + [ + "region:us-east-1", + "account_id:1234597598159", + "functionname:swf-hello-test", + ], + ) + + self.assertListEqual( + parse_lambda_tags_from_arn( + "arn:aws:lambda:us-west-1:1234597598159:function:other-function:function-alias" + ), + [ + "region:us-west-1", + "account_id:1234597598159", + "functionname:other-function", + ], + ) + + def test_get_tags_from_context(self): + cold_start_request_id = "first-request-id" + self.assertListEqual( + get_tags_from_context( + get_mock_context(aws_request_id=cold_start_request_id), + cold_start_request_id, + ), + [ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:true", + ], + ) + + self.assertListEqual( + get_tags_from_context( + get_mock_context(aws_request_id="non-cold-start-request-id"), + cold_start_request_id, + ), + [ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:false", + ], + ) + diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index b1fac540..9e64a83c 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -1,38 +1,54 @@ import os import unittest + try: - from unittest.mock import patch, call, ANY + from unittest.mock import patch, call, ANY, MagicMock except ImportError: - from mock import patch, call, ANY + from mock import patch, call, ANY, MagicMock -from datadog_lambda.wrapper import datadog_lambda_wrapper +from datadog_lambda.wrapper import datadog_lambda_wrapper, cold_start_request_id from datadog_lambda.metric import lambda_metric -class TestDatadogLambdaWrapper(unittest.TestCase): +def get_mock_context( + aws_request_id="request-id-1", + memory_limit_in_mb="256", + invoked_function_arn="arn:aws:lambda:us-west-1:123457598159:function:python-layer-test", +): + lambda_context = MagicMock() + lambda_context.aws_request_id = aws_request_id + lambda_context.memory_limit_in_mb = memory_limit_in_mb + lambda_context.invoked_function_arn = invoked_function_arn + return lambda_context + +class TestDatadogLambdaWrapper(unittest.TestCase): def setUp(self): - patcher = patch('datadog_lambda.metric.lambda_stats') + patcher = patch("datadog_lambda.metric.lambda_stats") self.mock_metric_lambda_stats = patcher.start() self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.wrapper.lambda_stats') + patcher = patch("datadog_lambda.wrapper.lambda_stats") self.mock_wrapper_lambda_stats = patcher.start() self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.wrapper.extract_dd_trace_context') + patcher = patch("datadog_lambda.wrapper.lambda_metric") + self.mock_wrapper_lambda_metric = patcher.start() + self.addCleanup(patcher.stop) + + patcher = patch("datadog_lambda.wrapper.extract_dd_trace_context") self.mock_extract_dd_trace_context = patcher.start() self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.wrapper.set_correlation_ids') + patcher = patch("datadog_lambda.wrapper.set_correlation_ids") self.mock_set_correlation_ids = patcher.start() self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.wrapper.inject_correlation_ids') + patcher = patch("datadog_lambda.wrapper.inject_correlation_ids") self.mock_inject_correlation_ids = patcher.start() self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.wrapper.patch_all') + patcher = patch("datadog_lambda.wrapper.patch_all") self.mock_patch_all = patcher.start() self.addCleanup(patcher.stop) @@ -42,12 +58,12 @@ def lambda_handler(event, context): lambda_metric("test.metric", 100) lambda_event = {} - lambda_context = {} - lambda_handler(lambda_event, lambda_context) - self.mock_metric_lambda_stats.distribution.assert_has_calls([ - call('test.metric', 100, timestamp=None, tags=ANY) - ]) + lambda_handler(lambda_event, get_mock_context()) + + self.mock_metric_lambda_stats.distribution.assert_has_calls( + [call("test.metric", 100, timestamp=None, tags=ANY)] + ) self.mock_wrapper_lambda_stats.flush.assert_called() self.mock_extract_dd_trace_context.assert_called_with(lambda_event) self.mock_set_correlation_ids.assert_called() @@ -55,15 +71,14 @@ def lambda_handler(event, context): self.mock_patch_all.assert_called() def test_datadog_lambda_wrapper_flush_to_log(self): - os.environ["DD_FLUSH_TO_LOG"] = 'True' + os.environ["DD_FLUSH_TO_LOG"] = "True" @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) lambda_event = {} - lambda_context = {} - lambda_handler(lambda_event, lambda_context) + lambda_handler(lambda_event, get_mock_context()) self.mock_metric_lambda_stats.distribution.assert_not_called() self.mock_wrapper_lambda_stats.flush.assert_not_called() @@ -71,17 +86,118 @@ def lambda_handler(event, context): del os.environ["DD_FLUSH_TO_LOG"] def test_datadog_lambda_wrapper_inject_correlation_ids(self): - os.environ["DD_LOGS_INJECTION"] = 'True' + os.environ["DD_LOGS_INJECTION"] = "True" @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) lambda_event = {} - lambda_context = {} - lambda_handler(lambda_event, lambda_context) + lambda_handler(lambda_event, get_mock_context()) self.mock_set_correlation_ids.assert_called() self.mock_inject_correlation_ids.assert_called() del os.environ["DD_LOGS_INJECTION"] + + def test_invocations_metric(self): + @datadog_lambda_wrapper + def lambda_handler(event, context): + lambda_metric("test.metric", 100) + + lambda_event = {} + + lambda_handler(lambda_event, get_mock_context()) + + self.mock_wrapper_lambda_metric.assert_has_calls( + [ + call( + "aws.lambda.enhanced.invocations", + 1, + tags=[ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:true", + ], + ) + ] + ) + + def test_errors_metric(self): + @datadog_lambda_wrapper + def lambda_handler(event, context): + raise RuntimeError() + + lambda_event = {} + + with self.assertRaises(RuntimeError): + lambda_handler(lambda_event, get_mock_context()) + + self.mock_wrapper_lambda_metric.assert_has_calls( + [ + call( + "aws.lambda.enhanced.invocations", + 1, + tags=[ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:true", + ], + ), + call( + "aws.lambda.enhanced.errors", + 1, + tags=[ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:true", + ], + ), + ] + ) + + def test_cold_start_tag(self): + @datadog_lambda_wrapper + def lambda_handler(event, context): + lambda_metric("test.metric", 100) + + lambda_event = {} + self.assertIsNone(cold_start_request_id) + lambda_handler(lambda_event, get_mock_context()) + + lambda_handler( + lambda_event, get_mock_context(aws_request_id="second-request-id") + ) + + self.mock_wrapper_lambda_metric.assert_has_calls( + [ + call( + "aws.lambda.enhanced.invocations", + 1, + tags=[ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:true", + ], + ), + call( + "aws.lambda.enhanced.invocations", + 1, + tags=[ + "region:us-west-1", + "account_id:123457598159", + "functionname:python-layer-test", + "memorysize:256", + "cold_start:false", + ], + ), + ] + ) From 6ab22dd164e5b76f69a8190bc6496d39df2c7e2c Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Mon, 7 Oct 2019 14:47:47 -0400 Subject: [PATCH 2/4] Implement PR feedback --- datadog_lambda/cold_start.py | 25 ++++++++++++++ datadog_lambda/metric.py | 67 +++++++++++++++++++++++++----------- datadog_lambda/tags.py | 20 ----------- datadog_lambda/wrapper.py | 30 +++++----------- tests/test_tags.py | 33 +----------------- tests/test_wrapper.py | 20 ++++++----- 6 files changed, 92 insertions(+), 103 deletions(-) create mode 100644 datadog_lambda/cold_start.py diff --git a/datadog_lambda/cold_start.py b/datadog_lambda/cold_start.py new file mode 100644 index 00000000..e53341f4 --- /dev/null +++ b/datadog_lambda/cold_start.py @@ -0,0 +1,25 @@ +_cold_start = True +_lambda_container_initialized = False + + +def set_cold_start(): + """Set the value of the cold start global + + This should be executed once per Lambda execution before the execution + """ + global _cold_start + global _lambda_container_initialized + _cold_start = not _lambda_container_initialized + _lambda_container_initialized = True + + +def is_cold_start(): + """Returns the value of the global cold_start + """ + return _cold_start + + +def get_cold_start_tag(): + """Returns the cold start tag to be used in metrics + """ + return "cold_start:{}".format(str(is_cold_start()).lower()) diff --git a/datadog_lambda/metric.py b/datadog_lambda/metric.py index 6463c72b..38354354 100644 --- a/datadog_lambda/metric.py +++ b/datadog_lambda/metric.py @@ -14,6 +14,11 @@ from datadog import api from datadog.threadstats import ThreadStats from datadog_lambda import __version__ +from datadog_lambda.cold_start import get_cold_start_tag +from datadog_lambda.tags import parse_lambda_tags_from_arn + + +ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" logger = logging.getLogger(__name__) @@ -54,39 +59,59 @@ def lambda_metric(metric_name, value, timestamp=None, tags=None): background thread. """ tags = _tag_dd_lambda_layer(tags) - if os.environ.get('DD_FLUSH_TO_LOG', '').lower() == 'true': - logger.debug('Sending metric %s to Datadog via log forwarder', metric_name) - print(json.dumps({ - 'm': metric_name, - 'v': value, - 'e': timestamp or int(time.time()), - 't': tags - })) - else: - logger.debug('Sending metric %s to Datadog via lambda layer', metric_name) - lambda_stats.distribution( - metric_name, value, timestamp=timestamp, tags=tags + if os.environ.get("DD_FLUSH_TO_LOG", "").lower() == "true": + logger.debug("Sending metric %s to Datadog via log forwarder", metric_name) + print( + json.dumps( + { + "m": metric_name, + "v": value, + "e": timestamp or int(time.time()), + "t": tags, + } + ) ) + else: + logger.debug("Sending metric %s to Datadog via lambda layer", metric_name) + lambda_stats.distribution(metric_name, value, timestamp=timestamp, tags=tags) + + +def submit_invocations_metric(lambda_arn): + """Increment aws.lambda.enhanced.invocations by 1 + """ + lambda_metric( + "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX), + 1, + tags=parse_lambda_tags_from_arn(lambda_arn) + [get_cold_start_tag()], + ) + + +def submit_errors_metric(lambda_arn): + """Increment aws.lambda.enhanced.errors by 1 + """ + lambda_metric( + "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX), + 1, + tags=parse_lambda_tags_from_arn(lambda_arn) + [get_cold_start_tag()], + ) # Decrypt code should run once and variables stored outside of the function # handler so that these are decrypted once per container -DD_KMS_API_KEY = os.environ.get('DD_KMS_API_KEY', '') +DD_KMS_API_KEY = os.environ.get("DD_KMS_API_KEY", "") if DD_KMS_API_KEY: - DD_KMS_API_KEY = boto3.client('kms').decrypt( + DD_KMS_API_KEY = boto3.client("kms").decrypt( CiphertextBlob=base64.b64decode(DD_KMS_API_KEY) - )['Plaintext'] + )["Plaintext"] # Set API Key and Host in the module, so they only set once per container api._api_key = os.environ.get( - 'DATADOG_API_KEY', - os.environ.get('DD_API_KEY', DD_KMS_API_KEY), + "DATADOG_API_KEY", os.environ.get("DD_API_KEY", DD_KMS_API_KEY) ) -logger.debug('Setting DATADOG_API_KEY of length %d', len(api._api_key)) +logger.debug("Setting DATADOG_API_KEY of length %d", len(api._api_key)) # Set DATADOG_HOST, to send data to a non-default Datadog datacenter api._api_host = os.environ.get( - 'DATADOG_HOST', - 'https://api.' + os.environ.get('DD_SITE', 'datadoghq.com') + "DATADOG_HOST", "https://api." + os.environ.get("DD_SITE", "datadoghq.com") ) -logger.debug('Setting DATADOG_HOST to %s', api._api_host) +logger.debug("Setting DATADOG_HOST to %s", api._api_host) diff --git a/datadog_lambda/tags.py b/datadog_lambda/tags.py index c3e5fc03..4d962257 100644 --- a/datadog_lambda/tags.py +++ b/datadog_lambda/tags.py @@ -18,23 +18,3 @@ def parse_lambda_tags_from_arn(arn): "account_id:{}".format(account_id), "functionname:{}".format(function_name), ] - - -def get_tags_from_context(context, cold_start_request_id): - """Uses properties of the Lambda context to create the list of tags - - Args: - context (dict): context this Lambda was invoked with - cold_start_request_id (str): the first request ID to execute in this container - - Returns: - tag list (str[]): list of string tags in key:value format - """ - tags = parse_lambda_tags_from_arn(context.invoked_function_arn) - tags.append("memorysize:{}".format(context.memory_limit_in_mb)) - - did_request_cold_start = cold_start_request_id == context.aws_request_id - cold_start_tag = "cold_start:{}".format(str(did_request_cold_start).lower()) - tags.append(cold_start_tag) - - return tags diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index 9264a2f9..a2c1fad4 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -7,16 +7,19 @@ import logging import traceback -from datadog_lambda.metric import lambda_stats, lambda_metric +from datadog_lambda.cold_start import set_cold_start +from datadog_lambda.metric import ( + lambda_stats, + submit_invocations_metric, + submit_errors_metric, +) from datadog_lambda.patch import patch_all -from datadog_lambda.tags import get_tags_from_context from datadog_lambda.tracing import ( extract_dd_trace_context, set_correlation_ids, inject_correlation_ids, ) -ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" logger = logging.getLogger(__name__) @@ -34,10 +37,6 @@ def my_lambda_handle(event, context): requests.get("https://www.datadoghq.com") """ -# On the first run of this Lambda container this variable is set -# to the str value of the request ID, taken from the Lambda context -cold_start_request_id = None - class _LambdaDecorator(object): """ @@ -59,17 +58,10 @@ def __init__(self, func): logger.debug("datadog_lambda_wrapper initialized") def _before(self, event, context): - global cold_start_request_id - # Assign this request ID as the cold start if there is no value yet - if cold_start_request_id is None: - cold_start_request_id = context.aws_request_id + set_cold_start() try: - lambda_metric( - "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX), - 1, - tags=get_tags_from_context(context, cold_start_request_id), - ) + submit_invocations_metric(context.invoked_function_arn) # Extract Datadog trace context from incoming requests extract_dd_trace_context(event) @@ -90,11 +82,7 @@ def __call__(self, event, context): try: return self.func(event, context) except Exception: - lambda_metric( - "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX), - 1, - tags=get_tags_from_context(context, cold_start_request_id), - ) + submit_errors_metric(context.invoked_function_arn) raise finally: self._after(event, context) diff --git a/tests/test_tags.py b/tests/test_tags.py index a1271ff8..f3dfc4cf 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -1,7 +1,6 @@ import unittest -from datadog_lambda.tags import parse_lambda_tags_from_arn, get_tags_from_context -from tests.test_wrapper import get_mock_context +from datadog_lambda.tags import parse_lambda_tags_from_arn class TestMetricTags(unittest.TestCase): @@ -28,33 +27,3 @@ def test_parse_lambda_tags_from_arn(self): ], ) - def test_get_tags_from_context(self): - cold_start_request_id = "first-request-id" - self.assertListEqual( - get_tags_from_context( - get_mock_context(aws_request_id=cold_start_request_id), - cold_start_request_id, - ), - [ - "region:us-west-1", - "account_id:123457598159", - "functionname:python-layer-test", - "memorysize:256", - "cold_start:true", - ], - ) - - self.assertListEqual( - get_tags_from_context( - get_mock_context(aws_request_id="non-cold-start-request-id"), - cold_start_request_id, - ), - [ - "region:us-west-1", - "account_id:123457598159", - "functionname:python-layer-test", - "memorysize:256", - "cold_start:false", - ], - ) - diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 9e64a83c..0bec5475 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -6,7 +6,7 @@ except ImportError: from mock import patch, call, ANY, MagicMock -from datadog_lambda.wrapper import datadog_lambda_wrapper, cold_start_request_id +from datadog_lambda.wrapper import datadog_lambda_wrapper from datadog_lambda.metric import lambda_metric @@ -32,7 +32,7 @@ def setUp(self): self.mock_wrapper_lambda_stats = patcher.start() self.addCleanup(patcher.stop) - patcher = patch("datadog_lambda.wrapper.lambda_metric") + patcher = patch("datadog_lambda.metric.lambda_metric") self.mock_wrapper_lambda_metric = patcher.start() self.addCleanup(patcher.stop) @@ -52,6 +52,11 @@ def setUp(self): self.mock_patch_all = patcher.start() self.addCleanup(patcher.stop) + patcher = patch("datadog_lambda.cold_start.is_cold_start") + self.mock_is_cold_start = patcher.start() + self.mock_is_cold_start.return_value = True + self.addCleanup(patcher.stop) + def test_datadog_lambda_wrapper(self): @datadog_lambda_wrapper def lambda_handler(event, context): @@ -118,7 +123,6 @@ def lambda_handler(event, context): "region:us-west-1", "account_id:123457598159", "functionname:python-layer-test", - "memorysize:256", "cold_start:true", ], ) @@ -144,7 +148,6 @@ def lambda_handler(event, context): "region:us-west-1", "account_id:123457598159", "functionname:python-layer-test", - "memorysize:256", "cold_start:true", ], ), @@ -155,22 +158,23 @@ def lambda_handler(event, context): "region:us-west-1", "account_id:123457598159", "functionname:python-layer-test", - "memorysize:256", "cold_start:true", ], ), ] ) - def test_cold_start_tag(self): + def test_enhanced_metrics_cold_start_tag(self): @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) lambda_event = {} - self.assertIsNone(cold_start_request_id) + lambda_handler(lambda_event, get_mock_context()) + self.mock_is_cold_start.return_value = False + lambda_handler( lambda_event, get_mock_context(aws_request_id="second-request-id") ) @@ -184,7 +188,6 @@ def lambda_handler(event, context): "region:us-west-1", "account_id:123457598159", "functionname:python-layer-test", - "memorysize:256", "cold_start:true", ], ), @@ -195,7 +198,6 @@ def lambda_handler(event, context): "region:us-west-1", "account_id:123457598159", "functionname:python-layer-test", - "memorysize:256", "cold_start:false", ], ), From b350ac4125af003071688ef653e245f8eb5b8336 Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Tue, 15 Oct 2019 17:22:13 -0400 Subject: [PATCH 3/4] Make enhanced metrics opt-in via an env var --- datadog_lambda/metric.py | 12 ++++++++++++ tests/test_wrapper.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/datadog_lambda/metric.py b/datadog_lambda/metric.py index 38354354..784b4286 100644 --- a/datadog_lambda/metric.py +++ b/datadog_lambda/metric.py @@ -76,9 +76,18 @@ def lambda_metric(metric_name, value, timestamp=None, tags=None): lambda_stats.distribution(metric_name, value, timestamp=timestamp, tags=tags) +def are_enhanced_metrics_enabled(): + """Check env var to find if enhanced metrics should be submitted + """ + return os.environ.get("DD_ENHANCED_METRICS", "false").lower() == "true" + + def submit_invocations_metric(lambda_arn): """Increment aws.lambda.enhanced.invocations by 1 """ + if not are_enhanced_metrics_enabled(): + return + lambda_metric( "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX), 1, @@ -89,6 +98,9 @@ def submit_invocations_metric(lambda_arn): def submit_errors_metric(lambda_arn): """Increment aws.lambda.enhanced.errors by 1 """ + if not are_enhanced_metrics_enabled(): + return + lambda_metric( "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX), 1, diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 0bec5475..4be85b5c 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -106,6 +106,8 @@ def lambda_handler(event, context): del os.environ["DD_LOGS_INJECTION"] def test_invocations_metric(self): + os.environ["DD_ENHANCED_METRICS"] = "True" + @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) @@ -129,7 +131,11 @@ def lambda_handler(event, context): ] ) + del os.environ["DD_ENHANCED_METRICS"] + def test_errors_metric(self): + os.environ["DD_ENHANCED_METRICS"] = "True" + @datadog_lambda_wrapper def lambda_handler(event, context): raise RuntimeError() @@ -164,7 +170,11 @@ def lambda_handler(event, context): ] ) + del os.environ["DD_ENHANCED_METRICS"] + def test_enhanced_metrics_cold_start_tag(self): + os.environ["DD_ENHANCED_METRICS"] = "True" + @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) @@ -203,3 +213,18 @@ def lambda_handler(event, context): ), ] ) + + del os.environ["DD_ENHANCED_METRICS"] + + def test_no_enhanced_metrics_without_env_var(self): + @datadog_lambda_wrapper + def lambda_handler(event, context): + raise RuntimeError() + + lambda_event = {} + + with self.assertRaises(RuntimeError): + lambda_handler(lambda_event, get_mock_context()) + + self.mock_wrapper_lambda_metric.assert_not_called() + From eb206fd7162c3f9ca7815fd5926ffc3b501fb26e Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Thu, 24 Oct 2019 11:06:13 -0400 Subject: [PATCH 4/4] Bump version to 7, update changelog --- CHANGELOG.md | 4 ++++ datadog_lambda/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad0ceeca..3e37ac8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +# Version: 7 / 2019-10-24 + +- Increment `aws.lambda.enhanced.invocations` and `aws.lambda.enhanced.errors` metrics for each invocation if `DD_ENHANCED_METRICS` env var is set to true. + # Version: 6 / 2019-09-16 - Support `DD_LOGS_INJECTION` for trace and log correlation diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index 7535f416..ac90d4f8 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = '0.6.0' +__version__ = '0.7.0' import os