[AWSX] feat(lambda logs forwarder): Allow to opt-out from S3 tags enrichment (#946)

* [AWSX] feat(lambda logs forwarder): Allow to opt-out from S3 tags enrichment

* Keep current behavior

Author: ge0Aja

aws/logs_monitoring/caching/s3_tags_cache.py

@@ -1,3 +1,4 @@
+import os
 from botocore.exceptions import ClientError
 from caching.base_tags_cache import BaseTagsCache
 from caching.common import parse_get_resources_response_for_tags_by_arn
@@ -16,7 +17,8 @@ def __init__(self, prefix):
         )
 
     def should_fetch_tags(self):
-        return True
+        # set it to true if we don't have the environment variable set to keep the default behavior
+        return os.environ.get("DD_FETCH_S3_TAGS", "true").lower() == "true"
 
     def build_tags_cache(self):
         """Makes API calls to GetResources to get the live tags of the account's S3 buckets
@@ -56,6 +58,13 @@ def build_tags_cache(self):
         return tags_fetch_success, tags_by_arn_cache
 
     def get(self, bucket_arn):
+        if not self.should_fetch_tags():
+            self.logger.debug(
+                "Not fetching S3  tags because the env variable DD_FETCH_S3_TAGS is "
+                "not set to true"
+            )
+            return []
+
         if self._is_expired():
             send_forwarder_internal_metrics("local_s3_tags_cache_expired")
             self.logger.debug("Local cache expired, fetching cache from S3")

aws/logs_monitoring/steps/handlers/s3_handler.py

@@ -109,7 +109,7 @@ def _add_s3_tags_from_cache(self):
             self.metadata[DD_CUSTOM_TAGS] = (
                 ",".join(s3_tags)
                 if not self.metadata[DD_CUSTOM_TAGS]
-                else self.metadata[DD_CUSTOM_TAGS] + "," + ",".join(s3_tags)
+                else ",".join(s3_tags) + "," + self.metadata[DD_CUSTOM_TAGS]
             )
 
     def _extract_data(self):

aws/logs_monitoring/template.yaml

@@ -96,6 +96,13 @@ Parameters:
       - true
       - false
     Description: Let the forwarder fetch Step Functions tags using GetResources API calls and apply them to logs, metrics and traces. If set to true, permission tag:GetResources will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.get_resources_api_calls metric for each API call made.
+  DdFetchS3Tags:
+    Type: String
+    Default: true
+    AllowedValues:
+      - true
+      - false
+    Description: Let the forwarder fetch S3 buckets tags using GetResources API calls and apply them to S3 based logs. If set to true, permission tag:GetResources will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.get_resources_api_calls metric for each API call made.
   DdUseTcp:
     Type: String
     Default: false
@@ -459,6 +466,7 @@ Resources:
             - SetDdFetchLambdaTags
             - !Ref DdFetchLambdaTags
             - !Ref AWS::NoValue
+          DD_FETCH_S3_TAGS: !Ref DdFetchS3Tags
           DD_FETCH_LOG_GROUP_TAGS: !If
             - SetDdFetchLogGroupTags
             - !Ref DdFetchLogGroupTags
@@ -1036,6 +1044,7 @@ Metadata:
           - DdFetchLambdaTags
           - DdFetchLogGroupTags
           - DdFetchStepFunctionsTags
+          - DdFetchS3Tags
           - DdStepFunctionsTraceEnabled
           - DdEnhancedMetrics
           - TagsCacheTTLSeconds