Improve the installation and upgrade process

Author: tianchu

aws/logs_monitoring/lambda_function.py

@@ -194,7 +194,12 @@ def compileRegex(rule, pattern):
 EXCLUDE_AT_MATCH = os.getenv("EXCLUDE_AT_MATCH", default=None)
 exclude_regex = compileRegex("EXCLUDE_AT_MATCH", EXCLUDE_AT_MATCH)
 
-if "DD_KMS_API_KEY" in os.environ:
+if "DD_API_KEY_SECRET_ID" in os.environ:
+    SECRET_ID = os.environ["DD_API_KEY_SECRET_ID"]
+    DD_API_KEY = boto3.client("secretsmanager").get_secret_value(
+        SecretId=SECRET_ID
+    )["SecretString"]
+elif "DD_KMS_API_KEY" in os.environ:
     ENCRYPTED = os.environ["DD_KMS_API_KEY"]
     DD_API_KEY = boto3.client("kms").decrypt(
         CiphertextBlob=base64.b64decode(ENCRYPTED)

aws/logs_monitoring/log-sam-template.yaml

@@ -1,17 +1,82 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Transform: AWS::Serverless-2016-10-31
-Description: Pushes logs and metrics from AWS to Datadog.
+Description: Pushes logs, metrics and traces from AWS to Datadog.
+Parameters:
+  DdApiKey:
+    Type: String
+    NoEcho: true
+    Default: ''
+    AllowedPattern: '([0-9a-f]{32}|$^)'
+    ConstraintDescription: DdApiKey needs to be either empty or a hex string of length 32
+    Description: The Datadog API key - use DdKmsApiKey or DdApiKeySecretId for security 
+  DdKmsApiKey:
+    Type: String
+    NoEcho: true
+    Default: ''
+    Description: The KSM-encrypted Datadog API key
+  DdApiKeySecretId:
+    Type: String
+    NoEcho: true
+    Default: ''
+    Description: The Secret Id to fetch the Datadog API key from Secrets Manager
+  DdSite:
+    Type: String
+    Default: datadoghq.com
+    AllowedValues: 
+      - datadoghq.com
+      - datadoghq.eu
+    ConstraintDescription: DdSite must be datadoghq.com or datadoghq.eu
+    Description: Set to datadoghq.eu to send data to the Datadog EU site.
+  DdTags:
+    Type: String
+    Default: ''
+    Description: Add custom tags to the forwarded log entries, e.g., env:prod,stack:classic
+  FunctionName:
+    Type: String
+    Default: 'DatadogForwarder'
+    Description: The Datadog Forwarder Lambda function
+  ReservedConcurrency:
+    Type: Number
+    Default: 100
+    Description: Reserved concurrency for the Datadog Forwarder Lambda function
+  LogRetentionInDays:
+    Type: Number
+    Default: 90
+    Description: Log retention for the Datadog Forwarder Lambda function
+Conditions:
+  SetDdApiKey: !Not [!Equals [!Ref DdApiKey, '']]
+  SetDdKmsApiKey: !Not [!Equals [!Ref DdKmsApiKey, '']]
+  SetDdApiKeySecretId: !Not [!Equals [!Ref DdApiKeySecretId, '']]
+  SetDdTags: !Not [!Equals [!Ref DdTags, '']]
 Resources:
-  loglambdaddfunction:
+  DatadogForwarder:
     Type: 'AWS::Serverless::Function'
     Properties:
-      Description: Pushes logs and metrics from AWS to Datadog.
+      FunctionName: !Ref FunctionName
+      Description: Pushes logs, metrics and traces from AWS to Datadog.
       Handler: lambda_function.lambda_handler
       MemorySize: 1024
-      Runtime: python2.7
+      Runtime: python3.7
       Timeout: 120
       Layers:
-        - !Sub 'arn:aws:lambda:${AWS::Region}:464622532012:layer:Datadog-Python27:3'
-        - !Sub 'arn:aws:lambda:${AWS::Region}:464622532012:layer:Datadog-Trace-Forwarder-Python27:1'
-
-    Type: AWS::Serverless::Function
+        - !Sub 'arn:aws:lambda:${AWS::Region}:464622532012:layer:Datadog-Python37:9'
+        - !Sub 'arn:aws:lambda:${AWS::Region}:464622532012:layer:Datadog-Trace-Forwarder-Python37:1'
+      Environment:
+        Variables:
+          DD_API_KEY: !If [SetDdApiKey, !Ref DdApiKey, !Ref AWS::NoValue]
+          DD_KMS_API_KEY: !If [SetDdKmsApiKey, !Ref DdKmsApiKey, !Ref AWS::NoValue]
+          DD_API_KEY_SECRET_ID: !If [SetDdApiKeySecretId, !Ref DdApiKeySecretId, !Ref AWS::NoValue]
+          DD_TAGS: !If [SetDdTags, !Ref DdTags, !Ref AWS::NoValue]
+          DD_SITE: !Ref DdSite
+      ReservedConcurrentExecutions: !Ref ReservedConcurrency
+  LogGroup:
+    Type: 'AWS::Logs::LogGroup'
+    Properties:
+      LogGroupName: !Sub '/aws/lambda/${DatadogForwarder}'
+      RetentionInDays: !Ref LogRetentionInDays
+Outputs:
+  DatadogForwarderArn:
+    Description: Datadog Forwarder Lambda Function ARN
+    Value: !GetAtt DatadogForwarder.Arn
+    Export:
+      Name: !Sub '${AWS::StackName}-DatadogForwarderArn'