ESM modules support

.github/workflows/build.yml

@@ -94,6 +94,13 @@ jobs:
         with:
           node-version: 14
 
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Complete Buildx Setup
+        run:  docker run --privileged --rm tonistiigi/binfmt --install all
+
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"

.github/workflows/check-size.yml

@@ -17,6 +17,13 @@ jobs:
         with:
           node-version: 14.15
 
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Complete Buildx Setup
+        run:  docker run --privileged --rm tonistiigi/binfmt --install all
+
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"

.gitignore

@@ -4,6 +4,7 @@ yarn-error.log
 coverage
 .layers
 .DS_Store
+.vendored
 package-lock.json
 
 **/.serverless

Dockerfile

@@ -12,6 +12,7 @@ RUN yarn install
 # Build the lambda layer
 RUN yarn build
 RUN cp -r dist /nodejs/node_modules/datadog-lambda-js
+RUN cp ./src/runtime/module_importer.js /nodejs/node_modules/datadog-lambda-js/runtime
 RUN rm -rf node_modules
 
 # Move dd-trace from devDependencies to production dependencies
@@ -22,6 +23,10 @@ RUN yarn install --production=true
 # Copy the dependencies to the modules folder
 RUN cp -rf node_modules/* /nodejs/node_modules
 
+# Remove unused deasync binaries, and replace them with ones copied from the AWS image
+RUN rm -rf /nodejs/node_modules/deasync/bin/*
+RUN cp -rf ./.vendored/* /nodejs/node_modules/deasync/bin
+
 # Remove the AWS SDK, which is installed in the lambda by default
 RUN rm -rf /nodejs/node_modules/aws-sdk
 RUN rm -rf /nodejs/node_modules/aws-xray-sdk-core/node_modules/aws-sdk
@@ -37,4 +42,6 @@ RUN rm -rf /nodejs/node_modules/protobufjs/dist
 RUN rm -rf /nodejs/node_modules/protobufjs/cli
 RUN find /nodejs/node_modules -name "*.d.ts" -delete
 RUN find /nodejs/node_modules -name "*.js.map" -delete
-RUN find /nodejs/node_modules -name "*.ts.map" -delete
+RUN find /nodejs/node_modules -name "*.ts.map" -delete
+
+

integration_tests/esm.mjs

@@ -0,0 +1,8 @@
+import { promisify } from 'util'
+
+// Verify top level await
+await promisify(setTimeout)(100);
+
+export function handle(ev) {
+    return { message: "hello, dog!" };
+}

integration_tests/serverless.yml

@@ -31,6 +31,19 @@ functions:
     environment:
       DD_FLUSH_TO_LOG: true
 
+  # esm
+  esm_node:
+    name: integration-tests-js-${sls:stage}-esm_${env:RUNTIME}
+    handler: /opt/nodejs/node_modules/datadog-lambda-js/handler.handler
+    runtime: ${env:SERVERLESS_RUNTIME}
+    layers:
+      - { Ref: NodeLambdaLayer }
+    environment:
+      DD_FLUSH_TO_LOG: true
+      DD_LAMBDA_HANDLER: esm.handle
+
+
+
   # sync-metrics
   sync-metrics_node:
     name: integration-tests-js-${sls:stage}-sync-metrics_${env:RUNTIME}

integration_tests/snapshots/logs/esm_node12.log

@@ -0,0 +1,18 @@
+XXXX-XX-XX XX:XX:XX.XXX	ERROR	Uncaught Exception 	{"errorType":"Runtime.UserCodeSyntaxError","errorMessage":"SyntaxError: Unexpected reserved word","stack":["Runtime.UserCodeSyntaxError: SyntaxError: Unexpected reserved word","    at UserCodeSyntaxError.ExtendedError [as constructor] (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at new UserCodeSyntaxError (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at _loadUserApp (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at load (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at Object.<anonymous> (/opt/nodejs/node_modules/datadog-lambda-js/handler.js:XXX:XXX)","    at Module._compile (internal/modules/cjs/loader.js:XXX:XXX)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.load (internal/modules/cjs/loader.js:XXX:XXX)","    at Function.Module._load (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.require (internal/modules/cjs/loader.js:XXX:XXX)"]}
+START
+XXXX-XX-XX XX:XX:XX.XXX	ERROR	Uncaught Exception 	{"errorType":"Runtime.UserCodeSyntaxError","errorMessage":"SyntaxError: Unexpected reserved word","stack":["Runtime.UserCodeSyntaxError: SyntaxError: Unexpected reserved word","    at UserCodeSyntaxError.ExtendedError [as constructor] (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at new UserCodeSyntaxError (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at _loadUserApp (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at load (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at Object.<anonymous> (/opt/nodejs/node_modules/datadog-lambda-js/handler.js:XXX:XXX)","    at Module._compile (internal/modules/cjs/loader.js:XXX:XXX)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.load (internal/modules/cjs/loader.js:XXX:XXX)","    at Function.Module._load (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.require (internal/modules/cjs/loader.js:XXX:XXX)"]}
+END Duration: XXXX ms Memory Used: XXXX MB
+Unknown application error occurred
+Runtime.UserCodeSyntaxError
+START
+XXXX-XX-XX XX:XX:XX.XXX	ERROR	Uncaught Exception 	{"errorType":"Runtime.UserCodeSyntaxError","errorMessage":"SyntaxError: Unexpected reserved word","stack":["Runtime.UserCodeSyntaxError: SyntaxError: Unexpected reserved word","    at UserCodeSyntaxError.ExtendedError [as constructor] (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at new UserCodeSyntaxError (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at _loadUserApp (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at load (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at Object.<anonymous> (/opt/nodejs/node_modules/datadog-lambda-js/handler.js:XXX:XXX)","    at Module._compile (internal/modules/cjs/loader.js:XXX:XXX)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.load (internal/modules/cjs/loader.js:XXX:XXX)","    at Function.Module._load (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.require (internal/modules/cjs/loader.js:XXX:XXX)"]}
+END Duration: XXXX ms Memory Used: XXXX MB
+Unknown application error occurred
+Runtime.UserCodeSyntaxError
+XXXX-XX-XX XX:XX:XX.XXX	ERROR	Uncaught Exception 	{"errorType":"Runtime.UserCodeSyntaxError","errorMessage":"SyntaxError: Unexpected reserved word","stack":["Runtime.UserCodeSyntaxError: SyntaxError: Unexpected reserved word","    at UserCodeSyntaxError.ExtendedError [as constructor] (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at new UserCodeSyntaxError (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at _loadUserApp (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at load (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at Object.<anonymous> (/opt/nodejs/node_modules/datadog-lambda-js/handler.js:XXX:XXX)","    at Module._compile (internal/modules/cjs/loader.js:XXX:XXX)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.load (internal/modules/cjs/loader.js:XXX:XXX)","    at Function.Module._load (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.require (internal/modules/cjs/loader.js:XXX:XXX)"]}
+START
+XXXX-XX-XX XX:XX:XX.XXX	ERROR	Uncaught Exception 	{"errorType":"Runtime.UserCodeSyntaxError","errorMessage":"SyntaxError: Unexpected reserved word","stack":["Runtime.UserCodeSyntaxError: SyntaxError: Unexpected reserved word","    at UserCodeSyntaxError.ExtendedError [as constructor] (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at new UserCodeSyntaxError (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at _loadUserApp (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at load (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at Object.<anonymous> (/opt/nodejs/node_modules/datadog-lambda-js/handler.js:XXX:XXX)","    at Module._compile (internal/modules/cjs/loader.js:XXX:XXX)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.load (internal/modules/cjs/loader.js:XXX:XXX)","    at Function.Module._load (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.require (internal/modules/cjs/loader.js:XXX:XXX)"]}
+END Duration: XXXX ms Memory Used: XXXX MB
+Unknown application error occurred
+Runtime.UserCodeSyntaxError
+XXXX-XX-XX XX:XX:XX.XXX	ERROR	Uncaught Exception 	{"errorType":"Runtime.UserCodeSyntaxError","errorMessage":"SyntaxError: Unexpected reserved word","stack":["Runtime.UserCodeSyntaxError: SyntaxError: Unexpected reserved word","    at UserCodeSyntaxError.ExtendedError [as constructor] (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at new UserCodeSyntaxError (/opt/nodejs/node_modules/datadog-lambda-js/runtime/errors.js:XXX:XXX)","    at _loadUserApp (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at load (/opt/nodejs/node_modules/datadog-lambda-js/runtime/user-function.js:XXX:XXX)","    at Object.<anonymous> (/opt/nodejs/node_modules/datadog-lambda-js/handler.js:XXX:XXX)","    at Module._compile (internal/modules/cjs/loader.js:XXX:XXX)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.load (internal/modules/cjs/loader.js:XXX:XXX)","    at Function.Module._load (internal/modules/cjs/loader.js:XXX:XXX)","    at Module.require (internal/modules/cjs/loader.js:XXX:XXX)"]}

integration_tests/snapshots/logs/esm_node14.log

@@ -0,0 +1,51 @@
+START
+{
+  "e": XXXX,
+  "m": "aws.lambda.enhanced.invocations",
+  "t": [
+    "region:sa-east-1",
+    "account_id:XXXX",
+    "functionname:integration-tests-js-XXXX-esm_node14",
+    "resource:integration-tests-js-XXXX-esm_node14",
+    "cold_start:true",
+    "memorysize:1024",
+    "datadog_lambda:vX.X.X",
+    "runtime:nodejs14.x"
+  ],
+  "v": 1
+}
+END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
+START
+{
+  "e": XXXX,
+  "m": "aws.lambda.enhanced.invocations",
+  "t": [
+    "region:sa-east-1",
+    "account_id:XXXX",
+    "functionname:integration-tests-js-XXXX-esm_node14",
+    "resource:integration-tests-js-XXXX-esm_node14",
+    "cold_start:false",
+    "memorysize:1024",
+    "datadog_lambda:vX.X.X",
+    "runtime:nodejs14.x"
+  ],
+  "v": 1
+}
+END Duration: XXXX ms Memory Used: XXXX MB
+START
+{
+  "e": XXXX,
+  "m": "aws.lambda.enhanced.invocations",
+  "t": [
+    "region:sa-east-1",
+    "account_id:XXXX",
+    "functionname:integration-tests-js-XXXX-esm_node14",
+    "resource:integration-tests-js-XXXX-esm_node14",
+    "cold_start:false",
+    "memorysize:1024",
+    "datadog_lambda:vX.X.X",
+    "runtime:nodejs14.x"
+  ],
+  "v": 1
+}
+END Duration: XXXX ms Memory Used: XXXX MB

integration_tests/snapshots/return_values/esm_node12_api-gateway-get.json

@@ -0,0 +1 @@
+Invocation failed

integration_tests/snapshots/return_values/esm_node12_sns.json

@@ -0,0 +1 @@
+Invocation failed

integration_tests/snapshots/return_values/esm_node12_sqs.json

@@ -0,0 +1 @@
+Invocation failed

integration_tests/snapshots/return_values/esm_node14_api-gateway-get.json

@@ -0,0 +1 @@
+null

integration_tests/snapshots/return_values/esm_node14_sns.json

@@ -0,0 +1 @@
+null

integration_tests/snapshots/return_values/esm_node14_sqs.json

@@ -0,0 +1 @@
+null

package.json

@@ -25,6 +25,7 @@
     "@types/node": "^15.6.1",
     "@types/promise-retry": "^1.1.3",
     "@types/shimmer": "^1.0.1",
+    "@types/deasync": "^0.1.2",
     "dd-trace": "^2.3.1",
     "jest": "^27.0.1",
     "mock-fs": "4.14.0",
@@ -36,6 +37,7 @@
   },
   "dependencies": {
     "bignumber.js": "^9.0.1",
+    "deasync": "^0.1.26",
     "hot-shots": "8.5.0",
     "promise-retry": "^2.0.1",
     "serialize-error": "^8.1.0",
@@ -65,4 +67,4 @@
       "src/**/*.ts"
     ]
   }
-}
+}

scripts/Dockerfile.deasync

@@ -0,0 +1,4 @@
+ARG image
+FROM $image
+RUN yarn add deasync
+

scripts/build_layers.sh

@@ -11,7 +11,9 @@ set -e
 LAYER_DIR=".layers"
 LAYER_FILES_PREFIX="datadog_lambda_node"
 
-NODE_VERSIONS=("12.13" "14.15")
+export NODE_VERSIONS=("12.13" "14.15")
+
+./scripts/fetch_deasync_binary.sh
 
 if [ -z "$NODE_VERSION" ]; then
     echo "Node version not specified, running for all node versions."

scripts/fetch_deasync_binary.sh

@@ -0,0 +1,36 @@
+
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022 Datadog, Inc.
+
+# Builds Datadog node layers for lambda functions, using Docker
+set -e
+export NODE_VERSIONS=("12" "14")
+
+script_path=${BASH_SOURCE[0]}
+cd $(dirname $script_path)/..
+ls
+rm -rf .vendored
+mkdir .vendored
+
+function docker_build {
+    # Args: [node version] [zip destination]
+    arch=$2
+    temp_dir=$(mktemp -d)
+    echo $temp_dir
+    docker buildx build -t deasync-node-${arch}:$1 . -f ./scripts/Dockerfile.deasync --no-cache \
+        --build-arg image=node:$1 \
+        --platform linux/${arch} \
+        --load
+    docker run deasync-node-${arch}:$1 tar cf - node_modules | tar -xf - -C $temp_dir
+    cp -rf $temp_dir/node_modules/deasync/bin/linux-*-node-$1 .vendored/
+
+}
+for node_version in "${NODE_VERSIONS[@]}"
+do
+    docker_build $node_version arm64
+done
+

scripts/run_integration_tests.sh

@@ -10,7 +10,7 @@ set -e
 
 # These values need to be in sync with serverless.yml, where there needs to be a function
 # defined for every handler_runtime combination
-LAMBDA_HANDLERS=("async-metrics" "sync-metrics" "http-requests" "process-input-traced" "throw-error-traced")
+LAMBDA_HANDLERS=("async-metrics" "esm" "sync-metrics" "http-requests" "process-input-traced" "throw-error-traced")
 
 LOGS_WAIT_SECONDS=20
 

src/runtime/module_importer.js

@@ -0,0 +1,6 @@
+
+// Currently no way to prevent typescript from auto-transpiling import into require,
+// so we expose a wrapper in js
+exports.import = function (path) {
+    return import(path);
+}