diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c569a53..de18cc4d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,6 +83,8 @@ jobs: - name: Install Serverless Framework run: sudo yarn global add serverless --prefix /usr/local + - name: Install Crossbuild Deps + run: sudo apt install -y qemu-user-static binfmt-support - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' diff --git a/.github/workflows/check-size.yml b/.github/workflows/check-size.yml index 2ce7e816..69d578d0 100644 --- a/.github/workflows/check-size.yml +++ b/.github/workflows/check-size.yml @@ -14,6 +14,9 @@ jobs: with: python-version: 3.7 + - name: Install Crossbuild Deps + run: sudo apt install -y qemu-user-static binfmt-support + - name: Install dependencies run: | pip install virtualenv diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index 84027a94..298ad48c 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -40,17 +40,21 @@ function docker_build_zip { # Args: [python version] [zip destination] destination=$(make_path_absolute $2) + arch=$3 # Install datadogpy in a docker container to avoid the mess from switching # between different python runtimes. temp_dir=$(mktemp -d) - docker build -t datadog-lambda-python:$1 . --no-cache \ + docker buildx build -t datadog-lambda-python-${arch}:$1 . --no-cache \ --build-arg image=python:$1 \ - --build-arg runtime=python$1 + --build-arg runtime=python$1 \ + --platform linux/${arch} \ + --load # Run the image by runtime tag, tar its generatd `python` directory to sdout, # then extract it to a temp directory. - docker run datadog-lambda-python:$1 tar cf - python | tar -xf - -C $temp_dir + docker run datadog-lambda-python-${arch}:$1 tar cf - python | tar -xf - -C $temp_dir + # Zip to destination, and keep directory structure as based in $temp_dir (cd $temp_dir && zip -q -r $destination ./) @@ -64,8 +68,12 @@ mkdir $LAYER_DIR for python_version in "${PYTHON_VERSIONS[@]}" do - echo "Building layer for Python ${python_version}" - docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}${python_version}.zip + if [ "$python_version" == "3.8" ] || [ "$python_version" == "3.9" ]; then + echo "Building layer for Python ${python_version} arch=arm64" + docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-arm64-${python_version}.zip arm64 + fi + echo "Building layer for Python ${python_version} arch=amd64" + docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${python_version}.zip amd64 done echo "Done creating layers:" diff --git a/scripts/check_layer_size.sh b/scripts/check_layer_size.sh index a0268702..962a688e 100755 --- a/scripts/check_layer_size.sh +++ b/scripts/check_layer_size.sh @@ -18,7 +18,7 @@ VERSIONS=("2.7" "3.6" "3.7" "3.8" "3.9") for version in "${VERSIONS[@]}" do - FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}${version}.zip + FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${version}.zip FILE_SIZE=$(stat --printf="%s" $FILE) FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))" echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb" diff --git a/scripts/list_layers.sh b/scripts/list_layers.sh index bb9057d6..3320a486 100755 --- a/scripts/list_layers.sh +++ b/scripts/list_layers.sh @@ -10,7 +10,7 @@ set -e -LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39") +LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') LAYERS_MISSING_REGIONS=() diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index 7fda18ea..e1a601e4 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -13,9 +13,10 @@ set -e # Makes sure any subprocesses will be terminated with this process trap "pkill -P $$; exit 1;" INT -PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.9") -LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip" ".layers/datadog_lambda_py3.9.zip") -AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39") +PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.8" "python3.9" "python3.9") +LAYER_PATHS=(".layers/datadog_lambda_py-amd64-2.7.zip" ".layers/datadog_lambda_py-amd64-3.6.zip" ".layers/datadog_lambda_py-amd64-3.7.zip" ".layers/datadog_lambda_py-amd64-3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip" ".layers/datadog_lambda_py-amd64-3.9.zip" ".layers/datadog_lambda_py-arm64-3.9.zip") +AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM") +ARCHS=("amd64" "amd64" "amd64""amd64" "amd64" "arm64") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') # Check that the layer files exist diff --git a/scripts/sign_layers.sh b/scripts/sign_layers.sh index be7d7e9a..ab2c40a0 100755 --- a/scripts/sign_layers.sh +++ b/scripts/sign_layers.sh @@ -9,11 +9,13 @@ set -e LAYER_DIR=".layers" LAYER_FILES=( - "datadog_lambda_py2.7.zip" - "datadog_lambda_py3.6.zip" - "datadog_lambda_py3.7.zip" - "datadog_lambda_py3.8.zip" - "datadog_lambda_py3.9.zip" + "datadog_lambda_py-amd64-2.7.zip" + "datadog_lambda_py-amd64-3.6.zip" + "datadog_lambda_py-amd64-3.7.zip" + "datadog_lambda_py-amd64-3.8.zip" + "datadog_lambda_py-arm64-3.8.zip" + "datadog_lambda_py-amd64-3.9.zip" + "datadog_lambda_py-arm64-3.9.zip" ) SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile" diff --git a/tests/integration/serverless.yml b/tests/integration/serverless.yml index 01cf2389..c947deff 100644 --- a/tests/integration/serverless.yml +++ b/tests/integration/serverless.yml @@ -27,7 +27,7 @@ provider: layers: python: package: - artifact: ../../.layers/datadog_lambda_py${env:PYTHON_VERSION}.zip + artifact: ../../.layers/datadog_lambda_py-amd64-${env:PYTHON_VERSION}.zip functions: # async-metrics (flushed to logs) @@ -46,4 +46,4 @@ functions: handler: handle.handle runtime: ${env:SERVERLESS_RUNTIME} layers: - - { Ref: PythonLambdaLayer } \ No newline at end of file + - { Ref: PythonLambdaLayer }