From 2d303856294b76c7a509677daab8ea4ccf6bc23b Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 16 Sep 2021 15:43:47 -0400 Subject: [PATCH 1/6] Add arm publishing code --- scripts/build_layers.sh | 18 +++++++++++++----- scripts/check_layer_size.sh | 2 +- scripts/list_layers.sh | 2 +- scripts/publish_layers.sh | 7 ++++--- scripts/sign_layers.sh | 9 +++++---- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index 0c4f80d9..eb78110f 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -40,17 +40,21 @@ function docker_build_zip { # Args: [python version] [zip destination] destination=$(make_path_absolute $2) + arch=$3 # Install datadogpy in a docker container to avoid the mess from switching # between different python runtimes. temp_dir=$(mktemp -d) - docker build -t datadog-lambda-python:$1 . --no-cache \ + docker buildx build -t datadog-lambda-python-${arch}:$1 . --no-cache \ --build-arg image=python:$1 \ - --build-arg runtime=python$1 + --build-arg runtime=python$1 \ + --platform linux/${arch} \ + --load # Run the image by runtime tag, tar its generatd `python` directory to sdout, # then extract it to a temp directory. - docker run datadog-lambda-python:$1 tar cf - python | tar -xf - -C $temp_dir + docker run datadog-lambda-python-${arch}:$1 tar cf - python | tar -xf - -C $temp_dir + # Zip to destination, and keep directory structure as based in $temp_dir (cd $temp_dir && zip -q -r $destination ./) @@ -64,8 +68,12 @@ mkdir $LAYER_DIR for python_version in "${PYTHON_VERSIONS[@]}" do - echo "Building layer for Python ${python_version}" - docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}${python_version}.zip + if [ "$python_version" == "3.8" ]; then + echo "Building layer for Python ${python_version} arch=arm64" + docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-arm64-${python_version}.zip arm64 + fi + echo "Building layer for Python ${python_version} arch=amd64" + docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${python_version}.zip amd64 done echo "Done creating layers:" diff --git a/scripts/check_layer_size.sh b/scripts/check_layer_size.sh index ad807416..54090b2a 100755 --- a/scripts/check_layer_size.sh +++ b/scripts/check_layer_size.sh @@ -18,7 +18,7 @@ VERSIONS=("2.7" "3.6" "3.7" "3.8") for version in "${VERSIONS[@]}" do - FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}${version}.zip + FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${version}.zip FILE_SIZE=$(stat --printf="%s" $FILE) FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))" echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb" diff --git a/scripts/list_layers.sh b/scripts/list_layers.sh index 4162bf32..b87163d7 100755 --- a/scripts/list_layers.sh +++ b/scripts/list_layers.sh @@ -10,7 +10,7 @@ set -e -LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38") +LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python36") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') LAYERS_MISSING_REGIONS=() diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index a361355d..82a646b7 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -13,9 +13,10 @@ set -e # Makes sure any subprocesses will be terminated with this process trap "pkill -P $$; exit 1;" INT -PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8") -LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip") -AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38") +PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.8") +LAYER_PATHS=(".layers/datadog_lambda_py-amd64-2.7.zip" ".layers/datadog_lambda_py-amd64-3.6.zip" ".layers/datadog_lambda_py-amd64-3.7.zip" ".layers/datadog_lambda_py3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip") +AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM") +ARCHS=("amd64" "amd64" "amd64""amd64" "amd64" "arm64") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') # Check that the layer files exist diff --git a/scripts/sign_layers.sh b/scripts/sign_layers.sh index 554e406d..9a0a6368 100755 --- a/scripts/sign_layers.sh +++ b/scripts/sign_layers.sh @@ -9,10 +9,11 @@ set -e LAYER_DIR=".layers" LAYER_FILES=( - "datadog_lambda_py2.7.zip" - "datadog_lambda_py3.6.zip" - "datadog_lambda_py3.7.zip" - "datadog_lambda_py3.8.zip" + "datadog_lambda_py-amd64-2.7.zip" + "datadog_lambda_py-amd64-3.6.zip" + "datadog_lambda_py-amd64-3.7.zip" + "datadog_lambda_py-amd64-3.8.zip" + "datadog_lambda_py-arm64-3.8.zip" ) SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile" From b05a2270edb48428ccbd5b837eca77b42301e598 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 29 Sep 2021 11:20:10 -0400 Subject: [PATCH 2/6] Fix issues with ARM 3.9 config in build scripts --- scripts/build_layers.sh | 2 +- scripts/publish_layers.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index b3f44ab8..298ad48c 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -68,7 +68,7 @@ mkdir $LAYER_DIR for python_version in "${PYTHON_VERSIONS[@]}" do - if [ "$python_version" == "3.8" ]; then + if [ "$python_version" == "3.8" ] || [ "$python_version" == "3.9" ]; then echo "Building layer for Python ${python_version} arch=arm64" docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-arm64-${python_version}.zip arm64 fi diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index 01f3bcd6..e1a601e4 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -14,7 +14,7 @@ set -e trap "pkill -P $$; exit 1;" INT PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.8" "python3.9" "python3.9") -LAYER_PATHS=(".layers/datadog_lambda_py-amd64-2.7.zip" ".layers/datadog_lambda_py-amd64-3.6.zip" ".layers/datadog_lambda_py-amd64-3.7.zip" ".layers/datadog_lambda_py3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip" ".layers/datadog_lambda_py3.9.zip" ".layers/datadog_lambda_py-arm64-3.9.zip") +LAYER_PATHS=(".layers/datadog_lambda_py-amd64-2.7.zip" ".layers/datadog_lambda_py-amd64-3.6.zip" ".layers/datadog_lambda_py-amd64-3.7.zip" ".layers/datadog_lambda_py-amd64-3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip" ".layers/datadog_lambda_py-amd64-3.9.zip" ".layers/datadog_lambda_py-arm64-3.9.zip") AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM") ARCHS=("amd64" "amd64" "amd64""amd64" "amd64" "arm64") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') From b20997cf0d92c310225fff57385bfdbddc26bc4a Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 29 Sep 2021 18:25:45 -0400 Subject: [PATCH 3/6] Update integration tests --- tests/integration/serverless.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/serverless.yml b/tests/integration/serverless.yml index 01cf2389..c947deff 100644 --- a/tests/integration/serverless.yml +++ b/tests/integration/serverless.yml @@ -27,7 +27,7 @@ provider: layers: python: package: - artifact: ../../.layers/datadog_lambda_py${env:PYTHON_VERSION}.zip + artifact: ../../.layers/datadog_lambda_py-amd64-${env:PYTHON_VERSION}.zip functions: # async-metrics (flushed to logs) @@ -46,4 +46,4 @@ functions: handler: handle.handle runtime: ${env:SERVERLESS_RUNTIME} layers: - - { Ref: PythonLambdaLayer } \ No newline at end of file + - { Ref: PythonLambdaLayer } From 25eaff7c32d1c309cf358959f59a82c7c74f3029 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 29 Sep 2021 18:46:28 -0400 Subject: [PATCH 4/6] Add cross-compiling deps to ubuntu build job --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c569a53..31200db7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ jobs: - name: Install dependencies run: | + sudo apt install -y qemu-user-static binfmt-support pip install virtualenv virtualenv venv source venv/bin/activate From b81b1854e5ae0ef3deeaf0e86f1dc7c7ab1767fe Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 29 Sep 2021 18:50:51 -0400 Subject: [PATCH 5/6] Fix install --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 31200db7..de18cc4d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,6 @@ jobs: - name: Install dependencies run: | - sudo apt install -y qemu-user-static binfmt-support pip install virtualenv virtualenv venv source venv/bin/activate @@ -84,6 +83,8 @@ jobs: - name: Install Serverless Framework run: sudo yarn global add serverless --prefix /usr/local + - name: Install Crossbuild Deps + run: sudo apt install -y qemu-user-static binfmt-support - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' From ca8932a7675506d96c79b319ac08092ebbc92abd Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 29 Sep 2021 19:01:54 -0400 Subject: [PATCH 6/6] Fix check-size --- .github/workflows/check-size.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/check-size.yml b/.github/workflows/check-size.yml index 2ce7e816..69d578d0 100644 --- a/.github/workflows/check-size.yml +++ b/.github/workflows/check-size.yml @@ -14,6 +14,9 @@ jobs: with: python-version: 3.7 + - name: Install Crossbuild Deps + run: sudo apt install -y qemu-user-static binfmt-support + - name: Install dependencies run: | pip install virtualenv