Skip to content

Commit 9d003c8

Browse files
Darcy.rayner/add arm publishing (#182)
* Add arm publishing code * Fix issues with ARM 3.9 config in build scripts * Update integration tests * Add cross-compiling deps to ubuntu build job * Fix install * Fix check-size
1 parent c86be18 commit 9d003c8

File tree

8 files changed

+33
-17
lines changed

8 files changed

+33
-17
lines changed

.github/workflows/build.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ jobs:
8383

8484
- name: Install Serverless Framework
8585
run: sudo yarn global add serverless --prefix /usr/local
86+
- name: Install Crossbuild Deps
87+
run: sudo apt install -y qemu-user-static binfmt-support
8688

8789
- name: Install dependencies
8890
if: steps.cache-node-modules.outputs.cache-hit != 'true'

.github/workflows/check-size.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ jobs:
1414
with:
1515
python-version: 3.7
1616

17+
- name: Install Crossbuild Deps
18+
run: sudo apt install -y qemu-user-static binfmt-support
19+
1720
- name: Install dependencies
1821
run: |
1922
pip install virtualenv

scripts/build_layers.sh

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,21 @@ function docker_build_zip {
4040
# Args: [python version] [zip destination]
4141

4242
destination=$(make_path_absolute $2)
43+
arch=$3
4344

4445
# Install datadogpy in a docker container to avoid the mess from switching
4546
# between different python runtimes.
4647
temp_dir=$(mktemp -d)
47-
docker build -t datadog-lambda-python:$1 . --no-cache \
48+
docker buildx build -t datadog-lambda-python-${arch}:$1 . --no-cache \
4849
--build-arg image=python:$1 \
49-
--build-arg runtime=python$1
50+
--build-arg runtime=python$1 \
51+
--platform linux/${arch} \
52+
--load
5053

5154
# Run the image by runtime tag, tar its generatd `python` directory to sdout,
5255
# then extract it to a temp directory.
53-
docker run datadog-lambda-python:$1 tar cf - python | tar -xf - -C $temp_dir
56+
docker run datadog-lambda-python-${arch}:$1 tar cf - python | tar -xf - -C $temp_dir
57+
5458

5559
# Zip to destination, and keep directory structure as based in $temp_dir
5660
(cd $temp_dir && zip -q -r $destination ./)
@@ -64,8 +68,12 @@ mkdir $LAYER_DIR
6468

6569
for python_version in "${PYTHON_VERSIONS[@]}"
6670
do
67-
echo "Building layer for Python ${python_version}"
68-
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}${python_version}.zip
71+
if [ "$python_version" == "3.8" ] || [ "$python_version" == "3.9" ]; then
72+
echo "Building layer for Python ${python_version} arch=arm64"
73+
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-arm64-${python_version}.zip arm64
74+
fi
75+
echo "Building layer for Python ${python_version} arch=amd64"
76+
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${python_version}.zip amd64
6977
done
7078

7179
echo "Done creating layers:"

scripts/check_layer_size.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ VERSIONS=("2.7" "3.6" "3.7" "3.8" "3.9")
1818

1919
for version in "${VERSIONS[@]}"
2020
do
21-
FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}${version}.zip
21+
FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${version}.zip
2222
FILE_SIZE=$(stat --printf="%s" $FILE)
2323
FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))"
2424
echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb"

scripts/list_layers.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
set -e
1212

13-
LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
13+
LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM")
1414
AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
1515
LAYERS_MISSING_REGIONS=()
1616

scripts/publish_layers.sh

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@ set -e
1313
# Makes sure any subprocesses will be terminated with this process
1414
trap "pkill -P $$; exit 1;" INT
1515

16-
PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.9")
17-
LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip" ".layers/datadog_lambda_py3.9.zip")
18-
AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
16+
PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.8" "python3.9" "python3.9")
17+
LAYER_PATHS=(".layers/datadog_lambda_py-amd64-2.7.zip" ".layers/datadog_lambda_py-amd64-3.6.zip" ".layers/datadog_lambda_py-amd64-3.7.zip" ".layers/datadog_lambda_py-amd64-3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip" ".layers/datadog_lambda_py-amd64-3.9.zip" ".layers/datadog_lambda_py-arm64-3.9.zip")
18+
AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM")
19+
ARCHS=("amd64" "amd64" "amd64""amd64" "amd64" "arm64")
1920
AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
2021

2122
# Check that the layer files exist

scripts/sign_layers.sh

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ set -e
99

1010
LAYER_DIR=".layers"
1111
LAYER_FILES=(
12-
"datadog_lambda_py2.7.zip"
13-
"datadog_lambda_py3.6.zip"
14-
"datadog_lambda_py3.7.zip"
15-
"datadog_lambda_py3.8.zip"
16-
"datadog_lambda_py3.9.zip"
12+
"datadog_lambda_py-amd64-2.7.zip"
13+
"datadog_lambda_py-amd64-3.6.zip"
14+
"datadog_lambda_py-amd64-3.7.zip"
15+
"datadog_lambda_py-amd64-3.8.zip"
16+
"datadog_lambda_py-arm64-3.8.zip"
17+
"datadog_lambda_py-amd64-3.9.zip"
18+
"datadog_lambda_py-arm64-3.9.zip"
1719
)
1820
SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile"
1921

tests/integration/serverless.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ provider:
2727
layers:
2828
python:
2929
package:
30-
artifact: ../../.layers/datadog_lambda_py${env:PYTHON_VERSION}.zip
30+
artifact: ../../.layers/datadog_lambda_py-amd64-${env:PYTHON_VERSION}.zip
3131

3232
functions:
3333
# async-metrics (flushed to logs)
@@ -46,4 +46,4 @@ functions:
4646
handler: handle.handle
4747
runtime: ${env:SERVERLESS_RUNTIME}
4848
layers:
49-
- { Ref: PythonLambdaLayer }
49+
- { Ref: PythonLambdaLayer }

0 commit comments

Comments
 (0)