Skip to content

Commit c7ef484

Browse files
nhulstonnhulston
committed
automate layer releases
1 parent 186f733 commit c7ef484

File tree

11 files changed

+510
-0
lines changed

11 files changed

+510
-0
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,5 @@ tests/testfunctions/bin/
4040
tests/testfunctions/.serverless/
4141
tests/testfunctions/.settings/
4242
tests/testfunctions/build.gradle
43+
44+
.gitlab/build-pipeline.yaml

‎.gitlab-ci.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
variables:
2+
DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-java
3+
DOCKER_TARGET_VERSION: latest
4+
# Default version for development builds
5+
# This will be overwritten by the tag version if it is a release.
6+
VERSION: dev
7+
8+
stages:
9+
- pre
10+
- build
11+
12+
.go-cache: &go-cache
13+
key: datadog-lambda-java-go-cache
14+
policy: pull
15+
16+
ci image:
17+
stage: build
18+
image: registry.ddbuild.io/images/docker:20.10
19+
tags: ["arch:arm64"]
20+
rules:
21+
- if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'
22+
changes:
23+
- .gitlab/Dockerfile
24+
when: on_success
25+
variables:
26+
DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION}
27+
script:
28+
- docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile .
29+
30+
generator:
31+
stage: pre
32+
image: registry.ddbuild.io/images/mirror/golang:alpine
33+
tags: ["arch:amd64"]
34+
cache: *go-cache
35+
rules:
36+
- if: '$CI_PIPELINE_SOURCE =~ /external_pull_request_event|merge_request_event|push/'
37+
when: never
38+
- when: always
39+
script:
40+
- if [[ "$CI_COMMIT_TAG" =~ ^v[0-9]+$ ]]; then echo "VERSION=${CI_COMMIT_TAG//[!0-9]/}" >> .env; fi
41+
- apk add --no-cache gomplate
42+
- gomplate --config .gitlab/config.yaml
43+
artifacts:
44+
paths:
45+
- .gitlab/build-pipeline.yaml
46+
reports:
47+
dotenv: .env
48+
49+
build:
50+
stage: build
51+
needs: ["generator"]
52+
trigger:
53+
include:
54+
- artifact: .gitlab/build-pipeline.yaml
55+
job: generator
56+
strategy: depend
57+
needs:
58+
- job: generator
59+
artifacts: true
60+
rules:
61+
- if: '$CI_PIPELINE_SOURCE =~ /external_pull_request_event|merge_request_event|push/'
62+
when: never
63+
- when: always
64+
variables:
65+
VERSION: $VERSION

‎.gitlab/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM registry.ddbuild.io/images/docker:24.0.5
2+
3+
RUN apt-get update && apt-get install -y --no-install-recommends \
4+
curl gnupg unzip zip jq
5+
6+
# Install AWS CLI
7+
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
8+
RUN unzip awscliv2.zip && ./aws/install

‎.gitlab/config.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# gomplate template generation pipeline
2+
3+
inputFiles:
4+
- .gitlab/template.yaml.tpl
5+
6+
outputFiles:
7+
- .gitlab/build-pipeline.yaml
8+
9+
datasources:
10+
environments:
11+
url: .gitlab/datasources/environments.yaml
12+
13+
regions:
14+
url: .gitlab/datasources/regions.yaml

‎.gitlab/datasources/environments.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
environments:
2+
- name: sandbox
3+
external_id: sandbox-publish-externalid
4+
role_to_assume: sandbox-layer-deployer
5+
account: 425362996713
6+
- name: prod
7+
external_id: prod-publish-externalid
8+
role_to_assume: dd-serverless-layer-deployer-role
9+
account: 464622532012

‎.gitlab/datasources/regions.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
regions:
2+
- code: "us-east-1"
3+
- code: "us-east-2"
4+
- code: "us-west-1"
5+
- code: "us-west-2"
6+
- code: "af-south-1"
7+
- code: "ap-east-1"
8+
- code: "ap-south-1"
9+
- code: "ap-south-2"
10+
- code: "ap-southeast-1"
11+
- code: "ap-southeast-2"
12+
- code: "ap-southeast-3"
13+
- code: "ap-southeast-4"
14+
- code: "ap-southeast-5"
15+
- code: "ap-northeast-1"
16+
- code: "ap-northeast-2"
17+
- code: "ap-northeast-3"
18+
- code: "ca-central-1"
19+
- code: "ca-west-1"
20+
- code: "eu-central-1"
21+
- code: "eu-central-2"
22+
- code: "eu-north-1"
23+
- code: "eu-west-1"
24+
- code: "eu-west-2"
25+
- code: "eu-west-3"
26+
- code: "eu-south-1"
27+
- code: "eu-south-2"
28+
- code: "il-central-1"
29+
- code: "me-south-1"
30+
- code: "me-central-1"
31+
- code: "sa-east-1"

‎.gitlab/scripts/get_secrets.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2024 Datadog, Inc.
7+
8+
set -e
9+
10+
if [ -z "$EXTERNAL_ID_NAME" ]; then
11+
printf "[Error] No EXTERNAL_ID_NAME found.\n"
12+
printf "Exiting script...\n"
13+
exit 1
14+
fi
15+
16+
if [ -z "$ROLE_TO_ASSUME" ]; then
17+
printf "[Error] No ROLE_TO_ASSUME found.\n"
18+
printf "Exiting script...\n"
19+
exit 1
20+
fi
21+
22+
printf "Getting AWS External ID...\n"
23+
24+
EXTERNAL_ID=$(aws ssm get-parameter \
25+
--region us-east-1 \
26+
--name "ci.datadog-lambda-java.$EXTERNAL_ID_NAME" \
27+
--with-decryption \
28+
--query "Parameter.Value" \
29+
--out text)
30+
31+
printf "Getting DD API KEY...\n"
32+
33+
export DD_API_KEY=$(aws ssm get-parameter \
34+
--region us-east-1 \
35+
--name ci.datadog-lambda-java.dd-api-key \
36+
--with-decryption \
37+
--query "Parameter.Value" \
38+
--out text)
39+
40+
printf "Assuming role...\n"
41+
42+
export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
43+
$(aws sts assume-role \
44+
--role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME" \
45+
--role-session-name "ci.datadog-lambda-java-$CI_JOB_ID-$CI_JOB_STAGE" \
46+
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
47+
--external-id $EXTERNAL_ID \
48+
--output text))

‎.gitlab/scripts/publish_layers.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2024 Datadog, Inc.
7+
8+
set -e
9+
10+
LAYER_NAME="dd-trace-java"
11+
12+
publish_layer() {
13+
region=$1
14+
15+
version_nbr=$(aws lambda publish-layer-version --layer-name $LAYER_NAME \
16+
--description "Datadog Tracer Lambda Layer for Java" \
17+
--compatible-runtimes "java8" "java11" "java17" "java21" \
18+
--compatible-architectures "x86_64" "arm64" \
19+
--zip-file "fileb://.layers/tracer.zip" \
20+
--region $region \
21+
| jq -r '.Version')
22+
23+
# Add permissions only for prod
24+
if [ "$STAGE" == "prod" ]; then
25+
permission=$(aws lambda add-layer-version-permission --layer-name $LAYER_NAME \
26+
--version-number $version_nbr \
27+
--statement-id "release-$version_nbr" \
28+
--action lambda:GetLayerVersion \
29+
--principal "*" \
30+
--region $region
31+
)
32+
fi
33+
34+
echo $version_nbr
35+
}
36+
37+
38+
if [ ! -f ".layers/tracer.zip" ]; then
39+
printf "[ERROR]: Could not find .layers/tracer.zip."
40+
exit 1
41+
fi
42+
43+
AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
44+
45+
if [ -z "$REGION" ]; then
46+
printf "[ERROR]: REGION not specified."
47+
exit 1
48+
else
49+
printf "Region specified: $REGION\n"
50+
if [[ ! "$AVAILABLE_REGIONS" == *"$REGION"* ]]; then
51+
printf "Could not find $REGION in available regions: $AVAILABLE_REGIONS"
52+
exit 1
53+
fi
54+
fi
55+
56+
if [ -z "$STAGE" ]; then
57+
printf "[ERROR]: STAGE not specified.\n"
58+
exit 1
59+
fi
60+
61+
printf "[$REGION] Starting publishing layers...\n"
62+
63+
if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then
64+
# Deploy latest version
65+
latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`')
66+
VERSION=$(($latest_version + 1))
67+
else
68+
# Running on prod
69+
if [ -z "$CI_COMMIT_TAG" ]; then
70+
printf "[ERROR]: No CI_COMMIT_TAG found.\n"
71+
printf "Exiting script...\n"
72+
exit 1
73+
else
74+
printf "Tag found in environment: $CI_COMMIT_TAG\n"
75+
fi
76+
77+
VERSION="${CI_COMMIT_TAG//[!0-9]/}"
78+
printf "Version: ${VERSION}\n"
79+
fi
80+
81+
if [ -z "$VERSION" ]; then
82+
printf "[ERROR]: Layer VERSION not specified"
83+
exit 1
84+
else
85+
printf "Layer version parsed: $VERSION\n"
86+
fi
87+
88+
latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`')
89+
if [ $latest_version -ge $VERSION ]; then
90+
printf "[$REGION] Layer $LAYER_NAME version $VERSION already exists in region $REGION, skipping...\n"
91+
exit 1
92+
elif [ $latest_version -lt $((VERSION-1)) ]; then
93+
printf "[$REGION][WARNING] The latest version of layer $LAYER_NAME in region $REGION is $latest_version, this will publish all the missing versions including $VERSION\n"
94+
fi
95+
96+
while [ $latest_version -lt $VERSION ]; do
97+
latest_version=$(publish_layer $REGION)
98+
printf "[$REGION] Published version $latest_version of layer $LAYER_NAME in region $REGION\n"
99+
100+
# This shouldn't happen unless someone manually deleted the latest version, say 28, and
101+
# then tries to republish 28 again. The published version would actually be 29, because
102+
# Lambda layers are immutable and AWS will skip deleted version and use the next number.
103+
if [ $latest_version -gt $VERSION ]; then
104+
printf "[$REGION] Published version $latest_version is greater than the desired version $VERSION!"
105+
exit 1
106+
fi
107+
done
108+
109+
printf "[$REGION] Finished publishing layer...\n\n"

‎.gitlab/scripts/pull_and_zip_layers.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2024 Datadog, Inc.
7+
8+
set -e
9+
10+
rm -f dd-java-agent.jar
11+
wget -O dd-java-agent.jar https://dtdg.co/latest-java-tracer
12+
13+
rm -rf .layers
14+
mkdir .layers
15+
16+
rm -rf layer-temp
17+
mkdir -p layer-temp/java/lib
18+
19+
cp dd-java-agent.jar layer-temp/java/lib/dd-java-agent.jar
20+
cd layer-temp
21+
zip -r ../.layers/tracer.zip java
22+
cd ..
23+
rm -rf layer-temp

0 commit comments

Comments
 (0)