From c65b8fbc02567681f7fff2dde981f93468ab3a74 Mon Sep 17 00:00:00 2001 From: maxday Date: Wed, 28 Apr 2021 13:25:22 -0400 Subject: [PATCH 1/5] integration tests run faster --- .github/workflows/build.yml | 5 +- scripts/run_integration_tests.sh | 85 +++++++++++++++++++++++--------- tests/integration/serverless.yml | 69 ++++---------------------- 3 files changed, 76 insertions(+), 83 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d07bac58..7f81e4cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,7 +62,9 @@ jobs: integration-test: runs-on: ubuntu-latest - + strategy: + matrix: + runtime-param: [27, 36, 37, 38] steps: - name: Checkout uses: actions/checkout@v2 @@ -93,4 +95,5 @@ jobs: DD_API_KEY: ${{ secrets.DD_API_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RUNTIME_PARAM: ${{ matrix.runtime-param }} run: ./scripts/run_integration_tests.sh diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 492a8431..4a9ba9fe 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -24,6 +24,26 @@ script_utc_start_time=$(date -u +"%Y%m%dT%H%M%S") mismatch_found=false +# Format : +# [0]: serverless runtime name +# [1]: nodejs version +# [2]: random 8-character ID to avoid collisions with other runs +python27=("python2.7" "2.7" $(xxd -l 4 -c 4 -p < /dev/random)) +python36=("python3.6" "3.6" $(xxd -l 4 -c 4 -p < /dev/random)) +python37=("python3.7" "3.7" $(xxd -l 4 -c 4 -p < /dev/random)) +python38=("python3.8" "3.8" $(xxd -l 4 -c 4 -p < /dev/random)) + +PARAMETERS_SETS=("python27" "python36" "python37" "python38") + +if [ -z "$RUNTIME_PARAM" ]; then + echo "Python version not specified, running for all python versions." +else + echo "Python version is specified: $RUNTIME_PARAM" + PARAMETERS_SETS=(python${RUNTIME_PARAM}) + BUILD_LAYER_VERSION=python$RUNTIME_PARAM[1] +fi + + if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then echo "No AWS credentials were found in the environment." echo "Note that only Datadog employees can run these integration tests." @@ -41,7 +61,7 @@ fi if [ -n "$BUILD_LAYERS" ]; then echo "Building layers that will be deployed with our test functions" - source $scripts_dir/build_layers.sh + PYTHON_VERSION=${!BUILD_LAYER_VERSION} source $scripts_dir/build_layers.sh else echo "Not building layers, ensure they've already been built or re-run with 'BUILD_LAYERS=true DD_API_KEY=XXXX ./scripts/run_integration_tests.sh'" fi @@ -52,32 +72,48 @@ input_event_files=$(ls ./input_events) # Sort event files by name so that snapshots stay consistent input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort)) -# Generate a random 8-character ID to avoid collisions with other runs -run_id=$(xxd -l 4 -c 4 -p < /dev/random) - -# Always remove the stack before exiting, no matter what +# Always remove the stacks before exiting, no matter what function remove_stack() { - echo "Removing functions" - serverless remove --stage $run_id + for parameters_set in "${PARAMETERS_SETS[@]}"; do + serverless_runtime=$parameters_set[0] + python_version=$parameters_set[1] + run_id=$parameters_set[2] + echo "Removing stack for stage : ${!run_id}" + PYTHON_VERSION=${!python_version} RUNTIME=$parameters_set SERVERLESS_RUNTIME=${!serverless_runtime} \ + serverless remove --stage ${!run_id} + done } + + + trap remove_stack EXIT -echo "Deploying functions" -serverless deploy --stage $run_id +for parameters_set in "${PARAMETERS_SETS[@]}"; do + + serverless_runtime=$parameters_set[0] + python_version=$parameters_set[1] + run_id=$parameters_set[2] -echo "Invoking functions" -set +e # Don't exit this script if an invocation fails or there's a diff -for handler_name in "${LAMBDA_HANDLERS[@]}"; do - for runtime in "${RUNTIMES[@]}"; do - function_name="${handler_name}_${runtime}" + echo "Deploying functions for runtime : $parameters_set, serverless runtime : ${!serverless_runtime}, \ +python version : ${!python_version} and run id : ${!run_id}" + + PYTHON_VERSION=${!python_version} RUNTIME=$parameters_set SERVERLESS_RUNTIME=${!serverless_runtime} \ + serverless deploy --stage ${!run_id} + echo "Invoking functions for runtime $parameters_set" + set +e # Don't exit this script if an invocation fails or there's a diff + for handler_name in "${LAMBDA_HANDLERS[@]}"; do + + function_name="${handler_name}_python" + echo "$function_name" # Invoke function once for each input event for input_event_file in "${input_event_files[@]}"; do # Get event name without trailing ".json" so we can build the snapshot file name input_event_name=$(echo "$input_event_file" | sed "s/.json//") - snapshot_path="./snapshots/return_values/${function_name}_${input_event_name}.json" + snapshot_path="./snapshots/return_values/${handler_name}_${parameters_set}_${input_event_name}.json" - return_value=$(serverless invoke -f $function_name --stage $run_id --path "./input_events/$input_event_file") + return_value=$(PYTHON_VERSION=${!python_version} RUNTIME=$parameters_set SERVERLESS_RUNTIME=${!serverless_runtime} \ + serverless invoke --stage ${!run_id} -f "$function_name" --path "./input_events/$input_event_file") if [ ! -f $snapshot_path ]; then # If the snapshot file doesn't exist yet, we create it @@ -109,14 +145,17 @@ sleep $LOGS_WAIT_SECONDS set +e # Don't exit this script if there is a diff or the logs endpoint fails echo "Fetching logs for invocations and comparing to snapshots" for handler_name in "${LAMBDA_HANDLERS[@]}"; do - for runtime in "${RUNTIMES[@]}"; do - function_name="${handler_name}_${runtime}" - function_snapshot_path="./snapshots/logs/$function_name.log" - + for parameters_set in "${PARAMETERS_SETS[@]}"; do + function_name="${handler_name}_python" + function_snapshot_path="./snapshots/logs/${handler_name}_${parameters_set}.log" + serverless_runtime=$parameters_set[0] + python_version=$parameters_set[1] + run_id=$parameters_set[2] # Fetch logs with serverless cli, retrying to avoid AWS account-wide rate limit error retry_counter=0 while [ $retry_counter -lt 10 ]; do - raw_logs=$(serverless logs -f $function_name --stage $run_id --startTime $script_utc_start_time) + raw_logs=$(PYTHON_VERSION=${!python_version} RUNTIME=$parameters_set SERVERLESS_RUNTIME=${!serverless_runtime} \ + serverless logs --stage ${!run_id} -f $function_name --startTime $script_utc_start_time) fetch_logs_exit_code=$? if [ $fetch_logs_exit_code -eq 1 ]; then echo "Retrying fetch logs for $function_name..." @@ -158,7 +197,7 @@ for handler_name in "${LAMBDA_HANDLERS[@]}"; do sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_)[0-9]+\.[0-9]+\.[0-9]+/\1X\.X\.X/g" | sed -E "s/(datadog_lambda:v)([0-9]+\.[0-9]+\.[0-9])/\1XX/g" | # Strip out run ID (from function name, resource, etc.) - sed -E "s/$run_id/XXXX/g" | + sed -E "s/${!run_id}/XXXX/g" | # Strip out trace/span/parent/timestamps sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" | sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" | @@ -182,7 +221,7 @@ for handler_name in "${LAMBDA_HANDLERS[@]}"; do echo "$logs" >$function_snapshot_path else # Compare new logs to snapshots - diff_output=$(echo "$logs" | diff - $function_snapshot_path) + diff_output=$(echo "$logs" | sort | diff -w - <(sort $function_snapshot_path)) if [ $? -eq 1 ]; then echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):" echo "$diff_output" diff --git a/tests/integration/serverless.yml b/tests/integration/serverless.yml index 7c659a27..383cb1f1 100644 --- a/tests/integration/serverless.yml +++ b/tests/integration/serverless.yml @@ -19,74 +19,25 @@ provider: role: "arn:aws:iam::601427279990:role/serverless-integration-test-lambda-role" layers: - python27: + python: package: - artifact: ../../.layers/datadog_lambda_py2.7.zip - python36: - package: - artifact: ../../.layers/datadog_lambda_py3.6.zip - python37: - package: - artifact: ../../.layers/datadog_lambda_py3.7.zip - python38: - package: - artifact: ../../.layers/datadog_lambda_py3.8.zip + artifact: ../../.layers/datadog_lambda_py${env:PYTHON_VERSION}.zip functions: # async-metrics (flushed to logs) - async-metrics_python27: - handler: handle.handle - runtime: python2.7 - layers: - - { Ref: Python27LambdaLayer } - environment: - DD_FLUSH_TO_LOG: true - - async-metrics_python36: + async-metrics_python: + name: integration-tests-python-${sls:stage}-async-metrics_${env:RUNTIME} handler: handle.handle - runtime: python3.6 + runtime: ${env:SERVERLESS_RUNTIME} layers: - - { Ref: Python36LambdaLayer } - environment: - DD_FLUSH_TO_LOG: true - - async-metrics_python37: - handler: handle.handle - runtime: python3.7 - layers: - - { Ref: Python37LambdaLayer } - environment: - DD_FLUSH_TO_LOG: true - - async-metrics_python38: - handler: handle.handle - runtime: python3.8 - layers: - - { Ref: Python38LambdaLayer } + - { Ref: PythonLambdaLayer } environment: DD_FLUSH_TO_LOG: true # sync-metrics (sent via API) - sync-metrics_python27: - handler: handle.handle - runtime: python2.7 - layers: - - { Ref: Python27LambdaLayer } - - sync-metrics_python36: - handler: handle.handle - runtime: python3.6 - layers: - - { Ref: Python36LambdaLayer } - - sync-metrics_python37: - handler: handle.handle - runtime: python3.7 - layers: - - { Ref: Python37LambdaLayer } - - sync-metrics_python38: + sync-metrics_python: + name: integration-tests-python-${sls:stage}-sync-metrics_${env:RUNTIME} handler: handle.handle - runtime: python3.8 + runtime: ${env:SERVERLESS_RUNTIME} layers: - - { Ref: Python38LambdaLayer } + - { Ref: PythonLambdaLayer } \ No newline at end of file From feeb172a41bba1dbfb0828808afb5143fbf33c1c Mon Sep 17 00:00:00 2001 From: maxday Date: Wed, 28 Apr 2021 13:35:53 -0400 Subject: [PATCH 2/5] typo --- scripts/run_integration_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 4a9ba9fe..298209d2 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -26,7 +26,7 @@ mismatch_found=false # Format : # [0]: serverless runtime name -# [1]: nodejs version +# [1]: python version # [2]: random 8-character ID to avoid collisions with other runs python27=("python2.7" "2.7" $(xxd -l 4 -c 4 -p < /dev/random)) python36=("python3.6" "3.6" $(xxd -l 4 -c 4 -p < /dev/random)) @@ -221,7 +221,7 @@ for handler_name in "${LAMBDA_HANDLERS[@]}"; do echo "$logs" >$function_snapshot_path else # Compare new logs to snapshots - diff_output=$(echo "$logs" | sort | diff -w - <(sort $function_snapshot_path)) + diff_output=$(echo "$logs" | diff - $function_snapshot_path) if [ $? -eq 1 ]; then echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):" echo "$diff_output" From 5bef2063b166ef36adfecd158061a3834f9a54cd Mon Sep 17 00:00:00 2001 From: maxday Date: Wed, 28 Apr 2021 14:03:24 -0400 Subject: [PATCH 3/5] typo post review --- scripts/run_integration_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 298209d2..63ce53dc 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -72,7 +72,7 @@ input_event_files=$(ls ./input_events) # Sort event files by name so that snapshots stay consistent input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort)) -# Always remove the stacks before exiting, no matter what +# Always remove the stack(s) before exiting, no matter what function remove_stack() { for parameters_set in "${PARAMETERS_SETS[@]}"; do serverless_runtime=$parameters_set[0] From 002ee5d76f36ad0e53e7efd8151a8c53b0736293 Mon Sep 17 00:00:00 2001 From: maxday Date: Wed, 28 Apr 2021 14:08:54 -0400 Subject: [PATCH 4/5] add dot in matrix --- .github/workflows/build.yml | 2 +- scripts/run_integration_tests.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f81e4cc..0f7da714 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - runtime-param: [27, 36, 37, 38] + runtime-param: [2.7, 3.6, 3.7, 3.8] steps: - name: Checkout uses: actions/checkout@v2 diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 63ce53dc..110bea5c 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -38,9 +38,10 @@ PARAMETERS_SETS=("python27" "python36" "python37" "python38") if [ -z "$RUNTIME_PARAM" ]; then echo "Python version not specified, running for all python versions." else - echo "Python version is specified: $RUNTIME_PARAM" - PARAMETERS_SETS=(python${RUNTIME_PARAM}) - BUILD_LAYER_VERSION=python$RUNTIME_PARAM[1] + RUNTIME_PARAM_NO_DOT=$(echo $RUNTIME_PARAM | sed 's/\.//') + echo "Python version is specified: $RUNTIME_PARAM_NO_DOT" + PARAMETERS_SETS=(python${RUNTIME_PARAM_NO_DOT}) + BUILD_LAYER_VERSION=python$RUNTIME_PARAM_NO_DOT[1] fi From 0ba5a212788b2de28ec2a797560447acf0652248 Mon Sep 17 00:00:00 2001 From: maxday Date: Wed, 28 Apr 2021 14:17:56 -0400 Subject: [PATCH 5/5] log version with dot --- scripts/run_integration_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 110bea5c..630ab0aa 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -39,7 +39,7 @@ if [ -z "$RUNTIME_PARAM" ]; then echo "Python version not specified, running for all python versions." else RUNTIME_PARAM_NO_DOT=$(echo $RUNTIME_PARAM | sed 's/\.//') - echo "Python version is specified: $RUNTIME_PARAM_NO_DOT" + echo "Python version is specified: $RUNTIME_PARAM" PARAMETERS_SETS=(python${RUNTIME_PARAM_NO_DOT}) BUILD_LAYER_VERSION=python$RUNTIME_PARAM_NO_DOT[1] fi