Skip to content

Commit 9755f4a

Browse files
authored
GODRIVER-2928 Use AWS Secrets Manager for Remaining Secrets (mongodb#1564)
1 parent 9c6603d commit 9755f4a

9 files changed

+98
-163
lines changed

.evergreen/config.yml

Lines changed: 36 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ functions:
305305
script: |
306306
${PREPARE_SHELL}
307307
# Clean up cse servers
308-
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop_servers.sh
308+
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
309309
- command: shell.exec
310310
params:
311311
shell: "bash"
@@ -366,10 +366,10 @@ functions:
366366
params:
367367
shell: "bash"
368368
working_dir: src/go.mongodb.org/mongo-driver
369-
add_expansions_to_env: true
369+
include_expansions_in_env: ["TOPOLOGY", "AUTH", "SSL", "MONGODB_URI", "CRYPT_SHARED_LIB_PATH", "SKIP_CRYPT_SHARED_LIB", "RACE", "MONGO_GO_DRIVER_COMPRESSOR", "REQUIRE_API_VERSION", "LOAD_BALANCER"]
370370
script: |
371371
${PREPARE_SHELL}
372-
sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
372+
bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
373373
374374
create-api-report:
375375
- command: shell.exec
@@ -389,90 +389,32 @@ functions:
389389
file: src/go.mongodb.org/mongo-driver/perf.json
390390

391391
run-enterprise-auth-tests:
392+
- command: ec2.assume_role
393+
params:
394+
role_arn: "${aws_test_secrets_role}"
392395
- command: shell.exec
393396
type: test
394397
params:
395398
shell: "bash"
396399
working_dir: src/go.mongodb.org/mongo-driver
400+
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
397401
script: |
398-
# DO NOT ECHO WITH XTRACE
399-
if [ "Windows_NT" = "$OS" ]; then
400-
export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`))))
401-
export GOCACHE=$(cygpath -w "$(pwd)/.cache")
402-
else
403-
export GOPATH=$(dirname $(dirname $(dirname `pwd`)))
404-
export GOCACHE="$(pwd)/.cache"
405-
fi;
406-
export GOPATH="$GOPATH"
407-
export GOROOT="${GO_DIST}"
408-
export GOCACHE="$GOCACHE"
409-
export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH"
410-
export MONGODB_URI="${MONGODB_URI}"
411-
export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR}"
412-
413-
make -s evg-test-enterprise-auth
402+
${PREPARE_SHELL}
403+
bash ./etc/run-enterprise-plain-test.sh
414404
415405
run-enterprise-gssapi-auth-tests:
416-
- command: shell.exec
417-
type: test
406+
- command: ec2.assume_role
418407
params:
419-
shell: "bash"
420-
silent: true
421-
working_dir: src/go.mongodb.org/mongo-driver
422-
script: |
423-
# DO NOT ECHO WITH XTRACE
424-
if [ "Windows_NT" = "$OS" ]; then
425-
export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`))))
426-
export GOCACHE=$(cygpath -w "$(pwd)/.cache")
427-
export MONGODB_URI=${gssapi_auth_windows_mongodb_uri}
428-
else
429-
export GOPATH=$(dirname $(dirname $(dirname `pwd`)))
430-
export GOCACHE="$(pwd)/.cache"
431-
echo "${gssapi_auth_linux_keytab_base64}" > /tmp/drivers.keytab.base64
432-
base64 --decode /tmp/drivers.keytab.base64 > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab
433-
mkdir -p ~/.krb5
434-
cat .evergreen/krb5.config | tee -a ~/.krb5/config
435-
kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p "${gssapi_auth_username}"
436-
export MONGODB_URI="${gssapi_auth_linux_mongodb_uri}"
437-
fi;
438-
export GOPATH="$GOPATH"
439-
export GOROOT="${GO_DIST}"
440-
export GOCACHE="$GOCACHE"
441-
export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH"
442-
export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR}"
443-
444-
make -s evg-test-enterprise-auth
445-
446-
run-enterprise-gssapi-service-host-auth-tests:
408+
role_arn: "${aws_test_secrets_role}"
447409
- command: shell.exec
448410
type: test
449411
params:
450412
shell: "bash"
451-
silent: true
452413
working_dir: src/go.mongodb.org/mongo-driver
414+
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
453415
script: |
454-
# DO NOT ECHO WITH XTRACE
455-
if [ "Windows_NT" = "$OS" ]; then
456-
export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`))))
457-
export GOCACHE=$(cygpath -w "$(pwd)/.cache")
458-
export MONGODB_URI="${gssapi_service_host_auth_windows_mongodb_uri}"
459-
else
460-
export GOPATH=$(dirname $(dirname $(dirname `pwd`)))
461-
export GOCACHE="$(pwd)/.cache"
462-
echo "${gssapi_auth_linux_keytab_base64}" > /tmp/drivers.keytab.base64
463-
base64 --decode /tmp/drivers.keytab.base64 > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab
464-
mkdir -p ~/.krb5
465-
cat .evergreen/krb5.config | tee -a ~/.krb5/config
466-
kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p "${gssapi_auth_username}"
467-
export MONGODB_URI="${gssapi_service_host_auth_linux_mongodb_uri}"
468-
fi;
469-
export GOPATH="$GOPATH"
470-
export GOROOT="${GO_DIST}"
471-
export GOCACHE="$GOCACHE"
472-
export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH"
473-
export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR}"
474-
475-
make -s evg-test-enterprise-auth
416+
${PREPARE_SHELL}
417+
bash ./etc/run-enterprise-gssapi-test.sh
476418
477419
run-atlas-test:
478420
- command: ec2.assume_role
@@ -513,7 +455,6 @@ functions:
513455
type: test
514456
params:
515457
shell: "bash"
516-
silent: true
517458
working_dir: src/go.mongodb.org/mongo-driver
518459
script: |
519460
${PREPARE_SHELL}
@@ -593,17 +534,9 @@ functions:
593534
params:
594535
shell: "bash"
595536
working_dir: src/go.mongodb.org/mongo-driver
596-
add_expansions_to_env: true
597537
script: |
598538
${PREPARE_SHELL}
599-
600-
AUTH="auth" \
601-
SSL="ssl" \
602-
MONGODB_URI="${SERVERLESS_URI}" \
603-
SERVERLESS="serverless" \
604-
SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" \
605-
MAKEFILE_TARGET=evg-test-serverless \
606-
sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
539+
bash etc/run-serverless-test.sh
607540
608541
run-atlas-data-lake-test:
609542
- command: shell.exec
@@ -719,16 +652,13 @@ functions:
719652
- command: ec2.assume_role
720653
params:
721654
role_arn: ${aws_test_secrets_role}
722-
- command: shell.exec
655+
- command: subprocess.exec
723656
type: test
724657
params:
725658
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
726-
shell: "bash"
727-
working_dir: "src"
728-
script: |
729-
${PREPARE_SHELL}
730-
cd $DRIVERS_TOOLS/.evergreen/auth_aws
731-
./setup_secrets.sh drivers/aws_auth
659+
binary: "bash"
660+
args:
661+
- ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh
732662

733663
run-aws-auth-test-with-regular-aws-credentials:
734664
- command: shell.exec
@@ -862,14 +792,13 @@ functions:
862792
working_dir: src/go.mongodb.org/mongo-driver
863793
binary: bash
864794
args:
865-
- ${DRIVERS_TOOLS}/.evergreen/csfle/await_servers.sh
795+
- ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh
866796

867797
run-kms-tls-test:
868798
- command: shell.exec
869799
type: test
870800
params:
871801
shell: "bash"
872-
silent: true
873802
working_dir: src/go.mongodb.org/mongo-driver
874803
script: |
875804
${PREPARE_SHELL}
@@ -892,7 +821,6 @@ functions:
892821
type: test
893822
params:
894823
shell: "bash"
895-
silent: true
896824
working_dir: src/go.mongodb.org/mongo-driver
897825
script: |
898826
${PREPARE_SHELL}
@@ -1683,20 +1611,13 @@ tasks:
16831611
tags: ["test", "enterprise-auth"]
16841612
commands:
16851613
- func: run-enterprise-auth-tests
1686-
vars:
1687-
MONGODB_URI: "${plain_auth_mongodb_uri}"
16881614

16891615
- name: test-enterprise-auth-gssapi
16901616
tags: ["test", "enterprise-auth"]
16911617
commands:
16921618
- func: run-enterprise-gssapi-auth-tests
1693-
1694-
- name: test-enterprise-auth-gssapi-service-host
1695-
tags: ["test", "enterprise-auth"]
1696-
commands:
1697-
- func: run-enterprise-gssapi-service-host-auth-tests
16981619
vars:
1699-
MONGO_GO_DRIVER_COMPRESSOR: "snappy"
1620+
MONGO_GO_DRIVER_COMPRESSOR: "snappy"
17001621

17011622
# Build with the oldest supported version of Go.
17021623
- name: go1.18-build
@@ -1837,7 +1758,7 @@ tasks:
18371758
PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
18381759
make build-kms-test
18391760
echo "Building build-kms-test ... end"
1840-
1761+
source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh
18411762
echo "Copying files ... begin"
18421763
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
18431764
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
@@ -1858,6 +1779,7 @@ tasks:
18581779
working_dir: src/go.mongodb.org/mongo-driver
18591780
script: |
18601781
${PREPARE_SHELL}
1782+
source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh
18611783
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
18621784
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
18631785
export GCPKMS_ZONE=${GCPKMS_ZONE}
@@ -1892,23 +1814,10 @@ tasks:
18921814
type: test
18931815
params:
18941816
shell: "bash"
1895-
silent: true
18961817
working_dir: src/go.mongodb.org/mongo-driver
18971818
script: |
18981819
${PREPARE_SHELL}
1899-
echo "Building build-kms-test ... begin"
1900-
BUILD_TAGS="-tags=cse" \
1901-
PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
1902-
make build-kms-test
1903-
echo "Building build-kms-test ... end"
1904-
1905-
export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
1906-
export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
1907-
1908-
LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
1909-
MONGODB_URI='${atlas_free_tier_uri}' \
1910-
PROVIDER='aws' \
1911-
./testkms
1820+
bash etc/run-awskms-test.sh
19121821
19131822
- name: "testawskms-fail-task"
19141823
# testawskms-fail-task runs without environment variables.
@@ -1921,17 +1830,8 @@ tasks:
19211830
working_dir: src/go.mongodb.org/mongo-driver
19221831
script: |
19231832
${PREPARE_SHELL}
1924-
echo "Building build-kms-test ... begin"
1925-
BUILD_TAGS="-tags=cse" \
1926-
PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
1927-
make build-kms-test
1928-
echo "Building build-kms-test ... end"
1929-
1930-
LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
1931-
MONGODB_URI='${atlas_free_tier_uri}' \
1932-
EXPECT_ERROR='status=400' \
1933-
PROVIDER='aws' \
1934-
./testkms
1833+
export EXPECT_ERROR='status=400'
1834+
bash etc/run-awskms-test.sh
19351835
19361836
- name: "testazurekms-task"
19371837
commands:
@@ -1966,8 +1866,7 @@ tasks:
19661866
working_dir: src/go.mongodb.org/mongo-driver
19671867
script: |
19681868
${PREPARE_SHELL}
1969-
# Get azurekms credentials from the vault.
1970-
. ./etc/get_aws_secrets.sh drivers/azurekms
1869+
source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh
19711870
export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
19721871
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
19731872
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
@@ -2011,14 +1910,13 @@ tasks:
20111910
params:
20121911
working_dir: src/go.mongodb.org/mongo-driver
20131912
shell: bash
2014-
add_expansions_to_env: true
20151913
env:
20161914
TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/internal/test/faas/awslambda
20171915
LAMBDA_STACK_NAME: dbx-go-lambda
20181916
AWS_REGION: us-east-1
1917+
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
20191918
script: |
20201919
${PREPARE_SHELL}
2021-
ls $TEST_LAMBDA_DIRECTORY
20221920
./.evergreen/run-deployed-lambda-aws-tests.sh
20231921
20241922
- name: "test-search-index"
@@ -2252,10 +2150,6 @@ task_groups:
22522150
shell: "bash"
22532151
script: |
22542152
${PREPARE_SHELL}
2255-
echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json
2256-
export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json
2257-
export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
2258-
export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}"
22592153
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
22602154
# Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions.
22612155
- command: expansions.update
@@ -2267,10 +2161,6 @@ task_groups:
22672161
shell: "bash"
22682162
script: |
22692163
${PREPARE_SHELL}
2270-
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
2271-
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
2272-
export GCPKMS_ZONE=${GCPKMS_ZONE}
2273-
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
22742164
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh
22752165
- func: handle-test-artifacts
22762166
- func: cleanup
@@ -2294,9 +2184,8 @@ task_groups:
22942184
export AZUREKMS_VMNAME_PREFIX="GODRIVER"
22952185
export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
22962186
# Get azurekms credentials from the vault.
2297-
bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup_secrets.sh drivers/azurekms
2298-
source ./secrets-export.sh
2299-
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh
2187+
. ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh
2188+
${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh
23002189
- command: expansions.update
23012190
params:
23022191
file: testazurekms-expansions.yml
@@ -2326,21 +2215,18 @@ task_groups:
23262215
params:
23272216
working_dir: src/go.mongodb.org/mongo-driver
23282217
binary: bash
2329-
add_expansions_to_env: true
23302218
env:
23312219
LAMBDA_STACK_NAME: dbx-go-lambda
23322220
AWS_REGION: us-east-1
2221+
task_id: ${task_id}
2222+
execution: ${execution}
23332223
args:
23342224
- ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
2335-
- command: expansions.update
2336-
params:
2337-
file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml
23382225
teardown_group:
23392226
- command: subprocess.exec
23402227
params:
23412228
working_dir: src/go.mongodb.org/mongo-driver
23422229
binary: bash
2343-
add_expansions_to_env: true
23442230
env:
23452231
LAMBDA_STACK_NAME: dbx-go-lambda
23462232
AWS_REGION: us-east-1
@@ -2361,9 +2247,12 @@ task_groups:
23612247
params:
23622248
working_dir: src/go.mongodb.org/mongo-driver
23632249
binary: bash
2364-
add_expansions_to_env: true
23652250
env:
23662251
MONGODB_VERSION: "7.0"
2252+
LAMBDA_STACK_NAME: dbx-go-lambda
2253+
AWS_REGION: us-east-1
2254+
task_id: ${task_id}
2255+
execution: ${execution}
23672256
args:
23682257
- ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
23692258
- command: expansions.update
@@ -2383,7 +2272,6 @@ task_groups:
23832272
params:
23842273
working_dir: src/go.mongodb.org/mongo-driver
23852274
binary: bash
2386-
add_expansions_to_env: true
23872275
args:
23882276
- ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
23892277
- func: handle-test-artifacts

.evergreen/run-deployed-lambda-aws-tests.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
#
33
set -o errexit # Exit the script with error if any of the commands fail.
44

5+
source ${DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh
6+
57
VARLIST=(
68
AWS_REGION
79
DRIVERS_TOOLS

0 commit comments

Comments
 (0)