Merge branch 'main' into NODE-5906/optimize-toArray-to-batches

Author: aditi-khare-mongoDB

.evergreen/config.in.yml

@@ -272,6 +272,7 @@ functions:
     - command: shell.exec
       type: test
       params:
+        shell: bash
         working_dir: src
         timeout_secs: 300
         script: |
@@ -475,8 +476,6 @@ functions:
           export AWS_CMK_ID='${AWS_CMK_ID}'
           export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
-          export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
-          export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
           EOT
     - command: shell.exec
       type: test
@@ -914,8 +913,6 @@ functions:
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
           export AWS_REGION='${AWS_REGION}'
           export AWS_CMK_ID='${AWS_CMK_ID}'
-          export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
-          export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
           EOT
     - command: shell.exec
       type: test
@@ -1074,6 +1071,18 @@ functions:
         args:
           - ${PROJECT_DIRECTORY}/.evergreen/run-x509-tests.sh
 
+  install mongodb-client-encryption:
+    - command: subprocess.exec
+      type: setup
+      params:
+        working_dir: "src"
+        env:
+          INSTALL_DIR: mongodb-client-encryption
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+        binary: bash
+        args:
+          - ${PROJECT_DIRECTORY}/.evergreen/install-mongodb-client-encryption.sh
+
 tasks:
   - name: 'test-atlas-data-lake'
     tags: ["datalake", "mongohouse"]

.evergreen/config.yml

@@ -241,6 +241,7 @@ functions:
     - command: shell.exec
       type: test
       params:
+        shell: bash
         working_dir: src
         timeout_secs: 300
         script: |
@@ -428,8 +429,6 @@ functions:
           export AWS_CMK_ID='${AWS_CMK_ID}'
           export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
-          export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
-          export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
           EOT
     - command: shell.exec
       type: test
@@ -886,8 +885,6 @@ functions:
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
           export AWS_REGION='${AWS_REGION}'
           export AWS_CMK_ID='${AWS_CMK_ID}'
-          export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
-          export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
           EOT
     - command: shell.exec
       type: test
@@ -1035,6 +1032,17 @@ functions:
         binary: bash
         args:
           - ${PROJECT_DIRECTORY}/.evergreen/run-x509-tests.sh
+  install mongodb-client-encryption:
+    - command: subprocess.exec
+      type: setup
+      params:
+        working_dir: src
+        env:
+          INSTALL_DIR: mongodb-client-encryption
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+        binary: bash
+        args:
+          - ${PROJECT_DIRECTORY}/.evergreen/install-mongodb-client-encryption.sh
 tasks:
   - name: test-atlas-data-lake
     tags:
@@ -3516,7 +3524,7 @@ tasks:
         variant: '*'
         status: '*'
         patch_optional: true
-  - name: run-custom-csfle-tests-5.0-pinned-commit
+  - name: run-custom-csfle-tests-5.0
     tags:
       - run-custom-dependency-tests
     commands:
@@ -3528,46 +3536,12 @@ tasks:
             - {key: NPM_VERSION, value: '9'}
             - {key: VERSION, value: '5.0'}
             - {key: TOPOLOGY, value: replica_set}
-            - {key: CSFLE_GIT_REF, value: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd}
-      - func: install dependencies
-      - func: bootstrap mongo-orchestration
-      - func: bootstrap kms servers
-      - func: run custom csfle tests
-  - name: run-custom-csfle-tests-5.0-master
-    tags:
-      - run-custom-dependency-tests
-    commands:
-      - command: expansions.update
-        type: setup
-        params:
-          updates:
-            - {key: NODE_LTS_VERSION, value: '16'}
-            - {key: NPM_VERSION, value: '9'}
-            - {key: VERSION, value: '5.0'}
-            - {key: TOPOLOGY, value: replica_set}
-            - {key: CSFLE_GIT_REF, value: master}
-      - func: install dependencies
-      - func: bootstrap mongo-orchestration
-      - func: bootstrap kms servers
-      - func: run custom csfle tests
-  - name: run-custom-csfle-tests-rapid-pinned-commit
-    tags:
-      - run-custom-dependency-tests
-    commands:
-      - command: expansions.update
-        type: setup
-        params:
-          updates:
-            - {key: NODE_LTS_VERSION, value: '16'}
-            - {key: NPM_VERSION, value: '9'}
-            - {key: VERSION, value: rapid}
-            - {key: TOPOLOGY, value: replica_set}
-            - {key: CSFLE_GIT_REF, value: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd}
       - func: install dependencies
       - func: bootstrap mongo-orchestration
       - func: bootstrap kms servers
+      - func: install mongodb-client-encryption
       - func: run custom csfle tests
-  - name: run-custom-csfle-tests-rapid-master
+  - name: run-custom-csfle-tests-rapid
     tags:
       - run-custom-dependency-tests
     commands:
@@ -3579,29 +3553,12 @@ tasks:
             - {key: NPM_VERSION, value: '9'}
             - {key: VERSION, value: rapid}
             - {key: TOPOLOGY, value: replica_set}
-            - {key: CSFLE_GIT_REF, value: master}
-      - func: install dependencies
-      - func: bootstrap mongo-orchestration
-      - func: bootstrap kms servers
-      - func: run custom csfle tests
-  - name: run-custom-csfle-tests-latest-pinned-commit
-    tags:
-      - run-custom-dependency-tests
-    commands:
-      - command: expansions.update
-        type: setup
-        params:
-          updates:
-            - {key: NODE_LTS_VERSION, value: '16'}
-            - {key: NPM_VERSION, value: '9'}
-            - {key: VERSION, value: latest}
-            - {key: TOPOLOGY, value: replica_set}
-            - {key: CSFLE_GIT_REF, value: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd}
       - func: install dependencies
       - func: bootstrap mongo-orchestration
       - func: bootstrap kms servers
+      - func: install mongodb-client-encryption
       - func: run custom csfle tests
-  - name: run-custom-csfle-tests-latest-master
+  - name: run-custom-csfle-tests-latest
     tags:
       - run-custom-dependency-tests
     commands:
@@ -3613,10 +3570,10 @@ tasks:
             - {key: NPM_VERSION, value: '9'}
             - {key: VERSION, value: latest}
             - {key: TOPOLOGY, value: replica_set}
-            - {key: CSFLE_GIT_REF, value: master}
       - func: install dependencies
       - func: bootstrap mongo-orchestration
       - func: bootstrap kms servers
+      - func: install mongodb-client-encryption
       - func: run custom csfle tests
   - name: test-latest-driver-mongodb-client-encryption-6.0.0
     tags:
@@ -5118,12 +5075,9 @@ buildvariants:
     display_name: Custom Dependency Version Test
     run_on: rhel80-large
     tasks:
-      - run-custom-csfle-tests-5.0-pinned-commit
-      - run-custom-csfle-tests-5.0-master
-      - run-custom-csfle-tests-rapid-pinned-commit
-      - run-custom-csfle-tests-rapid-master
-      - run-custom-csfle-tests-latest-pinned-commit
-      - run-custom-csfle-tests-latest-master
+      - run-custom-csfle-tests-5.0
+      - run-custom-csfle-tests-rapid
+      - run-custom-csfle-tests-latest
       - test-latest-driver-mongodb-client-encryption-6.0.0
   - name: rhel8-test-serverless
     display_name: Serverless Test

.evergreen/generate_evergreen_tasks.js

@@ -579,33 +579,29 @@ BUILD_VARIANTS.push({
   tasks: ['test-atlas-data-lake']
 });
 
-const oneOffFuncAsTasks = [];
-
-const FLE_PINNED_COMMIT = '974a4614f8c1c3786e5e39fa63568d83f4f69ebd';
+const customDependencyTests = [];
 
 for (const version of ['5.0', 'rapid', 'latest']) {
-  for (const ref of [FLE_PINNED_COMMIT, 'master']) {
-    oneOffFuncAsTasks.push({
-      name: `run-custom-csfle-tests-${version}-${ref === 'master' ? ref : 'pinned-commit'}`,
-      tags: ['run-custom-dependency-tests'],
-      commands: [
-        updateExpansions({
-          NODE_LTS_VERSION: LOWEST_LTS,
-          NPM_VERSION: 9,
-          VERSION: version,
-          TOPOLOGY: 'replica_set',
-          CSFLE_GIT_REF: ref
-        }),
-        { func: 'install dependencies' },
-        { func: 'bootstrap mongo-orchestration' },
-        { func: 'bootstrap kms servers' },
-        { func: 'run custom csfle tests' }
-      ]
-    });
-  }
+  customDependencyTests.push({
+    name: `run-custom-csfle-tests-${version}`,
+    tags: ['run-custom-dependency-tests'],
+    commands: [
+      updateExpansions({
+        NODE_LTS_VERSION: LOWEST_LTS,
+        NPM_VERSION: 9,
+        VERSION: version,
+        TOPOLOGY: 'replica_set',
+      }),
+      { func: 'install dependencies' },
+      { func: 'bootstrap mongo-orchestration' },
+      { func: 'bootstrap kms servers' },
+      { func: 'install mongodb-client-encryption' },
+      { func: 'run custom csfle tests' }
+    ]
+  });
 }
 
-oneOffFuncAsTasks.push({
+customDependencyTests.push({
   name: `test-latest-driver-mongodb-client-encryption-6.0.0`,
   tags: ['run-custom-dependency-tests'],
   commands: [
@@ -641,13 +637,13 @@ const coverageTask = {
 };
 
 SINGLETON_TASKS.push(coverageTask);
-SINGLETON_TASKS.push(...oneOffFuncAsTasks);
+SINGLETON_TASKS.push(...customDependencyTests);
 
 BUILD_VARIANTS.push({
   name: 'rhel8-custom-dependency-tests',
   display_name: 'Custom Dependency Version Test',
   run_on: DEFAULT_OS,
-  tasks: oneOffFuncAsTasks.map(({ name }) => name)
+  tasks: customDependencyTests.map(({ name }) => name)
 });
 
 // special case for serverless testing
@@ -759,7 +755,6 @@ fileData.tasks = (fileData.tasks || [])
   .concat(AUTH_DISABLED_TASKS)
   .concat(AWS_LAMBDA_HANDLER_TASKS)
   .concat(MONGOCRYPTD_CSFLE_TASKS);
-
 fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);
 
 fs.writeFileSync(

.evergreen/install-mongodb-client-encryption.sh

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+set +o xtrace # Do not write AWS credentials to stderr
+
+# Initial checks for running these tests
+if [ -z ${PROJECT_DIRECTORY+omitted} ]; then echo "PROJECT_DIRECTORY is unset" && exit 1; fi
+
+source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh"
+
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+rm -rf $INSTALL_DIR
+git clone https://github.com/mongodb-js/mongodb-client-encryption.git
+pushd mongodb-client-encryption
+
+if [ -n "${LIBMONGOCRYPT_VERSION}" ]; then
+	# nightly tests test with `latest` to test against the laster FLE build.
+    npm run install:libmongocrypt -- --libVersion $LIBMONGOCRYPT_VERSION
+else
+	# otherwise use whatever is specified in the package.json.
+    npm run install:libmongocrypt
+fi
+
+echo "finished installing libmongocrypt"
+BINDINGS_DIR=$(pwd)
+
+popd
+
+echo "linking mongodb-client-encrytion"
+npm link ./mongodb-client-encryption

.evergreen/run-custom-csfle-tests.sh

@@ -22,48 +22,6 @@ echo "adding temporary AWS credentials to environment"
 # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
 . "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
 
-ABS_PATH_TO_PATCH=$(pwd)
-
-# Environment Variables:
-# CSFLE_GIT_REF - set the git reference to checkout for a custom CSFLE version
-# CDRIVER_GIT_REF - set the git reference to checkout for a custom CDRIVER version (this is for libbson)
-CSFLE_GIT_REF=${CSFLE_GIT_REF:-master}
-CDRIVER_GIT_REF=${CDRIVER_GIT_REF:-1.17.6}
-
-rm -rf ../csfle-deps-tmp
-mkdir -p ../csfle-deps-tmp
-pushd ../csfle-deps-tmp
-
-rm -rf libmongocrypt mongo-c-driver
-
-git clone https://github.com/mongodb/libmongocrypt.git
-pushd libmongocrypt
-git fetch --tags
-git checkout "$CSFLE_GIT_REF" -b csfle-custom
-echo "checked out libmongocrypt at $(git rev-parse HEAD)"
-popd # libmongocrypt
-
-git clone https://github.com/mongodb/mongo-c-driver.git
-pushd mongo-c-driver
-git fetch --tags
-git checkout "$CDRIVER_GIT_REF" -b cdriver-custom
-echo "checked out C driver at $(git rev-parse HEAD)"
-popd # mongo-c-driver
-
-pushd libmongocrypt/bindings/node
-
-npm install --production --ignore-scripts
-bash ./etc/build-static.sh
-
-npm run rebuild # just in case this is necessary?
-
-BINDINGS_DIR=$(pwd)
-popd # libmongocrypt/bindings/node
-popd # ../csfle-deps-tmp
-
-# copy mongodb-client-encryption into driver's node_modules
-npm link $BINDINGS_DIR
-
 export MONGODB_URI=${MONGODB_URI}
 export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
 export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"

README.md

@@ -70,7 +70,11 @@ Change history can be found in [`HISTORY.md`](https://github.com/mongodb/node-mo
 
 ### Compatibility
 
-For server and runtime version compatibility matrices, please refer to the following links:
+The driver currently supports 3.6+ servers.
+
+** 3.6 support is deprecated and support will be removed in a future version **
+
+For exhaustive server and runtime version compatibility matrices, please refer to the following links:
 
 - [MongoDB](https://www.mongodb.com/docs/drivers/node/current/compatibility/#mongodb-compatibility)
 - [NodeJS](https://www.mongodb.com/docs/drivers/node/current/compatibility/#language-compatibility)