PYTHON-3963 Migrate CSFLE Tests to AWS Secrets Vault (#1446)

.evergreen/config.yml

@@ -365,78 +365,28 @@ functions:
            PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test
 
   "run tests":
-    # If testing FLE, start the KMS mock servers, first create the virtualenv.
-    - command: shell.exec
-      params:
-        script: |
-          if [ -n "${test_encryption}" ]; then
-            ${PREPARE_SHELL}
-            cd ${DRIVERS_TOOLS}/.evergreen/csfle
-            . ./activate-kmstlsvenv.sh
-          fi
-    # Run in the background so the mock servers don't block the EVG task.
     - command: shell.exec
       params:
+        working_dir: "src"
+        shell: bash
         background: true
         script: |
+          ${PREPARE_SHELL}
           if [ -n "${test_encryption}" ]; then
-            ${PREPARE_SHELL}
-            cd ${DRIVERS_TOOLS}/.evergreen/csfle
-            . ./activate-kmstlsvenv.sh
-            # The -u options forces the stdout and stderr streams to be unbuffered.
-            # TMPDIR is required to avoid "AF_UNIX path too long" errors.
-            TMPDIR="$(dirname $DRIVERS_TOOLS)" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem  --port 5698 &
-            python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
-            python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
-            python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
-          fi
-    # Wait up to 10 seconds for the KMIP server to start.
-    - command: shell.exec
-      params:
-        script: |
-          if [ -n "${test_encryption}" ]; then
-            ${PREPARE_SHELL}
-            cd ${DRIVERS_TOOLS}/.evergreen/csfle
-            . ./activate-kmstlsvenv.sh
-            for i in $(seq 1 1 10); do
-               sleep 1
-               if python -u kms_kmip_client.py; then
-                  echo 'KMS KMIP server started!'
-                  exit 0
-               fi
-            done
-            echo 'Failed to start KMIP server!'
-            exit 1
-          fi
-    - command: shell.exec
-      type: test
-      params:
-        silent: true
-        working_dir: "src"
-        script: |
-          if [ -n "${test_encryption}" ]; then
-            cat <<EOT > fle_creds.sh
-            export FLE_AWS_KEY="${fle_aws_key}"
-            export FLE_AWS_SECRET="${fle_aws_secret}"
-            export FLE_AZURE_CLIENTID="${fle_azure_clientid}"
-            export FLE_AZURE_TENANTID="${fle_azure_tenantid}"
-            export FLE_AZURE_CLIENTSECRET="${fle_azure_clientsecret}"
-            export FLE_GCP_EMAIL="${fle_gcp_email}"
-            export FLE_GCP_PRIVATEKEY="${fle_gcp_privatekey}"
-            # Needed for generating temporary aws credentials.
-            export AWS_ACCESS_KEY_ID="${fle_aws_key}"
-            export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}"
-            export AWS_DEFAULT_REGION=us-east-1
-          EOT
+            bash ./.evergreen/start-kmip-server.sh
           fi
     - command: shell.exec
       type: test
       params:
         working_dir: "src"
+        shell: bash
+        include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
           # Disable xtrace
           set +x
           ${PREPARE_SHELL}
+          # Fetch vault secrets if needed
+          bash .evergreen/fetch-aws-vault-secrets.sh
           if [ -n "${MONGODB_STARTED}" ]; then
             export PYMONGO_MUST_CONNECT=true
           fi
@@ -446,10 +396,9 @@ functions:
           if [ -n "${test_encryption}" ]; then
             # Disable xtrace (just in case it was accidentally set).
             set +x
-            . ./fle_creds.sh
-            rm -f ./fle_creds.sh
             export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
             export TEST_ENCRYPTION=1
+            . .evergreen/setup-encryption-variables.sh
             if [ -n "${test_encryption_pyopenssl}" ]; then
               export TEST_ENCRYPTION_PYOPENSSL=1
             fi
@@ -667,8 +616,10 @@ functions:
   "cleanup":
     - command: shell.exec
       params:
+        working_dir: "src"
         script: |
           ${PREPARE_SHELL}
+          . .evergreen/stop-kmip-server.sh
           rm -rf $DRIVERS_TOOLS || true
           rm -f ./secrets-export.sh || true
 
@@ -724,6 +675,11 @@ functions:
           # Don't use ${file} syntax here because evergreen treats it as an empty expansion.
           [ -f "$file" ] && bash $file || echo "$file not available, skipping"
 
+  "assume ec2 role":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+
   "run-ocsp-test":
     - command: shell.exec
       type: test
@@ -939,6 +895,7 @@ pre:
   - func: "init test-results"
   - func: "make files executable"
   - func: "install dependencies"
+  - func: "assume ec2 role"
 
 post:
   # Disabled, causing timeouts

.evergreen/fetch-aws-vault-secrets.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+AWS_SECRETS_VAULTS=""
+
+if [ -n "${test_encryption}" ]; then
+  AWS_SECRETS_VAULTS+="drivers/csfle"
+fi
+
+if [ -n "${AWS_SECRETS_VAULTS}" ]; then
+  "$DRIVERS_TOOLS"/.evergreen/auth_aws/setup_secrets.sh ${AWS_SECRETS_VAULTS}
+fi

.evergreen/run-tests.sh

@@ -172,6 +172,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then
 
     # Get access to the AWS temporary credentials:
     # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
+    export AWS_SESSION_TOKEN=
     . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
 
     if [ -n "$TEST_CRYPT_SHARED" ]; then

.evergreen/setup-encryption-variables.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+source ./secrets-export.sh
+
+{
+  echo "export AWS_ACCESS_KEY_ID=$FLE_AWS_KEY";
+  echo "export AWS_SECRET_ACCESS_KEY=$FLE_AWS_SECRET";
+  echo "export AWS_DEFAULT_REGION=us-east-1";
+  echo "export AWS_SESSION_TOKEN=";
+} >> ./secrets-export.sh

.evergreen/start-kmip-server.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+cd "$DRIVERS_TOOLS"/.evergreen/csfle || exit
+. ./activate-kmstlsvenv.sh
+
+# The -u options forces the stdout and stderr streams to be unbuffered.
+# TMPDIR is required to avoid "AF_UNIX path too long" errors.
+TMPDIR="$(dirname "$DRIVERS_TOOLS")" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem  --port 5698 &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
+
+pgrep -f "kms_...._server.py" > kmip_pids.pid
+
+for _ in $(seq 1 1 10); do
+   sleep 1
+   if python -u kms_kmip_client.py; then
+      echo 'KMS KMIP server started!'
+      exit 0
+   fi
+done
+echo 'Failed to start KMIP server!'
+exit 1

.evergreen/stop-kmip-server.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Clean up CSFLE kmip servers
+cd "$DRIVERS_TOOLS"/.evergreen/csfle || exit
+
+if [ -f "kmip_pids.pid" ]; then
+  < kmip_pids.pid xargs kill -9
+  rm kmip_pids.pid
+fi

CONTRIBUTING.md

@@ -204,6 +204,12 @@ tox -e doc
 -   Run the tests from the `pymongo` checkout directory using:
     `TEST_LOADBALANCER=1 tox -m test-eg`.
 
+## Running Encryption Tests Locally
+- Run `AWS_PROFILE=<profile> tox -e setup-encryption` after setting up your AWS profile with `aws configure sso`
+- Run the tests with `TEST_ENCRYPTION=1 LIBMONGOCRYPT_URL=<url> tox -e test-eg`.
+  Look in `.evergreen/config.yml` for the correct `LIBMONGOCRYPT_URL` for your platform.
+- When done, run `tox -e teardown-encryption` to clean up.
+
 ## Re-sync Spec Tests
 
 If you would like to re-sync the copy of the specification tests in the

tox.ini

@@ -177,3 +177,24 @@ deps =
 passenv = *
 commands =
     python {env:DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.py {posargs}
+
+[testenv:setup-encryption]
+deps =
+    {[testenv:aws-secrets]deps}
+    pykmip
+passenv = *
+allowlist_externals =
+    bash
+commands =
+    {[testenv:aws-secrets]commands} drivers/csfle
+    bash .evergreen/start-kmip-server.sh
+    bash .evergreen/setup-encryption-variables.sh
+
+[testenv:teardown-encryption]
+passenv = *
+allowlist_externals =
+    bash
+    rm
+commands =
+    bash .evergreen/stop-kmip-server.sh
+    rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz