PYTHON-4151 Use Centralized Scripts for CSFLE Tests (#1477)

2024-01-29 17:19:24 -06:00 · 2024-01-29 17:19:24 -06:00 · 68d22b20bd
commit 68d22b20bd
parent 85b0c0e3c1
12 changed files with 83 additions and 142 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -369,24 +369,21 @@ functions:
      params:
        working_dir: "src"
        shell: bash
-        background: true
+        include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
        script: |
          ${PREPARE_SHELL}
          if [ -n "${test_encryption}" ]; then
-            bash ./.evergreen/start-kmip-server.sh
+            . .evergreen/tox.sh -m setup-encryption
          fi
    - command: shell.exec
      type: test
      params:
        working_dir: "src"
        shell: bash
-        include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
        script: |
          # Disable xtrace
          set +x
          ${PREPARE_SHELL}
-          # Fetch vault secrets if needed
-          bash .evergreen/fetch-aws-vault-secrets.sh
          if [ -n "${MONGODB_STARTED}" ]; then
            export PYMONGO_MUST_CONNECT=true
          fi
@ -396,9 +393,7 @@ functions:
          if [ -n "${test_encryption}" ]; then
            # Disable xtrace (just in case it was accidentally set).
            set +x
-            export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
            export TEST_ENCRYPTION=1
-            . .evergreen/setup-encryption-variables.sh
            if [ -n "${test_encryption_pyopenssl}" ]; then
              export TEST_ENCRYPTION_PYOPENSSL=1
            fi
@ -619,7 +614,7 @@ functions:
        working_dir: "src"
        script: |
          ${PREPARE_SHELL}
-          . .evergreen/stop-kmip-server.sh
+          . .evergreen/tox.sh -m teardown-encryption
          rm -rf $DRIVERS_TOOLS || true
          rm -f ./secrets-export.sh || true

@ -2164,7 +2159,6 @@ axes:
          skip_EC2_auth_test: true
          skip_ECS_auth_test: true
          skip_web_identity_auth_test: true
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
      - id: macos-1100
        display_name: "macOS 11.00"
        run_on: macos-1100
@ -2172,7 +2166,6 @@ axes:
          skip_EC2_auth_test: true
          skip_ECS_auth_test: true
          skip_web_identity_auth_test: true
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
      - id: macos-1100-arm64
        display_name: "macOS 11.00 Arm64"
        run_on: macos-1100-arm64
@ -2180,25 +2173,18 @@ axes:
          skip_EC2_auth_test: true
          skip_ECS_auth_test: true
          skip_web_identity_auth_test: true
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
      - id: rhel7
        display_name: "RHEL 7.x"
        run_on: rhel79-small
        batchtime: 10080  # 7 days
-        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz
      - id: rhel8
        display_name: "RHEL 8.x"
        run_on: rhel87-small
        batchtime: 10080  # 7 days
-        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz
      - id: rhel80-fips
        display_name: "RHEL 8.0 FIPS"
        run_on: rhel80-fips
        batchtime: 10080  # 7 days
-        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz
      - id: ubuntu-22.04
        display_name: "Ubuntu 22.04"
        run_on: ubuntu2204-small
@ -2220,7 +2206,6 @@ axes:
        run_on: rhel82-arm64-small
        batchtime: 10080  # 7 days
        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-82-arm64/master/latest/libmongocrypt.tar.gz
      - id: windows-64-vsMulti-small
        display_name: "Windows 64"
        run_on: windows-64-vsMulti-small
@ -2230,7 +2215,6 @@ axes:
          skip_EC2_auth_test: true
          skip_web_identity_auth_test: true
          venv_bin_dir: "Scripts"
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz

  # Test with authentication?
  - id: auth
--- a/.evergreen/fetch-aws-vault-secrets.sh
+++ b/.evergreen/fetch-aws-vault-secrets.sh
@ -1,11 +0,0 @@
-#!/bin/bash
-
-AWS_SECRETS_VAULTS=""
-
-if [ -n "${test_encryption}" ]; then
-  AWS_SECRETS_VAULTS+="drivers/csfle"
-fi
-
-if [ -n "${AWS_SECRETS_VAULTS}" ]; then
-  "$DRIVERS_TOOLS"/.evergreen/auth_aws/setup_secrets.sh ${AWS_SECRETS_VAULTS}
-fi
--- a/.evergreen/run-tests.sh
+++ b/.evergreen/run-tests.sh
@ -111,32 +111,13 @@ fi

 if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then

-    # Work around for root certifi not being installed.
-    # TODO: Remove after PYTHON-3952 is deployed.
-    if [ "$(uname -s)" = "Darwin" ]; then
-        python -m pip install certifi
-        CERT_PATH=$(python -c "import certifi; print(certifi.where())")
-        export SSL_CERT_FILE=${CERT_PATH}
-        export REQUESTS_CA_BUNDLE=${CERT_PATH}
-        export AWS_CA_BUNDLE=${CERT_PATH}
-    fi
-
    python -m pip install '.[encryption]'

-    if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
-        # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
-        powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
+    # Install libmongocrypt if necessary.
+    if [ ! -d "libmongocrypt" ]; then
+        bash ./.evergreen/setup-libmongocrypt.sh
    fi

-    if [ -z "$LIBMONGOCRYPT_URL" ]; then
-        echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
-        exit 1
-    fi
-    curl -O "$LIBMONGOCRYPT_URL"
-    mkdir libmongocrypt
-    tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
-    ls -la libmongocrypt
-    ls -la libmongocrypt/nocrypto
    # Use the nocrypto build to avoid dependency issues with older windows/python versions.
    BASE=$(pwd)/libmongocrypt/nocrypto
    if [ -f "${BASE}/lib/libmongocrypt.so" ]; then
@ -155,11 +136,6 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE
        exit 1
    fi
    export PYMONGOCRYPT_LIB
-
-    # TODO: Test with 'pip install pymongocrypt'
-    git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
-    python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt
-    python -m pip install ./libmongocrypt_git/bindings/python
    python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
    python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
    # PATH is updated by PREPARE_SHELL for access to mongocryptd.
@ -170,11 +146,6 @@ if [ -n "$TEST_ENCRYPTION" ]; then
        python -m pip install '.[ocsp]'
    fi

-    # Get access to the AWS temporary credentials:
-    # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
-    export AWS_SESSION_TOKEN=
-    . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
-
    if [ -n "$TEST_CRYPT_SHARED" ]; then
        CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH`
        echo "using crypt_shared_dir $CRYPT_SHARED_DIR"
@ -229,16 +200,6 @@ fi
 if [ -n "$TEST_AUTH_OIDC" ]; then
    python -m pip install ".[aws]"

-    # Work around for root certifi not being installed.
-    # TODO: Remove after PYTHON-3952 is deployed.
-    if [ "$(uname -s)" = "Darwin" ]; then
-        python -m pip install certifi
-        CERT_PATH=$(python -c "import certifi; print(certifi.where())")
-        export SSL_CERT_FILE=${CERT_PATH}
-        export REQUESTS_CA_BUNDLE=${CERT_PATH}
-        export AWS_CA_BUNDLE=${CERT_PATH}
-    fi
-
    TEST_ARGS="test/auth_oidc/test_auth_oidc.py"
 fi

--- a/.evergreen/setup-encryption-variables.sh
+++ b/.evergreen/setup-encryption-variables.sh
@ -1,10 +0,0 @@
-#!/bin/bash
-
-source ./secrets-export.sh
-
-{
-  echo "export AWS_ACCESS_KEY_ID=$FLE_AWS_KEY";
-  echo "export AWS_SECRET_ACCESS_KEY=$FLE_AWS_SECRET";
-  echo "export AWS_DEFAULT_REGION=us-east-1";
-  echo "export AWS_SESSION_TOKEN=";
-} >> ./secrets-export.sh
--- a/.evergreen/setup-libmongocrypt.sh
+++ b/.evergreen/setup-libmongocrypt.sh
@ -0,0 +1,47 @@
+#!/bin/bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set -o xtrace
+
+TARGET=""
+
+if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
+    # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
+    powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
+    TARGET="windows-test"
+fi
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    TARGET="macos"
+fi
+
+if [ "$(uname -s)" = "Linux" ]; then
+    rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release)
+    arch=$(uname -m)
+    echo "RHEL $rhel_ver $arch"
+    if [[ $rhel_ver =~ 7 ]]; then
+        TARGET="rhel-70-64-bit"
+    elif [[ $rhel_ver =~ 8 ]]; then
+        if [ "$arch" = "x86_64" ]; then
+            TARGET="rhel-80-64-bit"
+        elif [ "$arch" = "arm" ]; then
+            TARGET="rhel-82-arm64"
+        fi
+    fi
+fi
+
+if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then
+    LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz"
+fi
+
+if [ -z "$LIBMONGOCRYPT_URL" ]; then
+    echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
+    exit 1
+fi
+rm -rf libmongocrypt libmongocrypt.tar.gz
+echo "Fetching $LIBMONGOCRYPT_URL..."
+curl -O "$LIBMONGOCRYPT_URL"
+echo "Fetching $LIBMONGOCRYPT_URL...done"
+mkdir libmongocrypt
+tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
+ls -la libmongocrypt
+ls -la libmongocrypt/nocrypto
--- a/.evergreen/start-kmip-server.sh
+++ b/.evergreen/start-kmip-server.sh
@ -1,23 +0,0 @@
-#!/bin/bash
-
-cd "$DRIVERS_TOOLS"/.evergreen/csfle || exit
-. ./activate-kmstlsvenv.sh
-
-# The -u options forces the stdout and stderr streams to be unbuffered.
-# TMPDIR is required to avoid "AF_UNIX path too long" errors.
-TMPDIR="$(dirname "$DRIVERS_TOOLS")" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem  --port 5698 &
-python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
-python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
-python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
-
-pgrep -f "kms_...._server.py" > kmip_pids.pid
-
-for _ in $(seq 1 1 10); do
-   sleep 1
-   if python -u kms_kmip_client.py; then
-      echo 'KMS KMIP server started!'
-      exit 0
-   fi
-done
-echo 'Failed to start KMIP server!'
-exit 1
--- a/.evergreen/stop-kmip-server.sh
+++ b/.evergreen/stop-kmip-server.sh
@ -1,9 +0,0 @@
-#!/bin/bash
-
-# Clean up CSFLE kmip servers
-cd "$DRIVERS_TOOLS"/.evergreen/csfle || exit
-
-if [ -f "kmip_pids.pid" ]; then
-  < kmip_pids.pid xargs kill -9
-  rm kmip_pids.pid
-fi
--- a/.evergreen/test-encryption-requirements.txt
+++ b/.evergreen/test-encryption-requirements.txt
@ -1,3 +0,0 @@
-cffi>=1.12.0,<2
-# boto3 is required by drivers-evergreen-tools/.evergreen/csfle/set-temp-creds.sh
-boto3<2
--- a/.gitignore
+++ b/.gitignore
@ -20,6 +20,9 @@ mongocryptd.pid
 .nova/
 venv/
 secrets-export.sh
+libmongocrypt.tar.gz
+libmongocrypt/
+libmongocrypt_git/

 # Lambda temp files
 test/lambda/.aws-sam
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -216,10 +216,9 @@ the pages will re-render and the browser will automatically refresh.
    `TEST_LOADBALANCER=1 tox -m test-eg`.

 ## Running Encryption Tests Locally
- Run `AWS_PROFILE=<profile> tox -e setup-encryption` after setting up your AWS profile with `aws configure sso`
- Run the tests with `TEST_ENCRYPTION=1 LIBMONGOCRYPT_URL=<url> tox -e test-eg`.
-  Look in `.evergreen/config.yml` for the correct `LIBMONGOCRYPT_URL` for your platform.
- When done, run `tox -e teardown-encryption` to clean up.
+- Run `AWS_PROFILE=<profile> tox -m setup-encryption` after setting up your AWS profile with `aws configure sso`.
+- Run the tests with `TEST_ENCRYPTION=1 tox -e test-eg`.
+- When done, run `tox -m teardown-encryption` to clean up.

 ## Re-sync Spec Tests

--- a/test/test_encryption.py
+++ b/test/test_encryption.py
@ -2007,7 +2007,7 @@ class TestKmsTLSProse(EncryptionIntegrationTest):
        key = {
            "region": "us-east-1",
            "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-            "endpoint": "mongodb://127.0.0.1:8000",
+            "endpoint": "mongodb://127.0.0.1:9000",
        }
        # Some examples:
        # certificate verify failed: certificate has expired (_ssl.c:1129)
@ -2019,7 +2019,7 @@ class TestKmsTLSProse(EncryptionIntegrationTest):
        key = {
            "region": "us-east-1",
            "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-            "endpoint": "mongodb://127.0.0.1:8001",
+            "endpoint": "mongodb://127.0.0.1:9001",
        }
        # Some examples:
        # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)"
@ -2038,8 +2038,8 @@ class TestKmsTLSOptions(EncryptionIntegrationTest):
        super().setUp()
        # 1, create client with only tlsCAFile.
        providers: dict = copy.deepcopy(ALL_KMS_PROVIDERS)
-        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8002"
-        providers["gcp"]["endpoint"] = "127.0.0.1:8002"
+        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9002"
+        providers["gcp"]["endpoint"] = "127.0.0.1:9002"
        kms_tls_opts_ca_only = {
            "aws": {"tlsCAFile": CA_PEM},
            "azure": {"tlsCAFile": CA_PEM},
@ -2060,18 +2060,18 @@ class TestKmsTLSOptions(EncryptionIntegrationTest):
        self.addCleanup(self.client_encryption_with_tls.close)
        # 3, update endpoints to expired host.
        providers: dict = copy.deepcopy(providers)
-        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8000"
-        providers["gcp"]["endpoint"] = "127.0.0.1:8000"
-        providers["kmip"]["endpoint"] = "127.0.0.1:8000"
+        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9000"
+        providers["gcp"]["endpoint"] = "127.0.0.1:9000"
+        providers["kmip"]["endpoint"] = "127.0.0.1:9000"
        self.client_encryption_expired = ClientEncryption(
            providers, "keyvault.datakeys", self.client, OPTS, kms_tls_options=kms_tls_opts_ca_only
        )
        self.addCleanup(self.client_encryption_expired.close)
        # 3, update endpoints to invalid host.
        providers: dict = copy.deepcopy(providers)
-        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8001"
-        providers["gcp"]["endpoint"] = "127.0.0.1:8001"
-        providers["kmip"]["endpoint"] = "127.0.0.1:8001"
+        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9001"
+        providers["gcp"]["endpoint"] = "127.0.0.1:9001"
+        providers["kmip"]["endpoint"] = "127.0.0.1:9001"
        self.client_encryption_invalid_hostname = ClientEncryption(
            providers, "keyvault.datakeys", self.client, OPTS, kms_tls_options=kms_tls_opts_ca_only
        )
@ -2095,7 +2095,7 @@ class TestKmsTLSOptions(EncryptionIntegrationTest):
        key = {
            "region": "us-east-1",
            "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-            "endpoint": "127.0.0.1:8002",
+            "endpoint": "127.0.0.1:9002",
        }
        with self.assertRaisesRegex(EncryptionError, self.cert_error):
            self.client_encryption_no_client_cert.create_data_key("aws", key)
@ -2105,14 +2105,14 @@ class TestKmsTLSOptions(EncryptionIntegrationTest):
        # Some examples:
        # certificate verify failed: certificate has expired (_ssl.c:1129)
        # amazon1-2018 Python 3.6: certificate verify failed (_ssl.c:852)
-        key["endpoint"] = "127.0.0.1:8000"
+        key["endpoint"] = "127.0.0.1:9000"
        with self.assertRaisesRegex(EncryptionError, "expired|certificate verify failed"):
            self.client_encryption_expired.create_data_key("aws", key)
        # Some examples:
        # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)"
        # hostname '127.0.0.1' doesn't match 'wronghost.com'
        # 127.0.0.1:8001: ('Certificate does not contain any `subjectAltName`s.',)
-        key["endpoint"] = "127.0.0.1:8001"
+        key["endpoint"] = "127.0.0.1:9001"
        with self.assertRaisesRegex(
            EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate"
        ):
--- a/tox.ini
+++ b/tox.ini
@ -7,6 +7,10 @@ envlist =
    test,
    # Test using the run-tests Evergreen script.
    test-eg,
+    # Set up encryption files and services.
+    setup-encryption,
+    # Tear down encryption files and services.
+    teardown-encryption,
    # Run pre-commit on all files.
    lint,
    # Run pre-commit on all files, including stages that require manual fixes.
@ -33,6 +37,8 @@ envlist =
 labels = # Use labels and -m instead of -e so that tox -m <label> fails instantly if the label does not exist
    test = test
    test-eg = test-eg
+    setup-encryption = setup-encryption
+    teardown-encryption = teardown-encryption
    lint = lint
    lint-manual = lint-manual
    typecheck-mypy = typecheck-mypy
@ -190,16 +196,13 @@ commands =
    python {env:DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.py {posargs}

 [testenv:setup-encryption]
-deps =
-    {[testenv:aws-secrets]deps}
-    pykmip
 passenv = *
 allowlist_externals =
    bash
 commands =
-    {[testenv:aws-secrets]commands} drivers/csfle
-    bash .evergreen/start-kmip-server.sh
-    bash .evergreen/setup-encryption-variables.sh
+    bash .evergreen/setup-libmongocrypt.sh
+    bash {env:DRIVERS_TOOLS}/.evergreen/csfle/setup_secrets.sh
+    bash {env:DRIVERS_TOOLS}/.evergreen/csfle/start_servers.sh

 [testenv:teardown-encryption]
 passenv = *
@ -207,5 +210,5 @@ allowlist_externals =
    bash
    rm
 commands =
-    bash .evergreen/stop-kmip-server.sh
-    rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz
+    bash {env:DRIVERS_TOOLS}/.evergreen/csfle/stop_servers.sh
+    rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz mongocryptd.pid