From 68d22b20bd573ff36f89c8e773abc1e4d49b624b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 29 Jan 2024 17:19:24 -0600 Subject: [PATCH] PYTHON-4151 Use Centralized Scripts for CSFLE Tests (#1477) --- .evergreen/config.yml | 22 ++-------- .evergreen/fetch-aws-vault-secrets.sh | 11 ----- .evergreen/run-tests.sh | 45 ++------------------ .evergreen/setup-encryption-variables.sh | 10 ----- .evergreen/setup-libmongocrypt.sh | 47 +++++++++++++++++++++ .evergreen/start-kmip-server.sh | 23 ---------- .evergreen/stop-kmip-server.sh | 9 ---- .evergreen/test-encryption-requirements.txt | 3 -- .gitignore | 3 ++ CONTRIBUTING.md | 7 ++- test/test_encryption.py | 26 ++++++------ tox.ini | 19 +++++---- 12 files changed, 83 insertions(+), 142 deletions(-) delete mode 100644 .evergreen/fetch-aws-vault-secrets.sh delete mode 100644 .evergreen/setup-encryption-variables.sh create mode 100644 .evergreen/setup-libmongocrypt.sh delete mode 100644 .evergreen/start-kmip-server.sh delete mode 100644 .evergreen/stop-kmip-server.sh delete mode 100644 .evergreen/test-encryption-requirements.txt diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c65431f17..3b31dbb13 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -369,24 +369,21 @@ functions: params: working_dir: "src" shell: bash - background: true + include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} if [ -n "${test_encryption}" ]; then - bash ./.evergreen/start-kmip-server.sh + . .evergreen/tox.sh -m setup-encryption fi - command: shell.exec type: test params: working_dir: "src" shell: bash - include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | # Disable xtrace set +x ${PREPARE_SHELL} - # Fetch vault secrets if needed - bash .evergreen/fetch-aws-vault-secrets.sh if [ -n "${MONGODB_STARTED}" ]; then export PYMONGO_MUST_CONNECT=true fi @@ -396,9 +393,7 @@ functions: if [ -n "${test_encryption}" ]; then # Disable xtrace (just in case it was accidentally set). set +x - export LIBMONGOCRYPT_URL="${libmongocrypt_url}" export TEST_ENCRYPTION=1 - . .evergreen/setup-encryption-variables.sh if [ -n "${test_encryption_pyopenssl}" ]; then export TEST_ENCRYPTION_PYOPENSSL=1 fi @@ -619,7 +614,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - . .evergreen/stop-kmip-server.sh + . .evergreen/tox.sh -m teardown-encryption rm -rf $DRIVERS_TOOLS || true rm -f ./secrets-export.sh || true @@ -2164,7 +2159,6 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100 display_name: "macOS 11.00" run_on: macos-1100 @@ -2172,7 +2166,6 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100-arm64 display_name: "macOS 11.00 Arm64" run_on: macos-1100-arm64 @@ -2180,25 +2173,18 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: rhel7 display_name: "RHEL 7.x" run_on: rhel79-small batchtime: 10080 # 7 days - variables: - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel8 display_name: "RHEL 8.x" run_on: rhel87-small batchtime: 10080 # 7 days - variables: - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel80-fips display_name: "RHEL 8.0 FIPS" run_on: rhel80-fips batchtime: 10080 # 7 days - variables: - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz - id: ubuntu-22.04 display_name: "Ubuntu 22.04" run_on: ubuntu2204-small @@ -2220,7 +2206,6 @@ axes: run_on: rhel82-arm64-small batchtime: 10080 # 7 days variables: - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-82-arm64/master/latest/libmongocrypt.tar.gz - id: windows-64-vsMulti-small display_name: "Windows 64" run_on: windows-64-vsMulti-small @@ -2230,7 +2215,6 @@ axes: skip_EC2_auth_test: true skip_web_identity_auth_test: true venv_bin_dir: "Scripts" - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz # Test with authentication? - id: auth diff --git a/.evergreen/fetch-aws-vault-secrets.sh b/.evergreen/fetch-aws-vault-secrets.sh deleted file mode 100644 index 70ccd3d70..000000000 --- a/.evergreen/fetch-aws-vault-secrets.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -AWS_SECRETS_VAULTS="" - -if [ -n "${test_encryption}" ]; then - AWS_SECRETS_VAULTS+="drivers/csfle" -fi - -if [ -n "${AWS_SECRETS_VAULTS}" ]; then - "$DRIVERS_TOOLS"/.evergreen/auth_aws/setup_secrets.sh ${AWS_SECRETS_VAULTS} -fi diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index e8374ab14..17728c0cf 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -111,32 +111,13 @@ fi if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then - # Work around for root certifi not being installed. - # TODO: Remove after PYTHON-3952 is deployed. - if [ "$(uname -s)" = "Darwin" ]; then - python -m pip install certifi - CERT_PATH=$(python -c "import certifi; print(certifi.where())") - export SSL_CERT_FILE=${CERT_PATH} - export REQUESTS_CA_BUNDLE=${CERT_PATH} - export AWS_CA_BUNDLE=${CERT_PATH} - fi - python -m pip install '.[encryption]' - if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - # PYTHON-2808 Ensure this machine has the CA cert for google KMS. - powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true + # Install libmongocrypt if necessary. + if [ ! -d "libmongocrypt" ]; then + bash ./.evergreen/setup-libmongocrypt.sh fi - if [ -z "$LIBMONGOCRYPT_URL" ]; then - echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!" - exit 1 - fi - curl -O "$LIBMONGOCRYPT_URL" - mkdir libmongocrypt - tar xzf libmongocrypt.tar.gz -C ./libmongocrypt - ls -la libmongocrypt - ls -la libmongocrypt/nocrypto # Use the nocrypto build to avoid dependency issues with older windows/python versions. BASE=$(pwd)/libmongocrypt/nocrypto if [ -f "${BASE}/lib/libmongocrypt.so" ]; then @@ -155,11 +136,6 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE exit 1 fi export PYMONGOCRYPT_LIB - - # TODO: Test with 'pip install pymongocrypt' - git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git - python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt - python -m pip install ./libmongocrypt_git/bindings/python python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" # PATH is updated by PREPARE_SHELL for access to mongocryptd. @@ -170,11 +146,6 @@ if [ -n "$TEST_ENCRYPTION" ]; then python -m pip install '.[ocsp]' fi - # Get access to the AWS temporary credentials: - # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN - export AWS_SESSION_TOKEN= - . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh - if [ -n "$TEST_CRYPT_SHARED" ]; then CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH` echo "using crypt_shared_dir $CRYPT_SHARED_DIR" @@ -229,16 +200,6 @@ fi if [ -n "$TEST_AUTH_OIDC" ]; then python -m pip install ".[aws]" - # Work around for root certifi not being installed. - # TODO: Remove after PYTHON-3952 is deployed. - if [ "$(uname -s)" = "Darwin" ]; then - python -m pip install certifi - CERT_PATH=$(python -c "import certifi; print(certifi.where())") - export SSL_CERT_FILE=${CERT_PATH} - export REQUESTS_CA_BUNDLE=${CERT_PATH} - export AWS_CA_BUNDLE=${CERT_PATH} - fi - TEST_ARGS="test/auth_oidc/test_auth_oidc.py" fi diff --git a/.evergreen/setup-encryption-variables.sh b/.evergreen/setup-encryption-variables.sh deleted file mode 100644 index c37ffdd29..000000000 --- a/.evergreen/setup-encryption-variables.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -source ./secrets-export.sh - -{ - echo "export AWS_ACCESS_KEY_ID=$FLE_AWS_KEY"; - echo "export AWS_SECRET_ACCESS_KEY=$FLE_AWS_SECRET"; - echo "export AWS_DEFAULT_REGION=us-east-1"; - echo "export AWS_SESSION_TOKEN="; -} >> ./secrets-export.sh diff --git a/.evergreen/setup-libmongocrypt.sh b/.evergreen/setup-libmongocrypt.sh new file mode 100644 index 000000000..99ca6ebd3 --- /dev/null +++ b/.evergreen/setup-libmongocrypt.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail +set -o xtrace + +TARGET="" + +if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin + # PYTHON-2808 Ensure this machine has the CA cert for google KMS. + powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true + TARGET="windows-test" +fi + +if [ "$(uname -s)" = "Darwin" ]; then + TARGET="macos" +fi + +if [ "$(uname -s)" = "Linux" ]; then + rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release) + arch=$(uname -m) + echo "RHEL $rhel_ver $arch" + if [[ $rhel_ver =~ 7 ]]; then + TARGET="rhel-70-64-bit" + elif [[ $rhel_ver =~ 8 ]]; then + if [ "$arch" = "x86_64" ]; then + TARGET="rhel-80-64-bit" + elif [ "$arch" = "arm" ]; then + TARGET="rhel-82-arm64" + fi + fi +fi + +if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then + LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz" +fi + +if [ -z "$LIBMONGOCRYPT_URL" ]; then + echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!" + exit 1 +fi +rm -rf libmongocrypt libmongocrypt.tar.gz +echo "Fetching $LIBMONGOCRYPT_URL..." +curl -O "$LIBMONGOCRYPT_URL" +echo "Fetching $LIBMONGOCRYPT_URL...done" +mkdir libmongocrypt +tar xzf libmongocrypt.tar.gz -C ./libmongocrypt +ls -la libmongocrypt +ls -la libmongocrypt/nocrypto diff --git a/.evergreen/start-kmip-server.sh b/.evergreen/start-kmip-server.sh deleted file mode 100644 index 9e09a15da..000000000 --- a/.evergreen/start-kmip-server.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -cd "$DRIVERS_TOOLS"/.evergreen/csfle || exit -. ./activate-kmstlsvenv.sh - -# The -u options forces the stdout and stderr streams to be unbuffered. -# TMPDIR is required to avoid "AF_UNIX path too long" errors. -TMPDIR="$(dirname "$DRIVERS_TOOLS")" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 5698 & -python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 & -python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 & -python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert & - -pgrep -f "kms_...._server.py" > kmip_pids.pid - -for _ in $(seq 1 1 10); do - sleep 1 - if python -u kms_kmip_client.py; then - echo 'KMS KMIP server started!' - exit 0 - fi -done -echo 'Failed to start KMIP server!' -exit 1 diff --git a/.evergreen/stop-kmip-server.sh b/.evergreen/stop-kmip-server.sh deleted file mode 100644 index cb5e5c7c9..000000000 --- a/.evergreen/stop-kmip-server.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Clean up CSFLE kmip servers -cd "$DRIVERS_TOOLS"/.evergreen/csfle || exit - -if [ -f "kmip_pids.pid" ]; then - < kmip_pids.pid xargs kill -9 - rm kmip_pids.pid -fi diff --git a/.evergreen/test-encryption-requirements.txt b/.evergreen/test-encryption-requirements.txt deleted file mode 100644 index 13ed7ebb1..000000000 --- a/.evergreen/test-encryption-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -cffi>=1.12.0,<2 -# boto3 is required by drivers-evergreen-tools/.evergreen/csfle/set-temp-creds.sh -boto3<2 diff --git a/.gitignore b/.gitignore index b0f418aa9..69dd20efa 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ mongocryptd.pid .nova/ venv/ secrets-export.sh +libmongocrypt.tar.gz +libmongocrypt/ +libmongocrypt_git/ # Lambda temp files test/lambda/.aws-sam diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89a39669d..ad7b341cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -216,10 +216,9 @@ the pages will re-render and the browser will automatically refresh. `TEST_LOADBALANCER=1 tox -m test-eg`. ## Running Encryption Tests Locally -- Run `AWS_PROFILE= tox -e setup-encryption` after setting up your AWS profile with `aws configure sso` -- Run the tests with `TEST_ENCRYPTION=1 LIBMONGOCRYPT_URL= tox -e test-eg`. - Look in `.evergreen/config.yml` for the correct `LIBMONGOCRYPT_URL` for your platform. -- When done, run `tox -e teardown-encryption` to clean up. +- Run `AWS_PROFILE= tox -m setup-encryption` after setting up your AWS profile with `aws configure sso`. +- Run the tests with `TEST_ENCRYPTION=1 tox -e test-eg`. +- When done, run `tox -m teardown-encryption` to clean up. ## Re-sync Spec Tests diff --git a/test/test_encryption.py b/test/test_encryption.py index 97f3d4b89..89c6d9f7f 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2007,7 +2007,7 @@ class TestKmsTLSProse(EncryptionIntegrationTest): key = { "region": "us-east-1", "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "endpoint": "mongodb://127.0.0.1:8000", + "endpoint": "mongodb://127.0.0.1:9000", } # Some examples: # certificate verify failed: certificate has expired (_ssl.c:1129) @@ -2019,7 +2019,7 @@ class TestKmsTLSProse(EncryptionIntegrationTest): key = { "region": "us-east-1", "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "endpoint": "mongodb://127.0.0.1:8001", + "endpoint": "mongodb://127.0.0.1:9001", } # Some examples: # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)" @@ -2038,8 +2038,8 @@ class TestKmsTLSOptions(EncryptionIntegrationTest): super().setUp() # 1, create client with only tlsCAFile. providers: dict = copy.deepcopy(ALL_KMS_PROVIDERS) - providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8002" - providers["gcp"]["endpoint"] = "127.0.0.1:8002" + providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9002" + providers["gcp"]["endpoint"] = "127.0.0.1:9002" kms_tls_opts_ca_only = { "aws": {"tlsCAFile": CA_PEM}, "azure": {"tlsCAFile": CA_PEM}, @@ -2060,18 +2060,18 @@ class TestKmsTLSOptions(EncryptionIntegrationTest): self.addCleanup(self.client_encryption_with_tls.close) # 3, update endpoints to expired host. providers: dict = copy.deepcopy(providers) - providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8000" - providers["gcp"]["endpoint"] = "127.0.0.1:8000" - providers["kmip"]["endpoint"] = "127.0.0.1:8000" + providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9000" + providers["gcp"]["endpoint"] = "127.0.0.1:9000" + providers["kmip"]["endpoint"] = "127.0.0.1:9000" self.client_encryption_expired = ClientEncryption( providers, "keyvault.datakeys", self.client, OPTS, kms_tls_options=kms_tls_opts_ca_only ) self.addCleanup(self.client_encryption_expired.close) # 3, update endpoints to invalid host. providers: dict = copy.deepcopy(providers) - providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8001" - providers["gcp"]["endpoint"] = "127.0.0.1:8001" - providers["kmip"]["endpoint"] = "127.0.0.1:8001" + providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9001" + providers["gcp"]["endpoint"] = "127.0.0.1:9001" + providers["kmip"]["endpoint"] = "127.0.0.1:9001" self.client_encryption_invalid_hostname = ClientEncryption( providers, "keyvault.datakeys", self.client, OPTS, kms_tls_options=kms_tls_opts_ca_only ) @@ -2095,7 +2095,7 @@ class TestKmsTLSOptions(EncryptionIntegrationTest): key = { "region": "us-east-1", "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "endpoint": "127.0.0.1:8002", + "endpoint": "127.0.0.1:9002", } with self.assertRaisesRegex(EncryptionError, self.cert_error): self.client_encryption_no_client_cert.create_data_key("aws", key) @@ -2105,14 +2105,14 @@ class TestKmsTLSOptions(EncryptionIntegrationTest): # Some examples: # certificate verify failed: certificate has expired (_ssl.c:1129) # amazon1-2018 Python 3.6: certificate verify failed (_ssl.c:852) - key["endpoint"] = "127.0.0.1:8000" + key["endpoint"] = "127.0.0.1:9000" with self.assertRaisesRegex(EncryptionError, "expired|certificate verify failed"): self.client_encryption_expired.create_data_key("aws", key) # Some examples: # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)" # hostname '127.0.0.1' doesn't match 'wronghost.com' # 127.0.0.1:8001: ('Certificate does not contain any `subjectAltName`s.',) - key["endpoint"] = "127.0.0.1:8001" + key["endpoint"] = "127.0.0.1:9001" with self.assertRaisesRegex( EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" ): diff --git a/tox.ini b/tox.ini index f0e7d2174..a542b12fd 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,10 @@ envlist = test, # Test using the run-tests Evergreen script. test-eg, + # Set up encryption files and services. + setup-encryption, + # Tear down encryption files and services. + teardown-encryption, # Run pre-commit on all files. lint, # Run pre-commit on all files, including stages that require manual fixes. @@ -33,6 +37,8 @@ envlist = labels = # Use labels and -m instead of -e so that tox -m