diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f854f6bd3..028caf4d9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -42,7 +42,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "TEST_ENCRYPTION", "TEST_ENCRYPTION_PYOPENSSL", "TEST_CRYPT_SHARED", "TEST_PYOPENSSL", "SETDEFAULTENCODING", "TEST_LOADBALANCER", "TEST_SEVERLESS", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"] binary: bash working_dir: "src" args: @@ -274,39 +274,22 @@ functions: "run tests": - command: subprocess.exec + type: test params: - include_expansions_in_env: ["TEST_DATA_LAKE", "PYTHON_BINARY", "AUTH", "SSL", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"] + include_expansions_in_env: ["TEST_DATA_LAKE", "PYTHON_BINARY", "AUTH", "SSL", + "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_SUITES", + "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "TEST_ENCRYPTION", "TEST_ENCRYPTION_PYOPENSSL", + "TEST_CRYPT_SHARED", "TEST_PYOPENSSL", "TEST_LOADBALANCER", "TEST_SEVERLESS", "MONGODB_URI"] binary: bash working_dir: "src" args: - .evergreen/scripts/setup-tests.sh - - command: subprocess.exec - params: - working_dir: "src" - binary: bash - background: true - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: - - .evergreen/scripts/run-with-env.sh - - .evergreen/scripts/setup-encryption.sh - command: subprocess.exec type: test params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"] - args: - - .evergreen/scripts/run-with-env.sh - - .evergreen/scripts/run-tests.sh - - "run direct tests": - - command: subprocess.exec - type: test - params: - working_dir: "src" - binary: bash - include_expansions_in_env: ["PYTHON_BINARY"] - args: [ .evergreen/scripts/run-direct-tests.sh ] + args: [.evergreen/just.sh, test-eg] "run enterprise auth tests": - command: subprocess.exec @@ -340,13 +323,6 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh "run aws auth test with regular aws credentials": - - command: subprocess.exec - params: - include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -359,13 +335,6 @@ functions: - regular "run aws auth test with assume role credentials": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -378,13 +347,6 @@ functions: - assume-role "run aws auth test with aws EC2 credentials": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -397,13 +359,6 @@ functions: - ec2 "run aws auth test with aws web identity credentials": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - # Test with and without AWS_ROLE_SESSION_NAME set. - command: subprocess.exec type: test @@ -429,13 +384,6 @@ functions: - web-identity "run aws auth test with aws credentials as environment variables": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -448,13 +396,6 @@ functions: - env-creds "run aws auth test with aws credentials and session token as environment variables": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -467,13 +408,6 @@ functions: - session-creds "run oidc auth test with test credentials": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -561,13 +495,6 @@ functions: file: atlas-expansion.yml "run-ocsp-test": - - command: subprocess.exec - params: - include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] - binary: bash - working_dir: "src" - args: - - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -587,25 +514,6 @@ functions: args: - ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh - "run load-balancer": - - command: subprocess.exec - params: - binary: bash - include_expansions_in_env: ["MONGODB_URI"] - args: - - src/.evergreen/scripts/run-with-env.sh - - src/.evergreen/scripts/run-load-balancer.sh - - command: expansions.update - params: - file: lb-expansion.yml - - "stop load-balancer": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/stop-load-balancer.sh - "teardown atlas": - command: subprocess.exec params: @@ -882,6 +790,7 @@ tasks: - func: "run tests" vars: TEST_INDEX_MANAGEMENT: "1" + AUTH: "auth" - name: "mod-wsgi-standalone" tags: ["mod_wsgi"] @@ -935,7 +844,7 @@ tasks: vars: VERSION: "8.0" TOPOLOGY: "replica_set" - - func: "run direct tests" + - func: "run tests" - name: "atlas-connect" tags: ["atlas-connect"] @@ -1503,7 +1412,7 @@ tasks: - name: "testgcpkms-task" commands: - command: subprocess.exec - type: setup + type: test params: working_dir: "src" binary: bash @@ -1531,6 +1440,7 @@ tasks: - name: testazurekms-task commands: - command: subprocess.exec + type: test params: binary: bash working_dir: src diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 37b7a622d..6b1703574 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -8,12 +8,11 @@ tasks: AUTH: auth SSL: ssl LOAD_BALANCER: "true" - - func: run load-balancer - func: run tests vars: AUTH: auth SSL: ssl - test_loadbalancer: "true" + TEST_LOADBALANCER: "true" tags: [load-balancer, auth, ssl] - name: test-load-balancer-noauth-ssl commands: @@ -23,12 +22,11 @@ tasks: AUTH: noauth SSL: ssl LOAD_BALANCER: "true" - - func: run load-balancer - func: run tests vars: AUTH: noauth SSL: ssl - test_loadbalancer: "true" + TEST_LOADBALANCER: "true" tags: [load-balancer, noauth, ssl] - name: test-load-balancer-noauth-nossl commands: @@ -38,12 +36,11 @@ tasks: AUTH: noauth SSL: nossl LOAD_BALANCER: "true" - - func: run load-balancer - func: run tests vars: AUTH: noauth SSL: nossl - test_loadbalancer: "true" + TEST_LOADBALANCER: "true" tags: [load-balancer, noauth, nossl] # Server tests diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 20b89f7e6..531f23eb6 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -318,7 +318,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: /opt/python/3.9/bin/python3 tags: [encryption_tag] - name: encryption-rhel8-python3.13 @@ -331,7 +331,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: /opt/python/3.13/bin/python3 tags: [encryption_tag] - name: encryption-rhel8-pypy3.10 @@ -344,7 +344,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 tags: [encryption_tag] - name: encryption-crypt_shared-rhel8-python3.9 @@ -357,8 +357,8 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: /opt/python/3.9/bin/python3 tags: [encryption_tag] - name: encryption-crypt_shared-rhel8-python3.13 @@ -371,8 +371,8 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: /opt/python/3.13/bin/python3 tags: [encryption_tag] - name: encryption-crypt_shared-rhel8-pypy3.10 @@ -385,8 +385,8 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 tags: [encryption_tag] - name: encryption-pyopenssl-rhel8-python3.9 @@ -399,8 +399,8 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" - test_encryption_pyopenssl: "true" + TEST_ENCRYPTION: "true" + TEST_ENCRYPTION_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/3.9/bin/python3 tags: [encryption_tag] - name: encryption-pyopenssl-rhel8-python3.13 @@ -413,8 +413,8 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" - test_encryption_pyopenssl: "true" + TEST_ENCRYPTION: "true" + TEST_ENCRYPTION_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/3.13/bin/python3 tags: [encryption_tag] - name: encryption-pyopenssl-rhel8-pypy3.10 @@ -427,8 +427,8 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_encryption: "true" - test_encryption_pyopenssl: "true" + TEST_ENCRYPTION: "true" + TEST_ENCRYPTION_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 tags: [encryption_tag] - name: encryption-rhel8-python3.10 @@ -438,7 +438,7 @@ buildvariants: run_on: - rhel87-small expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: /opt/python/3.10/bin/python3 - name: encryption-crypt_shared-rhel8-python3.11 tasks: @@ -447,8 +447,8 @@ buildvariants: run_on: - rhel87-small expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: /opt/python/3.11/bin/python3 - name: encryption-pyopenssl-rhel8-python3.12 tasks: @@ -457,8 +457,8 @@ buildvariants: run_on: - rhel87-small expansions: - test_encryption: "true" - test_encryption_pyopenssl: "true" + TEST_ENCRYPTION: "true" + TEST_ENCRYPTION_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/3.12/bin/python3 - name: encryption-macos-python3.9 tasks: @@ -468,7 +468,7 @@ buildvariants: - macos-14 batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 tags: [encryption_tag] - name: encryption-macos-python3.13 @@ -479,7 +479,7 @@ buildvariants: - macos-14 batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 tags: [encryption_tag] - name: encryption-crypt_shared-macos-python3.9 @@ -490,8 +490,8 @@ buildvariants: - macos-14 batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 tags: [encryption_tag] - name: encryption-crypt_shared-macos-python3.13 @@ -502,8 +502,8 @@ buildvariants: - macos-14 batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 tags: [encryption_tag] - name: encryption-win64-python3.9 @@ -514,7 +514,7 @@ buildvariants: - windows-64-vsMulti-small batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: C:/python/Python39/python.exe tags: [encryption_tag] - name: encryption-win64-python3.13 @@ -525,7 +525,7 @@ buildvariants: - windows-64-vsMulti-small batchtime: 10080 expansions: - test_encryption: "true" + TEST_ENCRYPTION: "true" PYTHON_BINARY: C:/python/Python313/python.exe tags: [encryption_tag] - name: encryption-crypt_shared-win64-python3.9 @@ -536,8 +536,8 @@ buildvariants: - windows-64-vsMulti-small batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: C:/python/Python39/python.exe tags: [encryption_tag] - name: encryption-crypt_shared-win64-python3.13 @@ -548,8 +548,8 @@ buildvariants: - windows-64-vsMulti-small batchtime: 10080 expansions: - test_encryption: "true" - test_crypt_shared: "true" + TEST_ENCRYPTION: "true" + TEST_CRYPT_SHARED: "true" PYTHON_BINARY: C:/python/Python313/python.exe tags: [encryption_tag] @@ -1010,7 +1010,7 @@ buildvariants: - macos-14 batchtime: 10080 expansions: - test_pyopenssl: "true" + TEST_PYOPENSSL: "true" PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 - name: pyopenssl-rhel8-python3.10 tasks: @@ -1021,7 +1021,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_pyopenssl: "true" + TEST_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/3.10/bin/python3 - name: pyopenssl-rhel8-python3.11 tasks: @@ -1032,7 +1032,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_pyopenssl: "true" + TEST_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/3.11/bin/python3 - name: pyopenssl-rhel8-python3.12 tasks: @@ -1043,7 +1043,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_pyopenssl: "true" + TEST_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/3.12/bin/python3 - name: pyopenssl-win64-python3.13 tasks: @@ -1054,7 +1054,7 @@ buildvariants: - windows-64-vsMulti-small batchtime: 10080 expansions: - test_pyopenssl: "true" + TEST_PYOPENSSL: "true" PYTHON_BINARY: C:/python/Python313/python.exe - name: pyopenssl-rhel8-pypy3.10 tasks: @@ -1065,7 +1065,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_pyopenssl: "true" + TEST_PYOPENSSL: "true" PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 # Search index tests @@ -1301,7 +1301,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_serverless: "true" + TEST_SERVERLESS: "true" AUTH: auth SSL: ssl PYTHON_BINARY: /opt/python/3.9/bin/python3 @@ -1313,7 +1313,7 @@ buildvariants: - rhel87-small batchtime: 10080 expansions: - test_serverless: "true" + TEST_SERVERLESS: "true" AUTH: auth SSL: ssl PYTHON_BINARY: /opt/python/3.13/bin/python3 diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh index d1117dcb3..31ca30b3e 100755 --- a/.evergreen/run-azurekms-fail-test.sh +++ b/.evergreen/run-azurekms-fail-test.sh @@ -3,10 +3,9 @@ set -o errexit # Exit the script with error if any of the commands fail HERE=$(dirname ${BASH_SOURCE:-$0}) . $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz -SKIP_SERVERS=1 bash $HERE/setup-encryption.sh +SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ KEY_NAME="${AZUREKMS_KEYNAME}" \ KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ - SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ $HERE/just.sh test-eg -bash $HERE/teardown-encryption.sh +bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh index 28a84a52e..27cb3fb31 100755 --- a/.evergreen/run-azurekms-test.sh +++ b/.evergreen/run-azurekms-test.sh @@ -6,24 +6,23 @@ echo "Copying files ... begin" export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey -export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz -SKIP_SERVERS=1 bash $HERE/setup-encryption.sh +LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz # Set up the remote files to test. git add . git commit -m "add files" || true -git archive -o /tmp/mongo-python-driver.tar HEAD -tar -rf /tmp/mongo-python-driver.tar libmongocrypt -gzip -f /tmp/mongo-python-driver.tar +git archive -o /tmp/mongo-python-driver.tgz HEAD # shellcheck disable=SC2088 -AZUREKMS_SRC="/tmp/mongo-python-driver.tar.gz" AZUREKMS_DST="~/" \ +AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" -AZUREKMS_CMD="tar xf mongo-python-driver.tar.gz" \ +AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash ./.evergreen/just.sh test-eg" \ +AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh +AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Running test ... end" -bash $HERE/teardown-encryption.sh +bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh index 37ec2bfe5..077ca0cb9 100755 --- a/.evergreen/run-gcpkms-test.sh +++ b/.evergreen/run-gcpkms-test.sh @@ -8,20 +8,18 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} -export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz -SKIP_SERVERS=1 bash $HERE/setup-encryption.sh +LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz # Set up the remote files to test. git add . git commit -m "add files" || true -git archive -o /tmp/mongo-python-driver.tar HEAD -tar -rf /tmp/mongo-python-driver.tar libmongocrypt -gzip -f /tmp/mongo-python-driver.tar -GCPKMS_SRC=/tmp/mongo-python-driver.tar.gz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh +git archive -o /tmp/mongo-python-driver.tgz HEAD +GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" -GCPKMS_CMD="tar xf mongo-python-driver.tar.gz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Running test ... end" -bash $HERE/teardown-encryption.sh +bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 91777be22..96d3c0611 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -31,4 +31,5 @@ export AUTH="auth" export SET_XTRACE_ON=1 cd src rm -rf .venv +bash ./.evergreen/just.sh setup-test bash .evergreen/just.sh test-eg diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 46c4f2496..552f9ef08 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -29,7 +29,5 @@ else exit 1 fi -export TEST_AUTH_OIDC=1 -export COVERAGE=1 -export AUTH="auth" +TEST_AUTH_OIDC=1 COVERAGE=1 AUTH="auth" bash ./.evergreen/just.sh setup-test bash ./.evergreen/just.sh test-eg "${@:1}" diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index e6a51b329..d0e001c5f 100755 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -16,4 +16,5 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 export PERF_TEST=1 +bash ./.evergreen/just.sh setup-test bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index fbe310ad1..12935b25a 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -1,283 +1,81 @@ #!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -set -o xtrace +set -eu -# Note: It is assumed that you have already set up a virtual environment before running this file. - -# Supported/used environment variables: -# AUTH Set to enable authentication. Defaults to "noauth" -# SSL Set to enable SSL. Defaults to "nossl" -# GREEN_FRAMEWORK The green framework to test with, if any. -# COVERAGE If non-empty, run the test suite with coverage. -# COMPRESSORS If non-empty, install appropriate compressor. -# LIBMONGOCRYPT_URL The URL to download libmongocrypt. -# TEST_DATA_LAKE If non-empty, run data lake tests. -# TEST_ENCRYPTION If non-empty, run encryption tests. -# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib. -# TEST_SERVERLESS If non-empy, test on serverless. -# TEST_LOADBALANCER If non-empy, test load balancing. -# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure -# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP -# TEST_PYOPENSSL If non-empy, test with PyOpenSSL -# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth -# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism -# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism -# TEST_PERF If non-empty, run performance tests -# TEST_OCSP If non-empty, run OCSP tests -# TEST_ATLAS If non-empty, test Atlas connections -# TEST_INDEX_MANAGEMENT If non-empty, run index management tests -# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL - -AUTH=${AUTH:-noauth} -SSL=${SSL:-nossl} -TEST_SUITES=${TEST_SUITES:-} -TEST_ARGS="${*:1}" +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")" export PIP_QUIET=1 # Quiet by default export PIP_PREFER_BINARY=1 # Prefer binary dists by default +export UV_FROZEN=1 # Do not modify lock files -set +x -PYTHON_IMPL=$(uv run --frozen python -c "import platform; print(platform.python_implementation())") - -# Try to source local Drivers Secrets -if [ -f ./secrets-export.sh ]; then - echo "Sourcing secrets" - source ./secrets-export.sh +# Try to source the env file. +if [ -f $SCRIPT_DIR/scripts/env.sh ]; then + echo "Sourcing env inputs" + . $SCRIPT_DIR/scripts/env.sh else - echo "Not sourcing secrets" + echo "Not sourcing env inputs" fi -# Start compiling the args we'll pass to uv. -# Run in an isolated environment so as not to pollute the base venv. -UV_ARGS=("--isolated --frozen --extra test") +# Ensure there are test inputs. +if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then + echo "Sourcing test inputs" + . $SCRIPT_DIR/scripts/test-env.sh +else + echo "Missing test inputs, please run 'just setup-test'" +fi + +# Source the local secrets export file if available. +if [ -f "$ROOT_DIR/secrets-export.sh" ]; then + . "$ROOT_DIR/secrets-export.sh" +fi + +PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())") # Ensure C extensions if applicable. if [ -z "${NO_EXT:-}" ] && [ "$PYTHON_IMPL" = "CPython" ]; then uv run --frozen tools/fail_if_no_c.py fi -if [ "$AUTH" != "noauth" ]; then - if [ -n "$TEST_DATA_LAKE" ]; then - export DB_USER="mhuser" - export DB_PASSWORD="pencil" - elif [ -n "$TEST_SERVERLESS" ]; then - source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh - export DB_USER=$SERVERLESS_ATLAS_USER - export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD - export MONGODB_URI="$SERVERLESS_URI" - echo "MONGODB_URI=$MONGODB_URI" - export SINGLE_MONGOS_LB_URI=$MONGODB_URI - export MULTI_MONGOS_LB_URI=$MONGODB_URI - elif [ -n "$TEST_AUTH_OIDC" ]; then - export DB_USER=$OIDC_ADMIN_USER - export DB_PASSWORD=$OIDC_ADMIN_PWD - export DB_IP="$MONGODB_URI" - else - export DB_USER="bob" - export DB_PASSWORD="pwd123" - fi - echo "Added auth, DB_USER: $DB_USER" -fi - -if [ -n "$TEST_ENTERPRISE_AUTH" ]; then - UV_ARGS+=("--extra gssapi") - if [ "Windows_NT" = "$OS" ]; then - echo "Setting GSSAPI_PASS" - export GSSAPI_PASS=${SASL_PASS} - export GSSAPI_CANONICALIZE="true" - else - # BUILD-3830 - touch krb5.conf.empty - export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty - - echo "Writing keytab" - echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab - echo "Running kinit" - kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL} - fi - echo "Setting GSSAPI variables" - export GSSAPI_HOST=${SASL_HOST} - export GSSAPI_PORT=${SASL_PORT} - export GSSAPI_PRINCIPAL=${PRINCIPAL} - - export TEST_SUITES="auth" -fi - -if [ -n "$TEST_LOADBALANCER" ]; then - export LOAD_BALANCER=1 - export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}" - export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}" - export TEST_SUITES="load_balancer" -fi - -if [ "$SSL" != "nossl" ]; then - export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" - export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" - - if [ -n "$TEST_LOADBALANCER" ]; then - export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}&tls=true" - export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}&tls=true" - fi -fi - -if [ "$COMPRESSORS" = "snappy" ]; then - UV_ARGS+=("--extra snappy") -elif [ "$COMPRESSORS" = "zstd" ]; then - UV_ARGS+=("--extra zstandard") -fi - -# PyOpenSSL test setup. -if [ -n "$TEST_PYOPENSSL" ]; then - UV_ARGS+=("--extra ocsp") -fi - -if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then - # Check for libmongocrypt download. - if [ ! -d "libmongocrypt" ]; then - echo "Run encryption setup first!" - exit 1 - fi - - UV_ARGS+=("--extra encryption") - # TODO: Test with 'pip install pymongocrypt' - UV_ARGS+=("--group pymongocrypt_source") - - # Use the nocrypto build to avoid dependency issues with older windows/python versions. - BASE=$(pwd)/libmongocrypt/nocrypto - if [ -f "${BASE}/lib/libmongocrypt.so" ]; then - PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so - elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then - PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib - elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then - PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll - # libmongocrypt's windows dll is not marked executable. - chmod +x $PYMONGOCRYPT_LIB - PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB) - elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then - PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so - else - echo "Cannot find libmongocrypt shared object file" - exit 1 - fi - export PYMONGOCRYPT_LIB +if [ -n "${PYMONGOCRYPT_LIB:-}" ]; then # Ensure pymongocrypt is working properly. # shellcheck disable=SC2048 - uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" + uv run ${UV_ARGS} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" # shellcheck disable=SC2048 - uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" + uv run ${UV_ARGS} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" # PATH is updated by configure-env.sh for access to mongocryptd. fi -if [ -n "$TEST_ENCRYPTION" ]; then - if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then - UV_ARGS+=("--extra ocsp") - fi - - if [ -n "$TEST_CRYPT_SHARED" ]; then - CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH` - echo "using crypt_shared_dir $CRYPT_SHARED_DIR" - export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:$DYLD_FALLBACK_LIBRARY_PATH - export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:$LD_LIBRARY_PATH - export PATH=$CRYPT_SHARED_DIR:$PATH - fi - # Only run the encryption tests. - TEST_SUITES="encryption" -fi - -if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then - if [[ -z "$SUCCESS" ]]; then - echo "Must define SUCCESS" - exit 1 - fi - - if echo "$MONGODB_URI" | grep -q "@"; then - echo "MONGODB_URI unexpectedly contains user credentials in FLE test!"; - exit 1 - fi - TEST_SUITES="csfle" -fi - -if [ -n "$TEST_INDEX_MANAGEMENT" ]; then - source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh - export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}" - set +x - export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" - set -x - TEST_SUITES="index_management" -fi - -if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then - TEST_SUITES="data_lake" -fi - -if [ -n "$TEST_ATLAS" ]; then - TEST_SUITES="atlas" -fi - -if [ -n "$TEST_OCSP" ]; then - UV_ARGS+=("--extra ocsp") - TEST_SUITES="ocsp" -fi - -if [ -n "$TEST_AUTH_AWS" ]; then - UV_ARGS+=("--extra aws") - TEST_SUITES="auth_aws" -fi - -if [ -n "$TEST_AUTH_OIDC" ]; then - UV_ARGS+=("--extra aws") - TEST_SUITES="auth_oidc" -fi - -if [ -n "$PERF_TEST" ]; then - UV_ARGS+=("--group perf") - start_time=$(date +%s) - TEST_SUITES="perf" - # PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively - # affects the benchmark results. - TEST_ARGS="test/performance/perf_test.py $TEST_ARGS" -fi - -echo "Running $AUTH tests over $SSL with python $(uv python find)" -uv run --frozen python -c 'import sys; print(sys.version)' - - -# Run the tests, and store the results in Evergreen compatible XUnit XML -# files in the xunit-results/ directory. - -# Run the tests with coverage if requested and coverage is installed. -# Only cover CPython. PyPy reports suspiciously low coverage. -if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then - # Keep in sync with combine-coverage.sh. - # coverage >=5 is needed for relative_files=true. - UV_ARGS+=("--group coverage") - TEST_ARGS="$TEST_ARGS --cov" -fi - -if [ -n "$GREEN_FRAMEWORK" ]; then - UV_ARGS+=("--group $GREEN_FRAMEWORK") -fi +PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())") +echo "Running ${AUTH:-noauth} tests over ${SSL:-nossl} with python $(uv python find)" +uv run python -c 'import sys; print(sys.version)' # Show the installed packages # shellcheck disable=SC2048 -PIP_QUIET=0 uv run ${UV_ARGS[*]} --with pip pip list +PIP_QUIET=0 uv run ${UV_ARGS} --with pip pip list -if [ -z "$GREEN_FRAMEWORK" ]; then - # Use --capture=tee-sys so pytest prints test output inline: - # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html - PYTEST_ARGS="-v --capture=tee-sys --durations=5 $TEST_ARGS" - if [ -n "$TEST_SUITES" ]; then - PYTEST_ARGS="-m $TEST_SUITES $PYTEST_ARGS" - fi - # shellcheck disable=SC2048 - uv run ${UV_ARGS[*]} pytest $PYTEST_ARGS -else - # shellcheck disable=SC2048 - uv run ${UV_ARGS[*]} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS +# Record the start time for a perf test. +if [ -n "${PERF_TEST:-}" ]; then + start_time=$(date +%s) fi +# Run the tests, and store the results in Evergreen compatible XUnit XML +# files in the xunit-results/ directory. +TEST_ARGS=${TEST_ARGS} +if [ "$#" -ne 0 ]; then + TEST_ARGS="$*" +fi +echo "Running tests with $TEST_ARGS and uv args $UV_ARGS..." +if [ -z "${GREEN_FRAMEWORK:-}" ]; then + # shellcheck disable=SC2048 + uv run ${UV_ARGS} pytest $TEST_ARGS +else + # shellcheck disable=SC2048 + uv run ${UV_ARGS} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS +fi +echo "Running tests with $TEST_ARGS... done." + # Handle perf test post actions. -if [ -n "$PERF_TEST" ]; then +if [ -n "${PERF_TEST:-}" ]; then end_time=$(date +%s) elapsed_secs=$((end_time-start_time)) @@ -289,6 +87,6 @@ if [ -n "$PERF_TEST" ]; then fi # Handle coverage post actions. -if [ -n "$COVERAGE" ]; then +if [ -n "${COVERAGE:-}" ]; then rm -rf .pytest_cache fi diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh deleted file mode 100755 index 70a337cd1..000000000 --- a/.evergreen/scripts/archive-mongodb-logs.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -o xtrace -mkdir out_dir -# shellcheck disable=SC2156 -find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; -tar zcvf mongodb-logs.tar.gz -C out_dir/ . -rm -rf out_dir diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index cb018d09f..16212ad3b 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -16,6 +16,18 @@ DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache +DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" +MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" + +# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. +if [ "${CI:-}" == "true" ]; then + PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-} +# We want to use a path that's already on PATH on spawn hosts. +else + PYMONGO_BIN_DIR=$HOME/cli_bin +fi + +PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH" # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin @@ -24,6 +36,9 @@ if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin CARGO_HOME=$(cygpath -m $CARGO_HOME) UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR") UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR") + DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES") + MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES") + PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" @@ -36,7 +51,6 @@ fi export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" -export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" cat < "$SCRIPT_DIR"/env.sh export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" @@ -63,16 +77,28 @@ export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}" export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export CARGO_HOME="$CARGO_HOME" -export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export UV_TOOL_DIR="$UV_TOOL_DIR" export UV_CACHE_DIR="$UV_CACHE_DIR" export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES" -export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH" +export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR" +export PATH="$PATH_EXT" # shellcheck disable=SC2154 export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT +# Write the .env file for drivers-tools. +rm -rf $DRIVERS_TOOLS +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + +cat < ${DRIVERS_TOOLS}/.env +SKIP_LEGACY_SHELL=1 +DRIVERS_TOOLS="$DRIVERS_TOOLS" +MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" +MONGODB_BINARIES="$MONGODB_BINARIES" +TMPDIR="$MONGO_ORCHESTRATION_HOME/db" +EOT + # Skip CSOT tests on non-linux platforms. if [ "$(uname -s)" != "Linux" ]; then echo "export SKIP_CSOT_TESTS=1" >> $SCRIPT_DIR/env.sh diff --git a/.evergreen/scripts/fix-absolute-paths.sh b/.evergreen/scripts/fix-absolute-paths.sh deleted file mode 100755 index eb9433c67..000000000 --- a/.evergreen/scripts/fix-absolute-paths.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -# shellcheck disable=SC2044 -for filename in $(find $DRIVERS_TOOLS -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename -done diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 133783637..59c3c720b 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -344,11 +344,11 @@ def create_encryption_variants() -> list[BuildVariant]: batchtime = BATCHTIME_WEEK def get_encryption_expansions(encryption): - expansions = dict(test_encryption="true") + expansions = dict(TEST_ENCRYPTION="true") if "crypt_shared" in encryption: - expansions["test_crypt_shared"] = "true" + expansions["TEST_CRYPT_SHARED"] = "true" if "PyOpenSSL" in encryption: - expansions["test_encryption_pyopenssl"] = "true" + expansions["TEST_ENCRYPTION_PYOPENSSL"] = "true" return expansions host = DEFAULT_HOST @@ -487,7 +487,7 @@ def create_enterprise_auth_variants(): def create_pyopenssl_variants(): base_name = "PyOpenSSL" batchtime = BATCHTIME_WEEK - expansions = dict(test_pyopenssl="true") + expansions = dict(TEST_PYOPENSSL="true") variants = [] for python in ALL_PYTHONS: @@ -645,7 +645,7 @@ def create_disable_test_commands_variants(): def create_serverless_variants(): host = DEFAULT_HOST batchtime = BATCHTIME_WEEK - expansions = dict(test_serverless="true", AUTH="auth", SSL="ssl") + expansions = dict(TEST_SERVERLESS="true", AUTH="auth", SSL="ssl") tasks = ["serverless_task_group"] base_name = "Serverless" return [ @@ -834,12 +834,9 @@ def create_load_balancer_tasks(): tags = ["load-balancer", auth, ssl] bootstrap_vars = dict(TOPOLOGY="sharded_cluster", AUTH=auth, SSL=ssl, LOAD_BALANCER="true") bootstrap_func = FunctionCall(func="bootstrap mongo-orchestration", vars=bootstrap_vars) - balancer_func = FunctionCall(func="run load-balancer") - test_vars = dict(AUTH=auth, SSL=ssl, test_loadbalancer="true") + test_vars = dict(AUTH=auth, SSL=ssl, TEST_LOADBALANCER="true") test_func = FunctionCall(func="run tests", vars=test_vars) - tasks.append( - EvgTask(name=name, tags=tags, commands=[bootstrap_func, balancer_func, test_func]) - ) + tasks.append(EvgTask(name=name, tags=tags, commands=[bootstrap_func, test_func])) return tasks diff --git a/.evergreen/scripts/init-test-results.sh b/.evergreen/scripts/init-test-results.sh deleted file mode 100755 index 666ac6062..000000000 --- a/.evergreen/scripts/init-test-results.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 39b77199b..31ae4c173 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -2,23 +2,34 @@ set -eu -# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. -if [ "${CI:-}" == "true" ]; then - _BIN_DIR=${DRIVERS_TOOLS_BINARIES:-} -else - _BIN_DIR=$HOME/.local/bin +HERE=$(dirname ${BASH_SOURCE:-$0}) +pushd "$(dirname "$(dirname $HERE)")" > /dev/null + +# Source the env files to pick up common variables. +if [ -f $HERE/env.sh ]; then + . $HERE/env.sh fi +_BIN_DIR=${PYMONGO_BIN_DIR:-$HOME/.local/bin} +export PATH="$PATH:${_BIN_DIR}" # Helper function to pip install a dependency using a temporary python env. function _pip_install() { _HERE=$(dirname ${BASH_SOURCE:-$0}) . $_HERE/../utils.sh _VENV_PATH=$(mktemp -d) + if [ "Windows_NT" = "${OS:-}" ]; then + _VENV_PATH=$(cygpath -m $_VENV_PATH) + fi echo "Installing $2 using pip..." createvirtualenv "$(find_python3)" $_VENV_PATH python -m pip install $1 - ln -s "$(which $2)" $_BIN_DIR/$2 + if [ "Windows_NT" = "${OS:-}" ]; then + ln -s "$(which $2)" $_BIN_DIR/$2.exe + else + ln -s "$(which $2)" $_BIN_DIR/$2 + fi + echo "Installed to ${_BIN_DIR}" echo "Installing $2 using pip... done." } @@ -35,9 +46,6 @@ if ! command -v just 2>/dev/null; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $_TARGET --to "$_BIN_DIR" || { _pip_install rust-just just } - if ! command -v just 2>/dev/null; then - export PATH="$PATH:$_BIN_DIR" - fi echo "Installing just... done." fi @@ -48,8 +56,10 @@ if ! command -v uv 2>/dev/null; then curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || { _pip_install uv uv } - if ! command -v uv 2>/dev/null; then - export PATH="$PATH:$_BIN_DIR" + if [ "Windows_NT" = "${OS:-}" ]; then + chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe" fi echo "Installing uv... done." fi + +popd > /dev/null diff --git a/.evergreen/scripts/make-files-executable.sh b/.evergreen/scripts/make-files-executable.sh deleted file mode 100755 index 806be7c59..000000000 --- a/.evergreen/scripts/make-files-executable.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -# shellcheck disable=SC2044 -for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do - chmod +x "$i" -done diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh index da869e705..f5285a39d 100755 --- a/.evergreen/scripts/prepare-resources.sh +++ b/.evergreen/scripts/prepare-resources.sh @@ -5,10 +5,6 @@ HERE=$(dirname ${BASH_SOURCE:-$0}) pushd $HERE . env.sh -rm -rf $DRIVERS_TOOLS -git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS -echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config - popd # Copy PyMongo's test certificates over driver-evergreen-tools' diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index 30b8d5a61..e5684b7cb 100755 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -4,4 +4,5 @@ set +x set -o errexit bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect -TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg +TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh index e015a34ca..6c300325d 100755 --- a/.evergreen/scripts/run-enterprise-auth-tests.sh +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -5,4 +5,5 @@ set -eu set +x # Use the default python to bootstrap secrets. bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth -TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg +TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh index 594a2984f..61f5e30cc 100755 --- a/.evergreen/scripts/run-gcpkms-fail-test.sh +++ b/.evergreen/scripts/run-gcpkms-fail-test.sh @@ -1,7 +1,8 @@ #!/bin/bash - -. .evergreen/scripts/env.sh +set -eu +HERE=$(dirname ${BASH_SOURCE:-$0}) +. $HERE/env.sh export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz -SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh -SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg +SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh +bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-load-balancer.sh b/.evergreen/scripts/run-load-balancer.sh deleted file mode 100755 index 7d431777e..000000000 --- a/.evergreen/scripts/run-load-balancer.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index 88c3236b3..255f84f29 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -24,10 +24,5 @@ echo "Running MONGODB-AWS authentication tests for $1" # Handle credentials and environment setup. . "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1" -# show test output -set -x - -export TEST_AUTH_AWS=1 -export AUTH="auth" -export SET_XTRACE_ON=1 +TEST_AUTH_AWS=1 AUTH="auth" bash ./.evergreen/just.sh setup-test bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh index 328bd2f20..2b9cbd476 100755 --- a/.evergreen/scripts/run-ocsp-test.sh +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -4,5 +4,6 @@ TEST_OCSP=1 \ PYTHON_BINARY="${PYTHON_BINARY}" \ CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh deleted file mode 100755 index ea923b3f5..000000000 --- a/.evergreen/scripts/run-tests.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Disable xtrace -set +x -if [ -n "${MONGODB_STARTED}" ]; then - export PYMONGO_MUST_CONNECT=true -fi -if [ -n "${DISABLE_TEST_COMMANDS}" ]; then - export PYMONGO_DISABLE_TEST_COMMANDS=1 -fi -if [ -n "${test_encryption}" ]; then - # Disable xtrace (just in case it was accidentally set). - set +x - bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh - export TEST_ENCRYPTION=1 - if [ -n "${test_encryption_pyopenssl}" ]; then - export TEST_ENCRYPTION_PYOPENSSL=1 - fi -fi -if [ -n "${test_crypt_shared}" ]; then - export TEST_CRYPT_SHARED=1 - export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} -fi -if [ -n "${test_pyopenssl}" ]; then - export TEST_PYOPENSSL=1 -fi -if [ -n "${SETDEFAULTENCODING}" ]; then - export SETDEFAULTENCODING="${SETDEFAULTENCODING}" -fi -if [ -n "${test_loadbalancer}" ]; then - export TEST_LOADBALANCER=1 - export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" - export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" -fi -if [ -n "${test_serverless}" ]; then - export TEST_SERVERLESS=1 -fi -if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then - export TEST_INDEX_MANAGEMENT=1 -fi -if [ -n "${SKIP_CSOT_TESTS}" ]; then - export SKIP_CSOT_TESTS=1 -fi -GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ - PYTHON_BINARY=${PYTHON_BINARY} \ - NO_EXT=${NO_EXT} \ - COVERAGE=${COVERAGE} \ - COMPRESSORS=${COMPRESSORS} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ - TEST_SUITES=${TEST_SUITES:-} \ - MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index b56897961..b26dc3ae0 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -1,18 +1,21 @@ #!/bin/bash -set -eu +set -eux HERE=$(dirname ${BASH_SOURCE:-$0}) pushd "$(dirname "$(dirname $HERE)")" > /dev/null -# Source the env file to pick up common variables. +# Source the env files to pick up common variables. if [ -f $HERE/env.sh ]; then - source $HERE/env.sh + . $HERE/env.sh +fi +# PYTHON_BINARY may be defined in test-env.sh. +if [ -f $HERE/test-env.sh ]; then + . $HERE/test-env.sh fi # Ensure dependencies are installed. -. $HERE/install-dependencies.sh - +bash $HERE/install-dependencies.sh # Set the location of the python bin dir. if [ "Windows_NT" = "${OS:-}" ]; then @@ -32,6 +35,12 @@ if [ ! -d $BIN_DIR ]; then echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh echo "Using python $UV_PYTHON" fi + +# Add the default install path to the path if needed. +if [ -z "${PYMONGO_BIN_DIR:-}" ]; then + export PATH="$PATH:$HOME/.local/bin" +fi + uv sync --frozen uv run --frozen --with pip pip install -e . echo "Setting up python environment... done." diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh deleted file mode 100755 index 5b7324020..000000000 --- a/.evergreen/scripts/setup-encryption.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -if [ -n "${test_encryption}" ]; then - bash .evergreen/setup-encryption.sh -fi diff --git a/.evergreen/setup-encryption.sh b/.evergreen/scripts/setup-libmongocrypt.sh similarity index 79% rename from .evergreen/setup-encryption.sh rename to .evergreen/scripts/setup-libmongocrypt.sh index b403ef9ca..775db07cb 100755 --- a/.evergreen/setup-encryption.sh +++ b/.evergreen/scripts/setup-libmongocrypt.sh @@ -2,11 +2,6 @@ set -o errexit # Exit the script with error if any of the commands fail set -o xtrace -if [ -z "${DRIVERS_TOOLS}" ]; then - echo "Missing environment variable DRIVERS_TOOLS" - exit 1 -fi - TARGET="" if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin @@ -51,10 +46,7 @@ tar xzf libmongocrypt.tar.gz -C ./libmongocrypt ls -la libmongocrypt ls -la libmongocrypt/nocrypto -if [ -z "${SKIP_SERVERS:-}" ]; then - PYTHON_BINARY_OLD=${PYTHON_BINARY} - export PYTHON_BINARY="" - bash "${DRIVERS_TOOLS}"/.evergreen/csfle/setup-secrets.sh - export PYTHON_BINARY=$PYTHON_BINARY_OLD - bash "${DRIVERS_TOOLS}"/.evergreen/csfle/start-servers.sh +if [ "Windows_NT" = "${OS:-''}" ]; then + # libmongocrypt's windows dll is not marked executable. + chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll fi diff --git a/.evergreen/scripts/setup-tests.py b/.evergreen/scripts/setup-tests.py new file mode 100644 index 000000000..4e14ff983 --- /dev/null +++ b/.evergreen/scripts/setup-tests.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import base64 +import logging +import os +import platform +import shlex +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any + +HERE = Path(__file__).absolute().parent +ROOT = HERE.parent.parent +ENV_FILE = HERE / "test-env.sh" +DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + +EXPECTED_VARS = [ + "TEST_ENCRYPTION", + "TEST_ENCRYPTION_PYOPENSSL", + "TEST_CRYPT_SHARED", + "TEST_PYOPENSSL", + "TEST_LOAD_BALANCER", + "TEST_SERVERLESS", + "TEST_INDEX_MANAGEMENT", + "TEST_ENTERPRISE_AUTH", + "TEST_FLE_AZURE_AUTO", + "TEST_FLE_GCP_AUTO", + "TEST_LOADBALANCER", + "TEST_DATA_LAKE", + "TEST_ATLAS", + "TEST_OCSP", + "TEST_AUTH_AWS", + "TEST_AUTH_OIDC", + "COMPRESSORS", + "MONGODB_URI", + "PERF_TEST", + "GREEN_FRAMEWORK", + "PYTHON_BINARY", + "LIBMONGOCRYPT_URL", +] + +# Handle the test suite based on the presence of env variables. +TEST_SUITE_MAP = dict( + TEST_DATA_LAKE="data_lake", + TEST_AUTH_OIDC="auth_oidc", + TEST_INDEX_MANAGEMENT="index_management", + TEST_ENTERPRISE_AUTH="auth", + TEST_LOADBALANCER="load_balancer", + TEST_ENCRYPTION="encryption", + TEST_FLE_AZURE_AUTO="csfle", + TEST_FLE_GCP_AUTO="csfle", + TEST_ATLAS="atlas", + TEST_OCSP="ocsp", + TEST_AUTH_AWS="auth_aws", + PERF_TEST="perf", +) + +# Handle extras based on the presence of env variables. +EXTRAS_MAP = dict( + TEST_AUTH_OIDC="aws", + TEST_AUTH_AWS="aws", + TEST_OCSP="ocsp", + TEST_PYOPENSSL="ocsp", + TEST_ENTERPRISE_AUTH="gssapi", + TEST_ENCRYPTION="encryption", + TEST_FLE_AZURE_AUTO="encryption", + TEST_FLE_GCP_AUTO="encryption", + TEST_ENCRYPTION_PYOPENSSL="ocsp", +) + + +def write_env(name: str, value: Any) -> None: + with ENV_FILE.open("a", newline="\n") as fid: + # Remove any existing quote chars. + value = str(value).replace('"', "") + fid.write(f'export {name}="{value}"\n') + + +def is_set(var: str) -> bool: + value = os.environ.get(var, "") + return len(value.strip()) > 0 + + +def run_command(cmd: str) -> None: + LOGGER.info("Running command %s...", cmd) + subprocess.check_call(shlex.split(cmd)) # noqa: S603 + LOGGER.info("Running command %s... done.", cmd) + + +def handle_test_env() -> None: + AUTH = os.environ.get("AUTH", "noauth") + SSL = os.environ.get("SSL", "nossl") + TEST_SUITES = os.environ.get("TEST_SUITES", "") + TEST_ARGS = "" + # Start compiling the args we'll pass to uv. + # Run in an isolated environment so as not to pollute the base venv. + UV_ARGS = ["--isolated --extra test"] + + # Save variables in EXPECTED_VARS that have values. + with ENV_FILE.open("w", newline="\n") as fid: + fid.write("#!/usr/bin/env bash\n") + fid.write("set +x\n") + fid.write(f"export AUTH={AUTH}\n") + fid.write(f"export SSL={SSL}\n") + for var in EXPECTED_VARS: + value = os.environ.get(var, "") + # Remove any existing quote chars. + value = value.replace('"', "") + if value: + fid.write(f'export {var}="{value}"\n') + ENV_FILE.chmod(ENV_FILE.stat().st_mode | stat.S_IEXEC) + + for env_var, extra in EXTRAS_MAP.items(): + if env_var in os.environ: + UV_ARGS.append(f"--extra {extra}") + + for env_var, suite in TEST_SUITE_MAP.items(): + if TEST_SUITES: + break + if env_var in os.environ: + TEST_SUITES = suite + + if AUTH != "noauth": + if is_set("TEST_DATA_LAKE"): + DB_USER = os.environ["ADL_USERNAME"] + DB_PASSWORD = os.environ["ADL_PASSWORD"] + elif is_set("TEST_SERVERLESS"): + DB_USER = os.environ("SERVERLESS_ATLAS_USER") + DB_PASSWORD = os.environ("SERVERLESS_ATLAS_PASSWORD") + write_env("MONGODB_URI", os.environ("SERVERLESS_URI")) + write_env("SINGLE_MONGOS_LB_URI", os.environ("SERVERLESS_URI")) + write_env("MULTI_MONGOS_LB_URI", os.environ("SERVERLESS_URI")) + elif is_set("TEST_AUTH_OIDC"): + DB_USER = os.environ["OIDC_ADMIN_USER"] + DB_PASSWORD = os.environ["OIDC_ADMIN_PWD"] + write_env("DB_IP", os.environ["MONGODB_URI"]) + elif is_set("TEST_INDEX_MANAGEMENT"): + DB_USER = os.environ["DRIVERS_ATLAS_LAMBDA_USER"] + DB_PASSWORD = os.environ["DRIVERS_ATLAS_LAMBDA_PASSWORD"] + else: + DB_USER = "bob" + DB_PASSWORD = "pwd123" # noqa: S105 + write_env("DB_USER", DB_USER) + write_env("DB_PASSWORD", DB_PASSWORD) + LOGGER.info("Added auth, DB_USER: %s", DB_USER) + + if is_set("MONGODB_STARTED"): + write_env("PYMONGO_MUST_CONNECT", "true") + + if is_set("DISABLE_TEST_COMMANDS"): + write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") + + if is_set("TEST_ENTERPRISE_AUTH"): + if os.name == "nt": + LOGGER.info("Setting GSSAPI_PASS") + write_env("GSSAPI_PASS", os.environ["SASL_PASS"]) + write_env("GSSAPI_CANONICALIZE", "true") + else: + # BUILD-3830 + krb_conf = ROOT / ".evergreen/krb5.conf.empty" + krb_conf.touch() + write_env("KRB5_CONFIG", krb_conf) + LOGGER.info("Writing keytab") + keytab = base64.b64decode(os.environ["KEYTAB_BASE64"]) + keytab_file = ROOT / ".evergreen/drivers.keytab" + with keytab_file.open("wb") as fid: + fid.write(keytab) + principal = os.environ["PRINCIPAL"] + LOGGER.info("Running kinit") + os.environ["KRB5_CONFIG"] = str(krb_conf) + cmd = f"kinit -k -t {keytab_file} -p {principal}" + run_command(cmd) + + LOGGER.info("Setting GSSAPI variables") + write_env("GSSAPI_HOST", os.environ["SASL_HOST"]) + write_env("GSSAPI_PORT", os.environ["SASL_PORT"]) + write_env("GSSAPI_PRINCIPAL", os.environ["PRINCIPAL"]) + + if is_set("TEST_LOADBALANCER"): + write_env("LOAD_BALANCER", "1") + SINGLE_MONGOS_LB_URI = os.environ.get( + "SINGLE_MONGOS_LB_URI", "mongodb://127.0.0.1:8000/?loadBalanced=true" + ) + MULTI_MONGOS_LB_URI = os.environ.get( + "MULTI_MONGOS_LB_URI", "mongodb://127.0.0.1:8001/?loadBalanced=true" + ) + if SSL != "nossl": + SINGLE_MONGOS_LB_URI += "&tls=true" + MULTI_MONGOS_LB_URI += "&tls=true" + write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI) + write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI) + if not DRIVERS_TOOLS: + raise RuntimeError("Missing DRIVERS_TOOLS") + cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' + run_command(cmd) + + if SSL != "nossl": + if not DRIVERS_TOOLS: + raise RuntimeError("Missing DRIVERS_TOOLS") + write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") + write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") + + compressors = os.environ.get("COMPRESSORS") + if compressors == "snappy": + UV_ARGS.append("--extra snappy") + elif compressors == "zstd": + UV_ARGS.append("--extra zstandard") + + if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"): + # Check for libmongocrypt download. + if not (ROOT / "libmongocrypt").exists(): + run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh") + + # TODO: Test with 'pip install pymongocrypt' + UV_ARGS.append("--group pymongocrypt_source") + + # Use the nocrypto build to avoid dependency issues with older windows/python versions. + BASE = ROOT / "libmongocrypt/nocrypto" + if sys.platform == "linux": + if (BASE / "lib/libmongocrypt.so").exists(): + PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so" + else: + PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so" + elif sys.platform == "darwin": + PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib" + else: + PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll" + if not PYMONGOCRYPT_LIB.exists(): + raise RuntimeError("Cannot find libmongocrypt shared object file") + write_env("PYMONGOCRYPT_LIB", PYMONGOCRYPT_LIB.as_posix()) + # PATH is updated by configure-env.sh for access to mongocryptd. + + if is_set("TEST_ENCRYPTION"): + if not DRIVERS_TOOLS: + raise RuntimeError("Missing DRIVERS_TOOLS") + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh") + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh") + + if is_set("TEST_CRYPT_SHARED"): + CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix() + LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR) + if os.name == "nt": + write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH") + else: + write_env( + "DYLD_FALLBACK_LIBRARY_PATH", + f"{CRYPT_SHARED_DIR}:${{DYLD_FALLBACK_LIBRARY_PATH:-}}", + ) + write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}") + + if is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"): + if "SUCCESS" not in os.environ: + raise RuntimeError("Must define SUCCESS") + + write_env("SUCCESS", os.environ["SUCCESS"]) + MONGODB_URI = os.environ.get("MONGODB_URI", "") + if "@" in MONGODB_URI: + raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!") + + if is_set("TEST_OCSP"): + write_env("CA_FILE", os.environ["CA_FILE"]) + write_env("OCSP_TLS_SHOULD_SUCCEED", os.environ["OCSP_TLS_SHOULD_SUCCEED"]) + + if is_set("PERF_TEST"): + UV_ARGS.append("--group perf") + # PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively + # affects the benchmark results. + TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}" + + # Add coverage if requested. + # Only cover CPython. PyPy reports suspiciously low coverage. + if is_set("COVERAGE") and platform.python_implementation() == "CPython": + # Keep in sync with combine-coverage.sh. + # coverage >=5 is needed for relative_files=true. + UV_ARGS.append("--group coverage") + TEST_ARGS = f"{TEST_ARGS} --cov" + + if is_set("GREEN_FRAMEWORK"): + framework = os.environ["GREEN_FRAMEWORK"] + UV_ARGS.append(f"--group {framework}") + + else: + # Use --capture=tee-sys so pytest prints test output inline: + # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html + TEST_ARGS = f"-v --capture=tee-sys --durations=5 {TEST_ARGS}" + if TEST_SUITES: + TEST_ARGS = f"-m {TEST_SUITES} {TEST_ARGS}" + + write_env("TEST_ARGS", TEST_ARGS) + write_env("UV_ARGS", " ".join(UV_ARGS)) + + +if __name__ == "__main__": + handle_test_env() diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index 65462b2a6..80fc71704 100755 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -1,27 +1,61 @@ -#!/bin/bash -eux +#!/bin/bash +set -eu -PROJECT_DIRECTORY="$(pwd)" -SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" +# Supported/used environment variables: +# AUTH Set to enable authentication. Defaults to "noauth" +# SSL Set to enable SSL. Defaults to "nossl" +# GREEN_FRAMEWORK The green framework to test with, if any. +# COVERAGE If non-empty, run the test suite with coverage. +# COMPRESSORS If non-empty, install appropriate compressor. +# LIBMONGOCRYPT_URL The URL to download libmongocrypt. +# TEST_DATA_LAKE If non-empty, run data lake tests. +# TEST_ENCRYPTION If non-empty, run encryption tests. +# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib. +# TEST_SERVERLESS If non-empy, test on serverless. +# TEST_LOADBALANCER If non-empy, test load balancing. +# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure +# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP +# TEST_PYOPENSSL If non-empy, test with PyOpenSSL +# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth +# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism +# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism +# TEST_PERF If non-empty, run performance tests +# TEST_OCSP If non-empty, run OCSP tests +# TEST_ATLAS If non-empty, test Atlas connections +# TEST_INDEX_MANAGEMENT If non-empty, run index management tests +# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL +# PERF_TEST If non-empty, run the performance tests. +# MONGODB_URI If non-empty, use as the MONGODB_URI in tests. +# PYTHON_BINARY The python binary to use in tests. -if [ -f "$SCRIPT_DIR/test-env.sh" ]; then - echo "Reading $SCRIPT_DIR/test-env.sh file" - . "$SCRIPT_DIR/test-env.sh" - exit 0 +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")" + +# Try to source the env file. +if [ -f $SCRIPT_DIR/env.sh ]; then + source $SCRIPT_DIR/env.sh fi -cat < "$SCRIPT_DIR"/test-env.sh -export test_encryption="${test_encryption:-}" -export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" -export test_crypt_shared="${test_crypt_shared:-}" -export test_pyopenssl="${test_pyopenssl:-}" -export test_loadbalancer="${test_loadbalancer:-}" -export test_serverless="${test_serverless:-}" -export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" -export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" -export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" -export AUTH="${AUTH:-noauth}" -export SSL="${SSL:-nossl}" -export PYTHON_BINARY="${PYTHON_BINARY:-}" -EOT +# Source serverless secrets if applicable. +if [ -n "${TEST_SERVERLESS:-}" ]; then + source $DRIVERS_TOOLS/.evergreen/serverless/secrets-export.sh +fi -chmod +x "$SCRIPT_DIR"/test-env.sh +# Source atlas secrets if applicable. +if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then + source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh +fi + +# Source ADL secrets if applicable. +if [ -n "${TEST_DATA_LAKE:-}" ]; then + source ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh +fi + +# Source local secrets if applicable. +if [ -f "$ROOT_DIR/secrets-export.sh" ]; then + source "$ROOT_DIR/secrets-export.sh" +fi + +. $ROOT_DIR/.evergreen/utils.sh +PYTHON=${PYTHON_BINARY:-$(find_python3)} +$PYTHON $SCRIPT_DIR/setup-tests.py diff --git a/.evergreen/scripts/stop-load-balancer.sh b/.evergreen/scripts/stop-load-balancer.sh deleted file mode 100755 index 2d3c5366e..000000000 --- a/.evergreen/scripts/stop-load-balancer.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cd "${DRIVERS_TOOLS}"/.evergreen || exit -DRIVERS_TOOLS=${DRIVERS_TOOLS} -bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh new file mode 100755 index 000000000..9c78c0965 --- /dev/null +++ b/.evergreen/scripts/teardown-tests.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -eu + +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")" + +# Remove temporary test files. +pushd $ROOT_DIR > /dev/null +rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid > /dev/null +popd > /dev/null + +if [ ! -f $SCRIPT_DIR/test-env.sh ]; then + exit 0 +fi +if [ -f $SCRIPT_DIR/env.sh ]; then + source $SCRIPT_DIR/env.sh +fi + +source $SCRIPT_DIR/test-env.sh + +# Shut down csfle servers if applicable +if [ -n "${TEST_ENCRYPTION:-}" ]; then + bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh +fi + +# Shut down load balancer if applicable. +if [ -n "${TEST_LOADBALANCER:-}" ]; then + bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop +fi diff --git a/.evergreen/scripts/windows-fix.sh b/.evergreen/scripts/windows-fix.sh deleted file mode 100755 index cb4fa4413..000000000 --- a/.evergreen/scripts/windows-fix.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -# shellcheck disable=SC2044 -for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do - < "$i" tr -d '\r' >"$i".new - mv "$i".new "$i" -done -# Copy client certificate because symlinks do not work on Windows. -cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem diff --git a/.evergreen/setup-spawn-host.sh b/.evergreen/setup-spawn-host.sh index c20e1c756..4c8fa65e2 100755 --- a/.evergreen/setup-spawn-host.sh +++ b/.evergreen/setup-spawn-host.sh @@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_ echo "Copying files to $target... done" ssh $target $remote_dir/.evergreen/scripts/setup-system.sh -ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} just install" +ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} .evergreen/scripts/setup-dev-env.sh" diff --git a/.evergreen/sync-spawn-host.sh b/.evergreen/sync-spawn-host.sh index de3374a00..3f6540ad4 100755 --- a/.evergreen/sync-spawn-host.sh +++ b/.evergreen/sync-spawn-host.sh @@ -7,9 +7,12 @@ fi target=$1 user=${target%@*} +remote_dir=/home/$user/mongo-python-driver +echo "Copying files to $target..." +rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_dir +echo "Copying files to $target... done." echo "Syncing files to $target..." -rsync -haz -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver # shellcheck disable=SC2034 fswatch -o . | while read f; do rsync -hazv -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver; done echo "Syncing files to $target... done." diff --git a/.evergreen/teardown-encryption.sh b/.evergreen/teardown-encryption.sh deleted file mode 100755 index 5ce2f1d71..000000000 --- a/.evergreen/teardown-encryption.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -set -o xtrace - -if [ -z "${DRIVERS_TOOLS}" ]; then - echo "Missing environment variable DRIVERS_TOOLS" -fi - -bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh -rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 3760e308a..5f95aa4a5 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -22,13 +22,13 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: 'pip' - cache-dependency-path: 'pyproject.toml' - name: Install just uses: extractions/setup-just@v2 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: "3.9" - name: Install Python dependencies run: | just install @@ -61,42 +61,21 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - if: ${{ matrix.python-version == '3.13t' }} - name: Setup free-threaded Python - uses: deadsnakes/action@v3.2.0 - with: - python-version: 3.13 - nogil: true - - if: ${{ matrix.python-version != '3.13t' }} - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: 'pyproject.toml' - allow-prereleases: true - name: Install just uses: extractions/setup-just@v2 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | - if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then - # Just can't be installed on 3.13t, use pytest directly. - pip install . - pip install -r requirements/test.txt - else - just install - fi + run: just install - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 with: mongodb-version: 6.0 - name: Run tests - run: | - if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then - pytest -v --durations=5 --maxfail=10 - else - just test - fi + run: just test doctest: runs-on: ubuntu-latest @@ -105,24 +84,21 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: 'pip' - cache-dependency-path: 'pyproject.toml' - name: Install just uses: extractions/setup-just@v2 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: "3.9" - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 with: mongodb-version: '8.0.0-rc4' - name: Install dependencies - run: | - just install + run: just install - name: Run tests - run: | - just docs-test + run: just docs-test docs: name: Docs Checks @@ -131,20 +107,17 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v5 with: - cache: 'pip' - cache-dependency-path: 'pyproject.toml' - # Build docs on lowest supported Python for furo - python-version: '3.9' + enable-cache: true + python-version: "3.9" - name: Install just uses: extractions/setup-just@v2 - name: Install dependencies - run: | - just install + run: just install - name: Build docs - run: | - just docs + run: just docs linkcheck: name: Link Check @@ -153,20 +126,17 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v5 with: - cache: 'pip' - cache-dependency-path: 'pyproject.toml' - # Build docs on lowest supported Python for furo - python-version: '3.9' + enable-cache: true + python-version: "3.9" - name: Install just uses: extractions/setup-just@v2 - name: Install dependencies - run: | - just install + run: just install - name: Build docs - run: | - just docs-linkcheck + run: just docs-linkcheck typing: name: Typing Tests @@ -178,11 +148,11 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v5 with: + enable-cache: true python-version: "${{matrix.python}}" - cache: 'pip' - cache-dependency-path: 'pyproject.toml' - name: Install just uses: extractions/setup-just@v2 - name: Install dependencies diff --git a/.gitignore b/.gitignore index 2582c517f..8c095c215 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ libmongocrypt/ expansion.yml *expansions.yml .evergreen/scripts/env.sh +.evergreen/scripts/test-env.sh # Lambda temp files test/lambda/.aws-sam diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f67077e57..ce5bebf18 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -211,18 +211,18 @@ the pages will re-render and the browser will automatically refresh. `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. - Start the servers using `LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh`. -- Start the load balancer using: - `MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`. +- Set up the test using: + `MONGODB_URI='mongodb://localhost:27017,localhost:27018/' TEST_LOADBALANCER=1 just setup-test`. - Run the tests from the `pymongo` checkout directory using: - `TEST_LOADBALANCER=1 just test-eg`. + `just test-eg`. ## Running Encryption Tests Locally - Clone `drivers-evergreen-tools`: `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. - Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools` -- Run `AWS_PROFILE= just setup-encryption` after setting up your AWS profile with `aws configure sso`. -- Run the tests with `TEST_ENCRYPTION=1 just test-eg`. -- When done, run `just teardown-encryption` to clean up. +- Run `TEST_ENCRYPTION=1 AWS_PROFILE= just setup-test` after setting up your AWS profile with `aws configure sso`. +- Run the tests with `just test-eg`. +- When done, run `just teardown-test` to clean up. ## Re-sync Spec Tests diff --git a/justfile b/justfile index 8a076038a..3840484bc 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,5 @@ # See https://just.systems/man/en/ for instructions set shell := ["bash", "-c"] -set dotenv-load -set dotenv-filename := "./.evergreen/scripts/env.sh" # Commonly used command segments. uv_run := "uv run --isolated --frozen " @@ -70,10 +68,10 @@ test-mockupdb *args: test-eg *args: bash ./.evergreen/run-tests.sh {{args}} -[group('encryption')] -setup-encryption: - bash .evergreen/setup-encryption.sh +[group('test')] +setup-test: + bash .evergreen/scripts/setup-tests.sh -[group('encryption')] -teardown-encryption: - bash .evergreen/teardown-encryption.sh +[group('test')] +teardown-test: + bash .evergreen/scripts/teardown-tests.sh diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py new file mode 100644 index 000000000..09b8fb085 --- /dev/null +++ b/test/asynchronous/test_pooling.py @@ -0,0 +1,595 @@ +# Copyright 2009-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test built in connection-pooling with threads.""" +from __future__ import annotations + +import asyncio +import gc +import random +import socket +import sys +import time + +from bson.codec_options import DEFAULT_CODEC_OPTIONS +from bson.son import SON +from pymongo import AsyncMongoClient, message, timeout +from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError +from pymongo.hello import HelloCompat +from pymongo.lock import _async_create_lock + +sys.path[0:0] = [""] + +from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest +from test.asynchronous.helpers import ConcurrentRunner +from test.utils import async_get_pool, async_joinall, delay + +from pymongo.asynchronous.pool import Pool, PoolOptions +from pymongo.socket_checker import SocketChecker + +_IS_SYNC = False + + +N = 10 +DB = "pymongo-pooling-tests" + + +async def gc_collect_until_done(tasks, timeout=60): + start = time.time() + running = list(tasks) + while running: + assert (time.time() - start) < timeout, "Tasks timed out" + for t in running: + await t.join(0.1) + if not t.is_alive(): + running.remove(t) + gc.collect() + + +class MongoTask(ConcurrentRunner): + """A thread/Task that uses a AsyncMongoClient.""" + + def __init__(self, client): + super().__init__() + self.daemon = True # Don't hang whole test if task hangs. + self.client = client + self.db = self.client[DB] + self.passed = False + + async def run(self): + await self.run_mongo_thread() + self.passed = True + + async def run_mongo_thread(self): + raise NotImplementedError + + +class InsertOneAndFind(MongoTask): + async def run_mongo_thread(self): + for _ in range(N): + rand = random.randint(0, N) + _id = (await self.db.sf.insert_one({"x": rand})).inserted_id + assert rand == (await self.db.sf.find_one(_id))["x"] + + +class Unique(MongoTask): + async def run_mongo_thread(self): + for _ in range(N): + await self.db.unique.insert_one({}) # no error + + +class NonUnique(MongoTask): + async def run_mongo_thread(self): + for _ in range(N): + try: + await self.db.unique.insert_one({"_id": "jesse"}) + except DuplicateKeyError: + pass + else: + raise AssertionError("Should have raised DuplicateKeyError") + + +class SocketGetter(MongoTask): + """Utility for TestPooling. + + Checks out a socket and holds it forever. Used in + test_no_wait_queue_timeout. + """ + + def __init__(self, client, pool): + super().__init__(client) + self.state = "init" + self.pool = pool + self.sock = None + + async def run_mongo_thread(self): + self.state = "get_socket" + + # Call 'pin_cursor' so we can hold the socket. + async with self.pool.checkout() as sock: + sock.pin_cursor() + self.sock = sock + + self.state = "connection" + + def __del__(self): + if self.sock: + self.sock.close_conn(None) + + +async def run_cases(client, cases): + tasks = [] + n_runs = 5 + + for case in cases: + for _i in range(n_runs): + t = case(client) + await t.start() + tasks.append(t) + + for t in tasks: + await t.join() + + for t in tasks: + assert t.passed, "%s.run() threw an exception" % repr(t) + + +class _TestPoolingBase(AsyncIntegrationTest): + """Base class for all connection-pool tests.""" + + @async_client_context.require_connection + async def asyncSetUp(self): + await super().asyncSetUp() + self.c = await self.async_rs_or_single_client() + db = self.c[DB] + await db.unique.drop() + await db.test.drop() + await db.unique.insert_one({"_id": "jesse"}) + await db.test.insert_many([{} for _ in range(10)]) + + async def create_pool(self, pair=None, *args, **kwargs): + if pair is None: + pair = (await async_client_context.host, await async_client_context.port) + # Start the pool with the correct ssl options. + pool_options = async_client_context.client._topology_settings.pool_options + kwargs["ssl_context"] = pool_options._ssl_context + kwargs["tls_allow_invalid_hostnames"] = pool_options.tls_allow_invalid_hostnames + kwargs["server_api"] = pool_options.server_api + pool = Pool(pair, PoolOptions(*args, **kwargs)) + await pool.ready() + return pool + + +class TestPooling(_TestPoolingBase): + async def test_max_pool_size_validation(self): + host, port = await async_client_context.host, await async_client_context.port + self.assertRaises(ValueError, AsyncMongoClient, host=host, port=port, maxPoolSize=-1) + + self.assertRaises(ValueError, AsyncMongoClient, host=host, port=port, maxPoolSize="foo") + + c = AsyncMongoClient(host=host, port=port, maxPoolSize=100, connect=False) + self.assertEqual(c.options.pool_options.max_pool_size, 100) + + async def test_no_disconnect(self): + await run_cases(self.c, [NonUnique, Unique, InsertOneAndFind]) + + async def test_pool_reuses_open_socket(self): + # Test Pool's _check_closed() method doesn't close a healthy socket. + cx_pool = await self.create_pool(max_pool_size=10) + cx_pool._check_interval_seconds = 0 # Always check. + async with cx_pool.checkout() as conn: + pass + + async with cx_pool.checkout() as new_connection: + self.assertEqual(conn, new_connection) + + self.assertEqual(1, len(cx_pool.conns)) + + async def test_get_socket_and_exception(self): + # get_socket() returns socket after a non-network error. + cx_pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=1) + with self.assertRaises(ZeroDivisionError): + async with cx_pool.checkout() as conn: + 1 / 0 + + # Socket was returned, not closed. + async with cx_pool.checkout() as new_connection: + self.assertEqual(conn, new_connection) + + self.assertEqual(1, len(cx_pool.conns)) + + async def test_pool_removes_closed_socket(self): + # Test that Pool removes explicitly closed socket. + cx_pool = await self.create_pool() + + async with cx_pool.checkout() as conn: + # Use Connection's API to close the socket. + conn.close_conn(None) + + self.assertEqual(0, len(cx_pool.conns)) + + async def test_pool_removes_dead_socket(self): + # Test that Pool removes dead socket and the socket doesn't return + # itself PYTHON-344 + cx_pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=1) + cx_pool._check_interval_seconds = 0 # Always check. + + async with cx_pool.checkout() as conn: + # Simulate a closed socket without telling the Connection it's + # closed. + conn.conn.close() + self.assertTrue(conn.conn_closed()) + + async with cx_pool.checkout() as new_connection: + self.assertEqual(0, len(cx_pool.conns)) + self.assertNotEqual(conn, new_connection) + + self.assertEqual(1, len(cx_pool.conns)) + + # Semaphore was released. + async with cx_pool.checkout(): + pass + + async def test_socket_closed(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((await async_client_context.host, await async_client_context.port)) + socket_checker = SocketChecker() + self.assertFalse(socket_checker.socket_closed(s)) + s.close() + self.assertTrue(socket_checker.socket_closed(s)) + + async def test_socket_checker(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((await async_client_context.host, await async_client_context.port)) + socket_checker = SocketChecker() + # Socket has nothing to read. + self.assertFalse(socket_checker.select(s, read=True)) + self.assertFalse(socket_checker.select(s, read=True, timeout=0)) + self.assertFalse(socket_checker.select(s, read=True, timeout=0.05)) + # Socket is writable. + self.assertTrue(socket_checker.select(s, write=True, timeout=None)) + self.assertTrue(socket_checker.select(s, write=True)) + self.assertTrue(socket_checker.select(s, write=True, timeout=0)) + self.assertTrue(socket_checker.select(s, write=True, timeout=0.05)) + # Make the socket readable + _, msg, _ = message._query( + 0, "admin.$cmd", 0, -1, SON([("ping", 1)]), None, DEFAULT_CODEC_OPTIONS + ) + s.sendall(msg) + # Block until the socket is readable. + self.assertTrue(socket_checker.select(s, read=True, timeout=None)) + self.assertTrue(socket_checker.select(s, read=True)) + self.assertTrue(socket_checker.select(s, read=True, timeout=0)) + self.assertTrue(socket_checker.select(s, read=True, timeout=0.05)) + # Socket is still writable. + self.assertTrue(socket_checker.select(s, write=True, timeout=None)) + self.assertTrue(socket_checker.select(s, write=True)) + self.assertTrue(socket_checker.select(s, write=True, timeout=0)) + self.assertTrue(socket_checker.select(s, write=True, timeout=0.05)) + s.close() + self.assertTrue(socket_checker.socket_closed(s)) + + async def test_return_socket_after_reset(self): + pool = await self.create_pool() + async with pool.checkout() as sock: + self.assertEqual(pool.active_sockets, 1) + self.assertEqual(pool.operation_count, 1) + await pool.reset() + + self.assertTrue(sock.closed) + self.assertEqual(0, len(pool.conns)) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) + + async def test_pool_check(self): + # Test that Pool recovers from two connection failures in a row. + # This exercises code at the end of Pool._check(). + cx_pool = await self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) + cx_pool._check_interval_seconds = 0 # Always check. + self.addAsyncCleanup(cx_pool.close) + + async with cx_pool.checkout() as conn: + # Simulate a closed socket without telling the Connection it's + # closed. + conn.conn.close() + + # Swap pool's address with a bad one. + address, cx_pool.address = cx_pool.address, ("foo.com", 1234) + with self.assertRaises(AutoReconnect): + async with cx_pool.checkout(): + pass + + # Back to normal, semaphore was correctly released. + cx_pool.address = address + async with cx_pool.checkout(): + pass + + async def test_wait_queue_timeout(self): + wait_queue_timeout = 2 # Seconds + pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=wait_queue_timeout) + self.addAsyncCleanup(pool.close) + + async with pool.checkout(): + start = time.time() + with self.assertRaises(ConnectionFailure): + async with pool.checkout(): + pass + + duration = time.time() - start + self.assertTrue( + abs(wait_queue_timeout - duration) < 1, + f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", + ) + + async def test_no_wait_queue_timeout(self): + # Verify get_socket() with no wait_queue_timeout blocks forever. + pool = await self.create_pool(max_pool_size=1) + self.addAsyncCleanup(pool.close) + + # Reach max_size. + async with pool.checkout() as s1: + t = SocketGetter(self.c, pool) + await t.start() + while t.state != "get_socket": + await asyncio.sleep(0.1) + + await asyncio.sleep(1) + self.assertEqual(t.state, "get_socket") + + while t.state != "connection": + await asyncio.sleep(0.1) + + self.assertEqual(t.state, "connection") + self.assertEqual(t.sock, s1) + + async def test_checkout_more_than_max_pool_size(self): + pool = await self.create_pool(max_pool_size=2) + + socks = [] + for _ in range(2): + # Call 'pin_cursor' so we can hold the socket. + async with pool.checkout() as sock: + sock.pin_cursor() + socks.append(sock) + + tasks = [] + for _ in range(30): + t = SocketGetter(self.c, pool) + await t.start() + tasks.append(t) + await asyncio.sleep(1) + for t in tasks: + self.assertEqual(t.state, "get_socket") + + for socket_info in socks: + socket_info.close_conn(None) + + async def test_maxConnecting(self): + client = await self.async_rs_or_single_client() + await self.client.test.test.insert_one({}) + self.addAsyncCleanup(self.client.test.test.delete_many, {}) + pool = await async_get_pool(client) + docs = [] + + # Run 50 short running operations + async def find_one(): + docs.append(await client.test.test.find_one({})) + + tasks = [ConcurrentRunner(target=find_one) for _ in range(50)] + for task in tasks: + await task.start() + for task in tasks: + await task.join(10) + + self.assertEqual(len(docs), 50) + self.assertLessEqual(len(pool.conns), 50) + # TLS and auth make connection establishment more expensive than + # the query which leads to more threads hitting maxConnecting. + # The end result is fewer total connections and better latency. + if async_client_context.tls and async_client_context.auth_enabled: + self.assertLessEqual(len(pool.conns), 30) + else: + self.assertLessEqual(len(pool.conns), 50) + # MongoDB 4.4.1 with auth + ssl: + # maxConnecting = 2: 6 connections in ~0.231+ seconds + # maxConnecting = unbounded: 50 connections in ~0.642+ seconds + # + # MongoDB 4.4.1 with no-auth no-ssl Python 3.8: + # maxConnecting = 2: 15-22 connections in ~0.108+ seconds + # maxConnecting = unbounded: 30+ connections in ~0.140+ seconds + print(len(pool.conns)) + + @async_client_context.require_failCommand_appName + async def test_csot_timeout_message(self): + client = await self.async_rs_or_single_client(appName="connectionTimeoutApp") + # Mock an operation failing due to pymongo.timeout(). + mock_connection_timeout = { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "blockConnection": True, + "blockTimeMS": 1000, + "failCommands": ["find"], + "appName": "connectionTimeoutApp", + }, + } + + await client.db.t.insert_one({"x": 1}) + + async with self.fail_point(mock_connection_timeout): + with self.assertRaises(Exception) as error: + with timeout(0.5): + await client.db.t.find_one({"$where": delay(2)}) + + self.assertTrue("(configured timeouts: timeoutMS: 500.0ms" in str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_socket_timeout_message(self): + client = await self.async_rs_or_single_client( + socketTimeoutMS=500, appName="connectionTimeoutApp" + ) + # Mock an operation failing due to socketTimeoutMS. + mock_connection_timeout = { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "blockConnection": True, + "blockTimeMS": 1000, + "failCommands": ["find"], + "appName": "connectionTimeoutApp", + }, + } + + await client.db.t.insert_one({"x": 1}) + + async with self.fail_point(mock_connection_timeout): + with self.assertRaises(Exception) as error: + await client.db.t.find_one({"$where": delay(2)}) + + self.assertTrue( + "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 20000.0ms)" + in str(error.exception) + ) + + @async_client_context.require_failCommand_appName + async def test_connection_timeout_message(self): + # Mock a connection creation failing due to timeout. + mock_connection_timeout = { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "blockConnection": True, + "blockTimeMS": 1000, + "failCommands": [HelloCompat.LEGACY_CMD, "hello"], + "appName": "connectionTimeoutApp", + }, + } + + client = await self.async_rs_or_single_client( + connectTimeoutMS=500, + socketTimeoutMS=500, + appName="connectionTimeoutApp", + heartbeatFrequencyMS=1000000, + ) + await client.admin.command("ping") + pool = await async_get_pool(client) + await pool.reset_without_pause() + async with self.fail_point(mock_connection_timeout): + with self.assertRaises(Exception) as error: + await client.admin.command("ping") + + self.assertTrue( + "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)" + in str(error.exception) + ) + + +class TestPoolMaxSize(_TestPoolingBase): + async def test_max_pool_size(self): + max_pool_size = 4 + c = await self.async_rs_or_single_client(maxPoolSize=max_pool_size) + collection = c[DB].test + + # Need one document. + await collection.drop() + await collection.insert_one({}) + + # ntasks had better be much larger than max_pool_size to ensure that + # max_pool_size connections are actually required at some point in this + # test's execution. + cx_pool = await async_get_pool(c) + ntasks = 10 + tasks = [] + lock = _async_create_lock() + self.n_passed = 0 + + async def f(): + for _ in range(5): + await collection.find_one({"$where": delay(0.1)}) + assert len(cx_pool.conns) <= max_pool_size + + async with lock: + self.n_passed += 1 + + for _i in range(ntasks): + t = ConcurrentRunner(target=f) + tasks.append(t) + await t.start() + + await async_joinall(tasks) + self.assertEqual(ntasks, self.n_passed) + self.assertTrue(len(cx_pool.conns) > 1) + self.assertEqual(0, cx_pool.requests) + + async def test_max_pool_size_none(self): + c = await self.async_rs_or_single_client(maxPoolSize=None) + collection = c[DB].test + + # Need one document. + await collection.drop() + await collection.insert_one({}) + + cx_pool = await async_get_pool(c) + ntasks = 10 + tasks = [] + lock = _async_create_lock() + self.n_passed = 0 + + async def f(): + for _ in range(5): + await collection.find_one({"$where": delay(0.1)}) + + async with lock: + self.n_passed += 1 + + for _i in range(ntasks): + t = ConcurrentRunner(target=f) + tasks.append(t) + await t.start() + + await async_joinall(tasks) + self.assertEqual(ntasks, self.n_passed) + self.assertTrue(len(cx_pool.conns) > 1) + self.assertEqual(cx_pool.max_pool_size, float("inf")) + + async def test_max_pool_size_zero(self): + c = await self.async_rs_or_single_client(maxPoolSize=0) + pool = await async_get_pool(c) + self.assertEqual(pool.max_pool_size, float("inf")) + + async def test_max_pool_size_with_connection_failure(self): + # The pool acquires its semaphore before attempting to connect; ensure + # it releases the semaphore on connection failure. + test_pool = Pool( + ("somedomainthatdoesntexist.org", 27017), + PoolOptions(max_pool_size=1, connect_timeout=1, socket_timeout=1, wait_queue_timeout=1), + ) + await test_pool.ready() + + # First call to get_socket fails; if pool doesn't release its semaphore + # then the second call raises "ConnectionFailure: Timed out waiting for + # socket from pool" instead of AutoReconnect. + for _i in range(2): + with self.assertRaises(AutoReconnect) as context: + async with test_pool.checkout(): + pass + + # Testing for AutoReconnect instead of ConnectionFailure, above, + # is sufficient right *now* to catch a semaphore leak. But that + # seems error-prone, so check the message too. + self.assertNotIn("waiting for socket from pool", str(context.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_pooling.py b/test/test_pooling.py index 3b867965b..5d23b85f2 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -15,11 +15,11 @@ """Test built in connection-pooling with threads.""" from __future__ import annotations +import asyncio import gc import random import socket import sys -import threading import time from bson.codec_options import DEFAULT_CODEC_OPTIONS @@ -27,30 +27,29 @@ from bson.son import SON from pymongo import MongoClient, message, timeout from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError from pymongo.hello import HelloCompat +from pymongo.lock import _create_lock sys.path[0:0] = [""] from test import IntegrationTest, client_context, unittest +from test.helpers import ConcurrentRunner from test.utils import delay, get_pool, joinall from pymongo.socket_checker import SocketChecker from pymongo.synchronous.pool import Pool, PoolOptions - -@client_context.require_connection -def setUpModule(): - pass +_IS_SYNC = True N = 10 DB = "pymongo-pooling-tests" -def gc_collect_until_done(threads, timeout=60): +def gc_collect_until_done(tasks, timeout=60): start = time.time() - running = list(threads) + running = list(tasks) while running: - assert (time.time() - start) < timeout, "Threads timed out" + assert (time.time() - start) < timeout, "Tasks timed out" for t in running: t.join(0.1) if not t.is_alive(): @@ -58,12 +57,12 @@ def gc_collect_until_done(threads, timeout=60): gc.collect() -class MongoThread(threading.Thread): - """A thread that uses a MongoClient.""" +class MongoTask(ConcurrentRunner): + """A thread/Task that uses a MongoClient.""" def __init__(self, client): super().__init__() - self.daemon = True # Don't hang whole test if thread hangs. + self.daemon = True # Don't hang whole test if task hangs. self.client = client self.db = self.client[DB] self.passed = False @@ -76,21 +75,21 @@ class MongoThread(threading.Thread): raise NotImplementedError -class InsertOneAndFind(MongoThread): +class InsertOneAndFind(MongoTask): def run_mongo_thread(self): for _ in range(N): rand = random.randint(0, N) - _id = self.db.sf.insert_one({"x": rand}).inserted_id - assert rand == self.db.sf.find_one(_id)["x"] + _id = (self.db.sf.insert_one({"x": rand})).inserted_id + assert rand == (self.db.sf.find_one(_id))["x"] -class Unique(MongoThread): +class Unique(MongoTask): def run_mongo_thread(self): for _ in range(N): self.db.unique.insert_one({}) # no error -class NonUnique(MongoThread): +class NonUnique(MongoTask): def run_mongo_thread(self): for _ in range(N): try: @@ -101,7 +100,7 @@ class NonUnique(MongoThread): raise AssertionError("Should have raised DuplicateKeyError") -class SocketGetter(MongoThread): +class SocketGetter(MongoTask): """Utility for TestPooling. Checks out a socket and holds it forever. Used in @@ -130,25 +129,26 @@ class SocketGetter(MongoThread): def run_cases(client, cases): - threads = [] + tasks = [] n_runs = 5 for case in cases: for _i in range(n_runs): t = case(client) t.start() - threads.append(t) + tasks.append(t) - for t in threads: + for t in tasks: t.join() - for t in threads: + for t in tasks: assert t.passed, "%s.run() threw an exception" % repr(t) class _TestPoolingBase(IntegrationTest): """Base class for all connection-pool tests.""" + @client_context.require_connection def setUp(self): super().setUp() self.c = self.rs_or_single_client() @@ -158,11 +158,9 @@ class _TestPoolingBase(IntegrationTest): db.unique.insert_one({"_id": "jesse"}) db.test.insert_many([{} for _ in range(10)]) - def tearDown(self): - self.c.close() - super().tearDown() - - def create_pool(self, pair=(client_context.host, client_context.port), *args, **kwargs): + def create_pool(self, pair=None, *args, **kwargs): + if pair is None: + pair = (client_context.host, client_context.port) # Start the pool with the correct ssl options. pool_options = client_context.client._topology_settings.pool_options kwargs["ssl_context"] = pool_options._ssl_context @@ -365,13 +363,13 @@ class TestPooling(_TestPoolingBase): sock.pin_cursor() socks.append(sock) - threads = [] + tasks = [] for _ in range(30): t = SocketGetter(self.c, pool) t.start() - threads.append(t) + tasks.append(t) time.sleep(1) - for t in threads: + for t in tasks: self.assertEqual(t.state, "get_socket") for socket_info in socks: @@ -379,7 +377,6 @@ class TestPooling(_TestPoolingBase): def test_maxConnecting(self): client = self.rs_or_single_client() - self.addCleanup(client.close) self.client.test.test.insert_one({}) self.addCleanup(self.client.test.test.delete_many, {}) pool = get_pool(client) @@ -389,11 +386,11 @@ class TestPooling(_TestPoolingBase): def find_one(): docs.append(client.test.test.find_one({})) - threads = [threading.Thread(target=find_one) for _ in range(50)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(10) + tasks = [ConcurrentRunner(target=find_one) for _ in range(50)] + for task in tasks: + task.start() + for task in tasks: + task.join(10) self.assertEqual(len(docs), 50) self.assertLessEqual(len(pool.conns), 50) @@ -416,7 +413,6 @@ class TestPooling(_TestPoolingBase): @client_context.require_failCommand_appName def test_csot_timeout_message(self): client = self.rs_or_single_client(appName="connectionTimeoutApp") - self.addCleanup(client.close) # Mock an operation failing due to pymongo.timeout(). mock_connection_timeout = { "configureFailPoint": "failCommand", @@ -441,7 +437,6 @@ class TestPooling(_TestPoolingBase): @client_context.require_failCommand_appName def test_socket_timeout_message(self): client = self.rs_or_single_client(socketTimeoutMS=500, appName="connectionTimeoutApp") - self.addCleanup(client.close) # Mock an operation failing due to socketTimeoutMS. mock_connection_timeout = { "configureFailPoint": "failCommand", @@ -485,7 +480,6 @@ class TestPooling(_TestPoolingBase): appName="connectionTimeoutApp", heartbeatFrequencyMS=1000000, ) - self.addCleanup(client.close) client.admin.command("ping") pool = get_pool(client) pool.reset_without_pause() @@ -503,20 +497,19 @@ class TestPoolMaxSize(_TestPoolingBase): def test_max_pool_size(self): max_pool_size = 4 c = self.rs_or_single_client(maxPoolSize=max_pool_size) - self.addCleanup(c.close) collection = c[DB].test # Need one document. collection.drop() collection.insert_one({}) - # nthreads had better be much larger than max_pool_size to ensure that + # ntasks had better be much larger than max_pool_size to ensure that # max_pool_size connections are actually required at some point in this # test's execution. cx_pool = get_pool(c) - nthreads = 10 - threads = [] - lock = threading.Lock() + ntasks = 10 + tasks = [] + lock = _create_lock() self.n_passed = 0 def f(): @@ -527,19 +520,18 @@ class TestPoolMaxSize(_TestPoolingBase): with lock: self.n_passed += 1 - for _i in range(nthreads): - t = threading.Thread(target=f) - threads.append(t) + for _i in range(ntasks): + t = ConcurrentRunner(target=f) + tasks.append(t) t.start() - joinall(threads) - self.assertEqual(nthreads, self.n_passed) + joinall(tasks) + self.assertEqual(ntasks, self.n_passed) self.assertTrue(len(cx_pool.conns) > 1) self.assertEqual(0, cx_pool.requests) def test_max_pool_size_none(self): c = self.rs_or_single_client(maxPoolSize=None) - self.addCleanup(c.close) collection = c[DB].test # Need one document. @@ -547,9 +539,9 @@ class TestPoolMaxSize(_TestPoolingBase): collection.insert_one({}) cx_pool = get_pool(c) - nthreads = 10 - threads = [] - lock = threading.Lock() + ntasks = 10 + tasks = [] + lock = _create_lock() self.n_passed = 0 def f(): @@ -559,19 +551,18 @@ class TestPoolMaxSize(_TestPoolingBase): with lock: self.n_passed += 1 - for _i in range(nthreads): - t = threading.Thread(target=f) - threads.append(t) + for _i in range(ntasks): + t = ConcurrentRunner(target=f) + tasks.append(t) t.start() - joinall(threads) - self.assertEqual(nthreads, self.n_passed) + joinall(tasks) + self.assertEqual(ntasks, self.n_passed) self.assertTrue(len(cx_pool.conns) > 1) self.assertEqual(cx_pool.max_pool_size, float("inf")) def test_max_pool_size_zero(self): c = self.rs_or_single_client(maxPoolSize=0) - self.addCleanup(c.close) pool = get_pool(c) self.assertEqual(pool.max_pool_size, float("inf")) diff --git a/tools/synchro.py b/tools/synchro.py index 69a2f07ba..519ebb102 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -228,6 +228,7 @@ converted_tests = [ "test_monitoring.py", "test_mongos_load_balancing.py", "test_on_demand_csfle.py", + "test_pooling.py", "test_raw_bson.py", "test_read_concern.py", "test_read_preferences.py",