Merge branch 'master' of github.com:mongodb/mongo-python-driver

This commit is contained in:
Steven Silvester 2025-02-14 06:18:04 -06:00
commit 230dd08965
No known key found for this signature in database
GPG Key ID: B1BF5EC3A8B32F91
43 changed files with 1291 additions and 757 deletions

View File

@ -42,7 +42,7 @@ functions:
# Make an evergreen expansion file with dynamic values
- command: subprocess.exec
params:
include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"]
include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "TEST_ENCRYPTION", "TEST_ENCRYPTION_PYOPENSSL", "TEST_CRYPT_SHARED", "TEST_PYOPENSSL", "SETDEFAULTENCODING", "TEST_LOADBALANCER", "TEST_SEVERLESS", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"]
binary: bash
working_dir: "src"
args:
@ -274,39 +274,22 @@ functions:
"run tests":
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["TEST_DATA_LAKE", "PYTHON_BINARY", "AUTH", "SSL", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"]
include_expansions_in_env: ["TEST_DATA_LAKE", "PYTHON_BINARY", "AUTH", "SSL",
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_SUITES",
"TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "TEST_ENCRYPTION", "TEST_ENCRYPTION_PYOPENSSL",
"TEST_CRYPT_SHARED", "TEST_PYOPENSSL", "TEST_LOADBALANCER", "TEST_SEVERLESS", "MONGODB_URI"]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
params:
working_dir: "src"
binary: bash
background: true
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/setup-encryption.sh
- command: subprocess.exec
type: test
params:
working_dir: "src"
binary: bash
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"]
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-tests.sh
"run direct tests":
- command: subprocess.exec
type: test
params:
working_dir: "src"
binary: bash
include_expansions_in_env: ["PYTHON_BINARY"]
args: [ .evergreen/scripts/run-direct-tests.sh ]
args: [.evergreen/just.sh, test-eg]
"run enterprise auth tests":
- command: subprocess.exec
@ -340,13 +323,6 @@ functions:
- ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh
"run aws auth test with regular aws credentials":
- command: subprocess.exec
params:
include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -359,13 +335,6 @@ functions:
- regular
"run aws auth test with assume role credentials":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -378,13 +347,6 @@ functions:
- assume-role
"run aws auth test with aws EC2 credentials":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -397,13 +359,6 @@ functions:
- ec2
"run aws auth test with aws web identity credentials":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- # Test with and without AWS_ROLE_SESSION_NAME set.
- command: subprocess.exec
type: test
@ -429,13 +384,6 @@ functions:
- web-identity
"run aws auth test with aws credentials as environment variables":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -448,13 +396,6 @@ functions:
- env-creds
"run aws auth test with aws credentials and session token as environment variables":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -467,13 +408,6 @@ functions:
- session-creds
"run oidc auth test with test credentials":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -561,13 +495,6 @@ functions:
file: atlas-expansion.yml
"run-ocsp-test":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -587,25 +514,6 @@ functions:
args:
- ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh
"run load-balancer":
- command: subprocess.exec
params:
binary: bash
include_expansions_in_env: ["MONGODB_URI"]
args:
- src/.evergreen/scripts/run-with-env.sh
- src/.evergreen/scripts/run-load-balancer.sh
- command: expansions.update
params:
file: lb-expansion.yml
"stop load-balancer":
- command: subprocess.exec
params:
binary: bash
args:
- src/.evergreen/scripts/stop-load-balancer.sh
"teardown atlas":
- command: subprocess.exec
params:
@ -882,6 +790,7 @@ tasks:
- func: "run tests"
vars:
TEST_INDEX_MANAGEMENT: "1"
AUTH: "auth"
- name: "mod-wsgi-standalone"
tags: ["mod_wsgi"]
@ -935,7 +844,7 @@ tasks:
vars:
VERSION: "8.0"
TOPOLOGY: "replica_set"
- func: "run direct tests"
- func: "run tests"
- name: "atlas-connect"
tags: ["atlas-connect"]
@ -1503,7 +1412,7 @@ tasks:
- name: "testgcpkms-task"
commands:
- command: subprocess.exec
type: setup
type: test
params:
working_dir: "src"
binary: bash
@ -1531,6 +1440,7 @@ tasks:
- name: testazurekms-task
commands:
- command: subprocess.exec
type: test
params:
binary: bash
working_dir: src

View File

@ -8,12 +8,11 @@ tasks:
AUTH: auth
SSL: ssl
LOAD_BALANCER: "true"
- func: run load-balancer
- func: run tests
vars:
AUTH: auth
SSL: ssl
test_loadbalancer: "true"
TEST_LOADBALANCER: "true"
tags: [load-balancer, auth, ssl]
- name: test-load-balancer-noauth-ssl
commands:
@ -23,12 +22,11 @@ tasks:
AUTH: noauth
SSL: ssl
LOAD_BALANCER: "true"
- func: run load-balancer
- func: run tests
vars:
AUTH: noauth
SSL: ssl
test_loadbalancer: "true"
TEST_LOADBALANCER: "true"
tags: [load-balancer, noauth, ssl]
- name: test-load-balancer-noauth-nossl
commands:
@ -38,12 +36,11 @@ tasks:
AUTH: noauth
SSL: nossl
LOAD_BALANCER: "true"
- func: run load-balancer
- func: run tests
vars:
AUTH: noauth
SSL: nossl
test_loadbalancer: "true"
TEST_LOADBALANCER: "true"
tags: [load-balancer, noauth, nossl]
# Server tests

View File

@ -318,7 +318,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: /opt/python/3.9/bin/python3
tags: [encryption_tag]
- name: encryption-rhel8-python3.13
@ -331,7 +331,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: /opt/python/3.13/bin/python3
tags: [encryption_tag]
- name: encryption-rhel8-pypy3.10
@ -344,7 +344,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
tags: [encryption_tag]
- name: encryption-crypt_shared-rhel8-python3.9
@ -357,8 +357,8 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: /opt/python/3.9/bin/python3
tags: [encryption_tag]
- name: encryption-crypt_shared-rhel8-python3.13
@ -371,8 +371,8 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: /opt/python/3.13/bin/python3
tags: [encryption_tag]
- name: encryption-crypt_shared-rhel8-pypy3.10
@ -385,8 +385,8 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
tags: [encryption_tag]
- name: encryption-pyopenssl-rhel8-python3.9
@ -399,8 +399,8 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
test_encryption_pyopenssl: "true"
TEST_ENCRYPTION: "true"
TEST_ENCRYPTION_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/3.9/bin/python3
tags: [encryption_tag]
- name: encryption-pyopenssl-rhel8-python3.13
@ -413,8 +413,8 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
test_encryption_pyopenssl: "true"
TEST_ENCRYPTION: "true"
TEST_ENCRYPTION_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/3.13/bin/python3
tags: [encryption_tag]
- name: encryption-pyopenssl-rhel8-pypy3.10
@ -427,8 +427,8 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_encryption: "true"
test_encryption_pyopenssl: "true"
TEST_ENCRYPTION: "true"
TEST_ENCRYPTION_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
tags: [encryption_tag]
- name: encryption-rhel8-python3.10
@ -438,7 +438,7 @@ buildvariants:
run_on:
- rhel87-small
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: /opt/python/3.10/bin/python3
- name: encryption-crypt_shared-rhel8-python3.11
tasks:
@ -447,8 +447,8 @@ buildvariants:
run_on:
- rhel87-small
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: /opt/python/3.11/bin/python3
- name: encryption-pyopenssl-rhel8-python3.12
tasks:
@ -457,8 +457,8 @@ buildvariants:
run_on:
- rhel87-small
expansions:
test_encryption: "true"
test_encryption_pyopenssl: "true"
TEST_ENCRYPTION: "true"
TEST_ENCRYPTION_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/3.12/bin/python3
- name: encryption-macos-python3.9
tasks:
@ -468,7 +468,7 @@ buildvariants:
- macos-14
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
tags: [encryption_tag]
- name: encryption-macos-python3.13
@ -479,7 +479,7 @@ buildvariants:
- macos-14
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
tags: [encryption_tag]
- name: encryption-crypt_shared-macos-python3.9
@ -490,8 +490,8 @@ buildvariants:
- macos-14
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
tags: [encryption_tag]
- name: encryption-crypt_shared-macos-python3.13
@ -502,8 +502,8 @@ buildvariants:
- macos-14
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
tags: [encryption_tag]
- name: encryption-win64-python3.9
@ -514,7 +514,7 @@ buildvariants:
- windows-64-vsMulti-small
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: C:/python/Python39/python.exe
tags: [encryption_tag]
- name: encryption-win64-python3.13
@ -525,7 +525,7 @@ buildvariants:
- windows-64-vsMulti-small
batchtime: 10080
expansions:
test_encryption: "true"
TEST_ENCRYPTION: "true"
PYTHON_BINARY: C:/python/Python313/python.exe
tags: [encryption_tag]
- name: encryption-crypt_shared-win64-python3.9
@ -536,8 +536,8 @@ buildvariants:
- windows-64-vsMulti-small
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: C:/python/Python39/python.exe
tags: [encryption_tag]
- name: encryption-crypt_shared-win64-python3.13
@ -548,8 +548,8 @@ buildvariants:
- windows-64-vsMulti-small
batchtime: 10080
expansions:
test_encryption: "true"
test_crypt_shared: "true"
TEST_ENCRYPTION: "true"
TEST_CRYPT_SHARED: "true"
PYTHON_BINARY: C:/python/Python313/python.exe
tags: [encryption_tag]
@ -1010,7 +1010,7 @@ buildvariants:
- macos-14
batchtime: 10080
expansions:
test_pyopenssl: "true"
TEST_PYOPENSSL: "true"
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
- name: pyopenssl-rhel8-python3.10
tasks:
@ -1021,7 +1021,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_pyopenssl: "true"
TEST_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/3.10/bin/python3
- name: pyopenssl-rhel8-python3.11
tasks:
@ -1032,7 +1032,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_pyopenssl: "true"
TEST_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/3.11/bin/python3
- name: pyopenssl-rhel8-python3.12
tasks:
@ -1043,7 +1043,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_pyopenssl: "true"
TEST_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/3.12/bin/python3
- name: pyopenssl-win64-python3.13
tasks:
@ -1054,7 +1054,7 @@ buildvariants:
- windows-64-vsMulti-small
batchtime: 10080
expansions:
test_pyopenssl: "true"
TEST_PYOPENSSL: "true"
PYTHON_BINARY: C:/python/Python313/python.exe
- name: pyopenssl-rhel8-pypy3.10
tasks:
@ -1065,7 +1065,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_pyopenssl: "true"
TEST_PYOPENSSL: "true"
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
# Search index tests
@ -1301,7 +1301,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_serverless: "true"
TEST_SERVERLESS: "true"
AUTH: auth
SSL: ssl
PYTHON_BINARY: /opt/python/3.9/bin/python3
@ -1313,7 +1313,7 @@ buildvariants:
- rhel87-small
batchtime: 10080
expansions:
test_serverless: "true"
TEST_SERVERLESS: "true"
AUTH: auth
SSL: ssl
PYTHON_BINARY: /opt/python/3.13/bin/python3

View File

@ -3,10 +3,9 @@ set -o errexit # Exit the script with error if any of the commands fail
HERE=$(dirname ${BASH_SOURCE:-$0})
. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash $HERE/setup-encryption.sh
SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh
PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
KEY_NAME="${AZUREKMS_KEYNAME}" \
KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
$HERE/just.sh test-eg
bash $HERE/teardown-encryption.sh
bash $HERE/scripts/teardown-tests.sh

View File

@ -6,24 +6,23 @@ echo "Copying files ... begin"
export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash $HERE/setup-encryption.sh
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
# Set up the remote files to test.
git add .
git commit -m "add files" || true
git archive -o /tmp/mongo-python-driver.tar HEAD
tar -rf /tmp/mongo-python-driver.tar libmongocrypt
gzip -f /tmp/mongo-python-driver.tar
git archive -o /tmp/mongo-python-driver.tgz HEAD
# shellcheck disable=SC2088
AZUREKMS_SRC="/tmp/mongo-python-driver.tar.gz" AZUREKMS_DST="~/" \
AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
AZUREKMS_CMD="tar xf mongo-python-driver.tar.gz" \
AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
echo "Untarring file ... end"
echo "Running test ... begin"
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash ./.evergreen/just.sh test-eg" \
AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
echo "Running test ... end"
bash $HERE/teardown-encryption.sh
bash $HERE/scripts/teardown-tests.sh

View File

@ -8,20 +8,18 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash $HERE/setup-encryption.sh
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
# Set up the remote files to test.
git add .
git commit -m "add files" || true
git archive -o /tmp/mongo-python-driver.tar HEAD
tar -rf /tmp/mongo-python-driver.tar libmongocrypt
gzip -f /tmp/mongo-python-driver.tar
GCPKMS_SRC=/tmp/mongo-python-driver.tar.gz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
git archive -o /tmp/mongo-python-driver.tgz HEAD
GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
GCPKMS_CMD="tar xf mongo-python-driver.tar.gz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Untarring file ... end"
echo "Running test ... begin"
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Running test ... end"
bash $HERE/teardown-encryption.sh
bash $HERE/scripts/teardown-tests.sh

View File

@ -31,4 +31,5 @@ export AUTH="auth"
export SET_XTRACE_ON=1
cd src
rm -rf .venv
bash ./.evergreen/just.sh setup-test
bash .evergreen/just.sh test-eg

View File

@ -29,7 +29,5 @@ else
exit 1
fi
export TEST_AUTH_OIDC=1
export COVERAGE=1
export AUTH="auth"
TEST_AUTH_OIDC=1 COVERAGE=1 AUTH="auth" bash ./.evergreen/just.sh setup-test
bash ./.evergreen/just.sh test-eg "${@:1}"

View File

@ -16,4 +16,5 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
export PERF_TEST=1
bash ./.evergreen/just.sh setup-test
bash ./.evergreen/just.sh test-eg

View File

@ -1,283 +1,81 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
set -o xtrace
set -eu
# Note: It is assumed that you have already set up a virtual environment before running this file.
# Supported/used environment variables:
# AUTH Set to enable authentication. Defaults to "noauth"
# SSL Set to enable SSL. Defaults to "nossl"
# GREEN_FRAMEWORK The green framework to test with, if any.
# COVERAGE If non-empty, run the test suite with coverage.
# COMPRESSORS If non-empty, install appropriate compressor.
# LIBMONGOCRYPT_URL The URL to download libmongocrypt.
# TEST_DATA_LAKE If non-empty, run data lake tests.
# TEST_ENCRYPTION If non-empty, run encryption tests.
# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib.
# TEST_SERVERLESS If non-empy, test on serverless.
# TEST_LOADBALANCER If non-empy, test load balancing.
# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure
# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP
# TEST_PYOPENSSL If non-empy, test with PyOpenSSL
# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth
# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism
# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism
# TEST_PERF If non-empty, run performance tests
# TEST_OCSP If non-empty, run OCSP tests
# TEST_ATLAS If non-empty, test Atlas connections
# TEST_INDEX_MANAGEMENT If non-empty, run index management tests
# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL
AUTH=${AUTH:-noauth}
SSL=${SSL:-nossl}
TEST_SUITES=${TEST_SUITES:-}
TEST_ARGS="${*:1}"
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
export PIP_QUIET=1 # Quiet by default
export PIP_PREFER_BINARY=1 # Prefer binary dists by default
export UV_FROZEN=1 # Do not modify lock files
set +x
PYTHON_IMPL=$(uv run --frozen python -c "import platform; print(platform.python_implementation())")
# Try to source local Drivers Secrets
if [ -f ./secrets-export.sh ]; then
echo "Sourcing secrets"
source ./secrets-export.sh
# Try to source the env file.
if [ -f $SCRIPT_DIR/scripts/env.sh ]; then
echo "Sourcing env inputs"
. $SCRIPT_DIR/scripts/env.sh
else
echo "Not sourcing secrets"
echo "Not sourcing env inputs"
fi
# Start compiling the args we'll pass to uv.
# Run in an isolated environment so as not to pollute the base venv.
UV_ARGS=("--isolated --frozen --extra test")
# Ensure there are test inputs.
if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then
echo "Sourcing test inputs"
. $SCRIPT_DIR/scripts/test-env.sh
else
echo "Missing test inputs, please run 'just setup-test'"
fi
# Source the local secrets export file if available.
if [ -f "$ROOT_DIR/secrets-export.sh" ]; then
. "$ROOT_DIR/secrets-export.sh"
fi
PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())")
# Ensure C extensions if applicable.
if [ -z "${NO_EXT:-}" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
uv run --frozen tools/fail_if_no_c.py
fi
if [ "$AUTH" != "noauth" ]; then
if [ -n "$TEST_DATA_LAKE" ]; then
export DB_USER="mhuser"
export DB_PASSWORD="pencil"
elif [ -n "$TEST_SERVERLESS" ]; then
source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh
export DB_USER=$SERVERLESS_ATLAS_USER
export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
export MONGODB_URI="$SERVERLESS_URI"
echo "MONGODB_URI=$MONGODB_URI"
export SINGLE_MONGOS_LB_URI=$MONGODB_URI
export MULTI_MONGOS_LB_URI=$MONGODB_URI
elif [ -n "$TEST_AUTH_OIDC" ]; then
export DB_USER=$OIDC_ADMIN_USER
export DB_PASSWORD=$OIDC_ADMIN_PWD
export DB_IP="$MONGODB_URI"
else
export DB_USER="bob"
export DB_PASSWORD="pwd123"
fi
echo "Added auth, DB_USER: $DB_USER"
fi
if [ -n "$TEST_ENTERPRISE_AUTH" ]; then
UV_ARGS+=("--extra gssapi")
if [ "Windows_NT" = "$OS" ]; then
echo "Setting GSSAPI_PASS"
export GSSAPI_PASS=${SASL_PASS}
export GSSAPI_CANONICALIZE="true"
else
# BUILD-3830
touch krb5.conf.empty
export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty
echo "Writing keytab"
echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab
echo "Running kinit"
kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL}
fi
echo "Setting GSSAPI variables"
export GSSAPI_HOST=${SASL_HOST}
export GSSAPI_PORT=${SASL_PORT}
export GSSAPI_PRINCIPAL=${PRINCIPAL}
export TEST_SUITES="auth"
fi
if [ -n "$TEST_LOADBALANCER" ]; then
export LOAD_BALANCER=1
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}"
export TEST_SUITES="load_balancer"
fi
if [ "$SSL" != "nossl" ]; then
export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
if [ -n "$TEST_LOADBALANCER" ]; then
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}&tls=true"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}&tls=true"
fi
fi
if [ "$COMPRESSORS" = "snappy" ]; then
UV_ARGS+=("--extra snappy")
elif [ "$COMPRESSORS" = "zstd" ]; then
UV_ARGS+=("--extra zstandard")
fi
# PyOpenSSL test setup.
if [ -n "$TEST_PYOPENSSL" ]; then
UV_ARGS+=("--extra ocsp")
fi
if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
# Check for libmongocrypt download.
if [ ! -d "libmongocrypt" ]; then
echo "Run encryption setup first!"
exit 1
fi
UV_ARGS+=("--extra encryption")
# TODO: Test with 'pip install pymongocrypt'
UV_ARGS+=("--group pymongocrypt_source")
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
BASE=$(pwd)/libmongocrypt/nocrypto
if [ -f "${BASE}/lib/libmongocrypt.so" ]; then
PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so
elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then
PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib
elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then
PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll
# libmongocrypt's windows dll is not marked executable.
chmod +x $PYMONGOCRYPT_LIB
PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB)
elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then
PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so
else
echo "Cannot find libmongocrypt shared object file"
exit 1
fi
export PYMONGOCRYPT_LIB
if [ -n "${PYMONGOCRYPT_LIB:-}" ]; then
# Ensure pymongocrypt is working properly.
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
uv run ${UV_ARGS} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
uv run ${UV_ARGS} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
# PATH is updated by configure-env.sh for access to mongocryptd.
fi
if [ -n "$TEST_ENCRYPTION" ]; then
if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then
UV_ARGS+=("--extra ocsp")
fi
if [ -n "$TEST_CRYPT_SHARED" ]; then
CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH`
echo "using crypt_shared_dir $CRYPT_SHARED_DIR"
export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:$DYLD_FALLBACK_LIBRARY_PATH
export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:$LD_LIBRARY_PATH
export PATH=$CRYPT_SHARED_DIR:$PATH
fi
# Only run the encryption tests.
TEST_SUITES="encryption"
fi
if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
if [[ -z "$SUCCESS" ]]; then
echo "Must define SUCCESS"
exit 1
fi
if echo "$MONGODB_URI" | grep -q "@"; then
echo "MONGODB_URI unexpectedly contains user credentials in FLE test!";
exit 1
fi
TEST_SUITES="csfle"
fi
if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh
export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}"
set +x
export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
set -x
TEST_SUITES="index_management"
fi
if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then
TEST_SUITES="data_lake"
fi
if [ -n "$TEST_ATLAS" ]; then
TEST_SUITES="atlas"
fi
if [ -n "$TEST_OCSP" ]; then
UV_ARGS+=("--extra ocsp")
TEST_SUITES="ocsp"
fi
if [ -n "$TEST_AUTH_AWS" ]; then
UV_ARGS+=("--extra aws")
TEST_SUITES="auth_aws"
fi
if [ -n "$TEST_AUTH_OIDC" ]; then
UV_ARGS+=("--extra aws")
TEST_SUITES="auth_oidc"
fi
if [ -n "$PERF_TEST" ]; then
UV_ARGS+=("--group perf")
start_time=$(date +%s)
TEST_SUITES="perf"
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
# affects the benchmark results.
TEST_ARGS="test/performance/perf_test.py $TEST_ARGS"
fi
echo "Running $AUTH tests over $SSL with python $(uv python find)"
uv run --frozen python -c 'import sys; print(sys.version)'
# Run the tests, and store the results in Evergreen compatible XUnit XML
# files in the xunit-results/ directory.
# Run the tests with coverage if requested and coverage is installed.
# Only cover CPython. PyPy reports suspiciously low coverage.
if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
# Keep in sync with combine-coverage.sh.
# coverage >=5 is needed for relative_files=true.
UV_ARGS+=("--group coverage")
TEST_ARGS="$TEST_ARGS --cov"
fi
if [ -n "$GREEN_FRAMEWORK" ]; then
UV_ARGS+=("--group $GREEN_FRAMEWORK")
fi
PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())")
echo "Running ${AUTH:-noauth} tests over ${SSL:-nossl} with python $(uv python find)"
uv run python -c 'import sys; print(sys.version)'
# Show the installed packages
# shellcheck disable=SC2048
PIP_QUIET=0 uv run ${UV_ARGS[*]} --with pip pip list
PIP_QUIET=0 uv run ${UV_ARGS} --with pip pip list
if [ -z "$GREEN_FRAMEWORK" ]; then
# Use --capture=tee-sys so pytest prints test output inline:
# https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
PYTEST_ARGS="-v --capture=tee-sys --durations=5 $TEST_ARGS"
if [ -n "$TEST_SUITES" ]; then
PYTEST_ARGS="-m $TEST_SUITES $PYTEST_ARGS"
fi
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} pytest $PYTEST_ARGS
else
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
# Record the start time for a perf test.
if [ -n "${PERF_TEST:-}" ]; then
start_time=$(date +%s)
fi
# Run the tests, and store the results in Evergreen compatible XUnit XML
# files in the xunit-results/ directory.
TEST_ARGS=${TEST_ARGS}
if [ "$#" -ne 0 ]; then
TEST_ARGS="$*"
fi
echo "Running tests with $TEST_ARGS and uv args $UV_ARGS..."
if [ -z "${GREEN_FRAMEWORK:-}" ]; then
# shellcheck disable=SC2048
uv run ${UV_ARGS} pytest $TEST_ARGS
else
# shellcheck disable=SC2048
uv run ${UV_ARGS} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
fi
echo "Running tests with $TEST_ARGS... done."
# Handle perf test post actions.
if [ -n "$PERF_TEST" ]; then
if [ -n "${PERF_TEST:-}" ]; then
end_time=$(date +%s)
elapsed_secs=$((end_time-start_time))
@ -289,6 +87,6 @@ if [ -n "$PERF_TEST" ]; then
fi
# Handle coverage post actions.
if [ -n "$COVERAGE" ]; then
if [ -n "${COVERAGE:-}" ]; then
rm -rf .pytest_cache
fi

View File

@ -1,8 +0,0 @@
#!/bin/bash
set -o xtrace
mkdir out_dir
# shellcheck disable=SC2156
find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
tar zcvf mongodb-logs.tar.gz -C out_dir/ .
rm -rf out_dir

View File

@ -16,6 +16,18 @@ DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
if [ "${CI:-}" == "true" ]; then
PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
# We want to use a path that's already on PATH on spawn hosts.
else
PYMONGO_BIN_DIR=$HOME/cli_bin
fi
PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH"
# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
@ -24,6 +36,9 @@ if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
CARGO_HOME=$(cygpath -m $CARGO_HOME)
UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES")
MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES")
PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR")
fi
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
@ -36,7 +51,6 @@ fi
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
cat <<EOT > "$SCRIPT_DIR"/env.sh
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
@ -63,16 +77,28 @@ export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
export CARGO_HOME="$CARGO_HOME"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export UV_TOOL_DIR="$UV_TOOL_DIR"
export UV_CACHE_DIR="$UV_CACHE_DIR"
export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH"
export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR"
export PATH="$PATH_EXT"
# shellcheck disable=SC2154
export PROJECT="${project:-mongo-python-driver}"
export PIP_QUIET=1
EOT
# Write the .env file for drivers-tools.
rm -rf $DRIVERS_TOOLS
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
cat <<EOT > ${DRIVERS_TOOLS}/.env
SKIP_LEGACY_SHELL=1
DRIVERS_TOOLS="$DRIVERS_TOOLS"
MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
MONGODB_BINARIES="$MONGODB_BINARIES"
TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
EOT
# Skip CSOT tests on non-linux platforms.
if [ "$(uname -s)" != "Linux" ]; then
echo "export SKIP_CSOT_TESTS=1" >> $SCRIPT_DIR/env.sh

View File

@ -1,8 +0,0 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
# shellcheck disable=SC2044
for filename in $(find $DRIVERS_TOOLS -name \*.json); do
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
done

View File

@ -344,11 +344,11 @@ def create_encryption_variants() -> list[BuildVariant]:
batchtime = BATCHTIME_WEEK
def get_encryption_expansions(encryption):
expansions = dict(test_encryption="true")
expansions = dict(TEST_ENCRYPTION="true")
if "crypt_shared" in encryption:
expansions["test_crypt_shared"] = "true"
expansions["TEST_CRYPT_SHARED"] = "true"
if "PyOpenSSL" in encryption:
expansions["test_encryption_pyopenssl"] = "true"
expansions["TEST_ENCRYPTION_PYOPENSSL"] = "true"
return expansions
host = DEFAULT_HOST
@ -487,7 +487,7 @@ def create_enterprise_auth_variants():
def create_pyopenssl_variants():
base_name = "PyOpenSSL"
batchtime = BATCHTIME_WEEK
expansions = dict(test_pyopenssl="true")
expansions = dict(TEST_PYOPENSSL="true")
variants = []
for python in ALL_PYTHONS:
@ -645,7 +645,7 @@ def create_disable_test_commands_variants():
def create_serverless_variants():
host = DEFAULT_HOST
batchtime = BATCHTIME_WEEK
expansions = dict(test_serverless="true", AUTH="auth", SSL="ssl")
expansions = dict(TEST_SERVERLESS="true", AUTH="auth", SSL="ssl")
tasks = ["serverless_task_group"]
base_name = "Serverless"
return [
@ -834,12 +834,9 @@ def create_load_balancer_tasks():
tags = ["load-balancer", auth, ssl]
bootstrap_vars = dict(TOPOLOGY="sharded_cluster", AUTH=auth, SSL=ssl, LOAD_BALANCER="true")
bootstrap_func = FunctionCall(func="bootstrap mongo-orchestration", vars=bootstrap_vars)
balancer_func = FunctionCall(func="run load-balancer")
test_vars = dict(AUTH=auth, SSL=ssl, test_loadbalancer="true")
test_vars = dict(AUTH=auth, SSL=ssl, TEST_LOADBALANCER="true")
test_func = FunctionCall(func="run tests", vars=test_vars)
tasks.append(
EvgTask(name=name, tags=tags, commands=[bootstrap_func, balancer_func, test_func])
)
tasks.append(EvgTask(name=name, tags=tags, commands=[bootstrap_func, test_func]))
return tasks

View File

@ -1,5 +0,0 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json

View File

@ -2,23 +2,34 @@
set -eu
# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
if [ "${CI:-}" == "true" ]; then
_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
else
_BIN_DIR=$HOME/.local/bin
HERE=$(dirname ${BASH_SOURCE:-$0})
pushd "$(dirname "$(dirname $HERE)")" > /dev/null
# Source the env files to pick up common variables.
if [ -f $HERE/env.sh ]; then
. $HERE/env.sh
fi
_BIN_DIR=${PYMONGO_BIN_DIR:-$HOME/.local/bin}
export PATH="$PATH:${_BIN_DIR}"
# Helper function to pip install a dependency using a temporary python env.
function _pip_install() {
_HERE=$(dirname ${BASH_SOURCE:-$0})
. $_HERE/../utils.sh
_VENV_PATH=$(mktemp -d)
if [ "Windows_NT" = "${OS:-}" ]; then
_VENV_PATH=$(cygpath -m $_VENV_PATH)
fi
echo "Installing $2 using pip..."
createvirtualenv "$(find_python3)" $_VENV_PATH
python -m pip install $1
ln -s "$(which $2)" $_BIN_DIR/$2
if [ "Windows_NT" = "${OS:-}" ]; then
ln -s "$(which $2)" $_BIN_DIR/$2.exe
else
ln -s "$(which $2)" $_BIN_DIR/$2
fi
echo "Installed to ${_BIN_DIR}"
echo "Installing $2 using pip... done."
}
@ -35,9 +46,6 @@ if ! command -v just 2>/dev/null; then
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $_TARGET --to "$_BIN_DIR" || {
_pip_install rust-just just
}
if ! command -v just 2>/dev/null; then
export PATH="$PATH:$_BIN_DIR"
fi
echo "Installing just... done."
fi
@ -48,8 +56,10 @@ if ! command -v uv 2>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {
_pip_install uv uv
}
if ! command -v uv 2>/dev/null; then
export PATH="$PATH:$_BIN_DIR"
if [ "Windows_NT" = "${OS:-}" ]; then
chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
fi
echo "Installing uv... done."
fi
popd > /dev/null

View File

@ -1,8 +0,0 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
# shellcheck disable=SC2044
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
chmod +x "$i"
done

View File

@ -5,10 +5,6 @@ HERE=$(dirname ${BASH_SOURCE:-$0})
pushd $HERE
. env.sh
rm -rf $DRIVERS_TOOLS
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config
popd
# Copy PyMongo's test certificates over driver-evergreen-tools'

View File

@ -4,4 +4,5 @@
set +x
set -o errexit
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg

View File

@ -5,4 +5,5 @@ set -eu
set +x
# Use the default python to bootstrap secrets.
bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth
TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg

View File

@ -1,7 +1,8 @@
#!/bin/bash
. .evergreen/scripts/env.sh
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
. $HERE/env.sh
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg
SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh
bash ./.evergreen/just.sh test-eg

View File

@ -1,3 +0,0 @@
#!/bin/bash
MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start

View File

@ -24,10 +24,5 @@ echo "Running MONGODB-AWS authentication tests for $1"
# Handle credentials and environment setup.
. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
# show test output
set -x
export TEST_AUTH_AWS=1
export AUTH="auth"
export SET_XTRACE_ON=1
TEST_AUTH_AWS=1 AUTH="auth" bash ./.evergreen/just.sh setup-test
bash ./.evergreen/just.sh test-eg

View File

@ -4,5 +4,6 @@ TEST_OCSP=1 \
PYTHON_BINARY="${PYTHON_BINARY}" \
CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh

View File

@ -1,54 +0,0 @@
#!/bin/bash
# Disable xtrace
set +x
if [ -n "${MONGODB_STARTED}" ]; then
export PYMONGO_MUST_CONNECT=true
fi
if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
export PYMONGO_DISABLE_TEST_COMMANDS=1
fi
if [ -n "${test_encryption}" ]; then
# Disable xtrace (just in case it was accidentally set).
set +x
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh
export TEST_ENCRYPTION=1
if [ -n "${test_encryption_pyopenssl}" ]; then
export TEST_ENCRYPTION_PYOPENSSL=1
fi
fi
if [ -n "${test_crypt_shared}" ]; then
export TEST_CRYPT_SHARED=1
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
fi
if [ -n "${test_pyopenssl}" ]; then
export TEST_PYOPENSSL=1
fi
if [ -n "${SETDEFAULTENCODING}" ]; then
export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
fi
if [ -n "${test_loadbalancer}" ]; then
export TEST_LOADBALANCER=1
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
fi
if [ -n "${test_serverless}" ]; then
export TEST_SERVERLESS=1
fi
if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
export TEST_INDEX_MANAGEMENT=1
fi
if [ -n "${SKIP_CSOT_TESTS}" ]; then
export SKIP_CSOT_TESTS=1
fi
GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
PYTHON_BINARY=${PYTHON_BINARY} \
NO_EXT=${NO_EXT} \
COVERAGE=${COVERAGE} \
COMPRESSORS=${COMPRESSORS} \
AUTH=${AUTH} \
SSL=${SSL} \
TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \
TEST_SUITES=${TEST_SUITES:-} \
MONGODB_API_VERSION=${MONGODB_API_VERSION} \
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg

View File

@ -1,18 +1,21 @@
#!/bin/bash
set -eu
set -eux
HERE=$(dirname ${BASH_SOURCE:-$0})
pushd "$(dirname "$(dirname $HERE)")" > /dev/null
# Source the env file to pick up common variables.
# Source the env files to pick up common variables.
if [ -f $HERE/env.sh ]; then
source $HERE/env.sh
. $HERE/env.sh
fi
# PYTHON_BINARY may be defined in test-env.sh.
if [ -f $HERE/test-env.sh ]; then
. $HERE/test-env.sh
fi
# Ensure dependencies are installed.
. $HERE/install-dependencies.sh
bash $HERE/install-dependencies.sh
# Set the location of the python bin dir.
if [ "Windows_NT" = "${OS:-}" ]; then
@ -32,6 +35,12 @@ if [ ! -d $BIN_DIR ]; then
echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
echo "Using python $UV_PYTHON"
fi
# Add the default install path to the path if needed.
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
export PATH="$PATH:$HOME/.local/bin"
fi
uv sync --frozen
uv run --frozen --with pip pip install -e .
echo "Setting up python environment... done."

View File

@ -1,5 +0,0 @@
#!/bin/bash
if [ -n "${test_encryption}" ]; then
bash .evergreen/setup-encryption.sh
fi

View File

@ -2,11 +2,6 @@
set -o errexit # Exit the script with error if any of the commands fail
set -o xtrace
if [ -z "${DRIVERS_TOOLS}" ]; then
echo "Missing environment variable DRIVERS_TOOLS"
exit 1
fi
TARGET=""
if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
@ -51,10 +46,7 @@ tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
ls -la libmongocrypt
ls -la libmongocrypt/nocrypto
if [ -z "${SKIP_SERVERS:-}" ]; then
PYTHON_BINARY_OLD=${PYTHON_BINARY}
export PYTHON_BINARY=""
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/setup-secrets.sh
export PYTHON_BINARY=$PYTHON_BINARY_OLD
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/start-servers.sh
if [ "Windows_NT" = "${OS:-''}" ]; then
# libmongocrypt's windows dll is not marked executable.
chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll
fi

View File

@ -0,0 +1,299 @@
from __future__ import annotations
import base64
import logging
import os
import platform
import shlex
import stat
import subprocess
import sys
from pathlib import Path
from typing import Any
HERE = Path(__file__).absolute().parent
ROOT = HERE.parent.parent
ENV_FILE = HERE / "test-env.sh"
DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)
EXPECTED_VARS = [
"TEST_ENCRYPTION",
"TEST_ENCRYPTION_PYOPENSSL",
"TEST_CRYPT_SHARED",
"TEST_PYOPENSSL",
"TEST_LOAD_BALANCER",
"TEST_SERVERLESS",
"TEST_INDEX_MANAGEMENT",
"TEST_ENTERPRISE_AUTH",
"TEST_FLE_AZURE_AUTO",
"TEST_FLE_GCP_AUTO",
"TEST_LOADBALANCER",
"TEST_DATA_LAKE",
"TEST_ATLAS",
"TEST_OCSP",
"TEST_AUTH_AWS",
"TEST_AUTH_OIDC",
"COMPRESSORS",
"MONGODB_URI",
"PERF_TEST",
"GREEN_FRAMEWORK",
"PYTHON_BINARY",
"LIBMONGOCRYPT_URL",
]
# Handle the test suite based on the presence of env variables.
TEST_SUITE_MAP = dict(
TEST_DATA_LAKE="data_lake",
TEST_AUTH_OIDC="auth_oidc",
TEST_INDEX_MANAGEMENT="index_management",
TEST_ENTERPRISE_AUTH="auth",
TEST_LOADBALANCER="load_balancer",
TEST_ENCRYPTION="encryption",
TEST_FLE_AZURE_AUTO="csfle",
TEST_FLE_GCP_AUTO="csfle",
TEST_ATLAS="atlas",
TEST_OCSP="ocsp",
TEST_AUTH_AWS="auth_aws",
PERF_TEST="perf",
)
# Handle extras based on the presence of env variables.
EXTRAS_MAP = dict(
TEST_AUTH_OIDC="aws",
TEST_AUTH_AWS="aws",
TEST_OCSP="ocsp",
TEST_PYOPENSSL="ocsp",
TEST_ENTERPRISE_AUTH="gssapi",
TEST_ENCRYPTION="encryption",
TEST_FLE_AZURE_AUTO="encryption",
TEST_FLE_GCP_AUTO="encryption",
TEST_ENCRYPTION_PYOPENSSL="ocsp",
)
def write_env(name: str, value: Any) -> None:
with ENV_FILE.open("a", newline="\n") as fid:
# Remove any existing quote chars.
value = str(value).replace('"', "")
fid.write(f'export {name}="{value}"\n')
def is_set(var: str) -> bool:
value = os.environ.get(var, "")
return len(value.strip()) > 0
def run_command(cmd: str) -> None:
LOGGER.info("Running command %s...", cmd)
subprocess.check_call(shlex.split(cmd)) # noqa: S603
LOGGER.info("Running command %s... done.", cmd)
def handle_test_env() -> None:
AUTH = os.environ.get("AUTH", "noauth")
SSL = os.environ.get("SSL", "nossl")
TEST_SUITES = os.environ.get("TEST_SUITES", "")
TEST_ARGS = ""
# Start compiling the args we'll pass to uv.
# Run in an isolated environment so as not to pollute the base venv.
UV_ARGS = ["--isolated --extra test"]
# Save variables in EXPECTED_VARS that have values.
with ENV_FILE.open("w", newline="\n") as fid:
fid.write("#!/usr/bin/env bash\n")
fid.write("set +x\n")
fid.write(f"export AUTH={AUTH}\n")
fid.write(f"export SSL={SSL}\n")
for var in EXPECTED_VARS:
value = os.environ.get(var, "")
# Remove any existing quote chars.
value = value.replace('"', "")
if value:
fid.write(f'export {var}="{value}"\n')
ENV_FILE.chmod(ENV_FILE.stat().st_mode | stat.S_IEXEC)
for env_var, extra in EXTRAS_MAP.items():
if env_var in os.environ:
UV_ARGS.append(f"--extra {extra}")
for env_var, suite in TEST_SUITE_MAP.items():
if TEST_SUITES:
break
if env_var in os.environ:
TEST_SUITES = suite
if AUTH != "noauth":
if is_set("TEST_DATA_LAKE"):
DB_USER = os.environ["ADL_USERNAME"]
DB_PASSWORD = os.environ["ADL_PASSWORD"]
elif is_set("TEST_SERVERLESS"):
DB_USER = os.environ("SERVERLESS_ATLAS_USER")
DB_PASSWORD = os.environ("SERVERLESS_ATLAS_PASSWORD")
write_env("MONGODB_URI", os.environ("SERVERLESS_URI"))
write_env("SINGLE_MONGOS_LB_URI", os.environ("SERVERLESS_URI"))
write_env("MULTI_MONGOS_LB_URI", os.environ("SERVERLESS_URI"))
elif is_set("TEST_AUTH_OIDC"):
DB_USER = os.environ["OIDC_ADMIN_USER"]
DB_PASSWORD = os.environ["OIDC_ADMIN_PWD"]
write_env("DB_IP", os.environ["MONGODB_URI"])
elif is_set("TEST_INDEX_MANAGEMENT"):
DB_USER = os.environ["DRIVERS_ATLAS_LAMBDA_USER"]
DB_PASSWORD = os.environ["DRIVERS_ATLAS_LAMBDA_PASSWORD"]
else:
DB_USER = "bob"
DB_PASSWORD = "pwd123" # noqa: S105
write_env("DB_USER", DB_USER)
write_env("DB_PASSWORD", DB_PASSWORD)
LOGGER.info("Added auth, DB_USER: %s", DB_USER)
if is_set("MONGODB_STARTED"):
write_env("PYMONGO_MUST_CONNECT", "true")
if is_set("DISABLE_TEST_COMMANDS"):
write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
if is_set("TEST_ENTERPRISE_AUTH"):
if os.name == "nt":
LOGGER.info("Setting GSSAPI_PASS")
write_env("GSSAPI_PASS", os.environ["SASL_PASS"])
write_env("GSSAPI_CANONICALIZE", "true")
else:
# BUILD-3830
krb_conf = ROOT / ".evergreen/krb5.conf.empty"
krb_conf.touch()
write_env("KRB5_CONFIG", krb_conf)
LOGGER.info("Writing keytab")
keytab = base64.b64decode(os.environ["KEYTAB_BASE64"])
keytab_file = ROOT / ".evergreen/drivers.keytab"
with keytab_file.open("wb") as fid:
fid.write(keytab)
principal = os.environ["PRINCIPAL"]
LOGGER.info("Running kinit")
os.environ["KRB5_CONFIG"] = str(krb_conf)
cmd = f"kinit -k -t {keytab_file} -p {principal}"
run_command(cmd)
LOGGER.info("Setting GSSAPI variables")
write_env("GSSAPI_HOST", os.environ["SASL_HOST"])
write_env("GSSAPI_PORT", os.environ["SASL_PORT"])
write_env("GSSAPI_PRINCIPAL", os.environ["PRINCIPAL"])
if is_set("TEST_LOADBALANCER"):
write_env("LOAD_BALANCER", "1")
SINGLE_MONGOS_LB_URI = os.environ.get(
"SINGLE_MONGOS_LB_URI", "mongodb://127.0.0.1:8000/?loadBalanced=true"
)
MULTI_MONGOS_LB_URI = os.environ.get(
"MULTI_MONGOS_LB_URI", "mongodb://127.0.0.1:8001/?loadBalanced=true"
)
if SSL != "nossl":
SINGLE_MONGOS_LB_URI += "&tls=true"
MULTI_MONGOS_LB_URI += "&tls=true"
write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI)
write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI)
if not DRIVERS_TOOLS:
raise RuntimeError("Missing DRIVERS_TOOLS")
cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start'
run_command(cmd)
if SSL != "nossl":
if not DRIVERS_TOOLS:
raise RuntimeError("Missing DRIVERS_TOOLS")
write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
compressors = os.environ.get("COMPRESSORS")
if compressors == "snappy":
UV_ARGS.append("--extra snappy")
elif compressors == "zstd":
UV_ARGS.append("--extra zstandard")
if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
# Check for libmongocrypt download.
if not (ROOT / "libmongocrypt").exists():
run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh")
# TODO: Test with 'pip install pymongocrypt'
UV_ARGS.append("--group pymongocrypt_source")
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
BASE = ROOT / "libmongocrypt/nocrypto"
if sys.platform == "linux":
if (BASE / "lib/libmongocrypt.so").exists():
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so"
else:
PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so"
elif sys.platform == "darwin":
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib"
else:
PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll"
if not PYMONGOCRYPT_LIB.exists():
raise RuntimeError("Cannot find libmongocrypt shared object file")
write_env("PYMONGOCRYPT_LIB", PYMONGOCRYPT_LIB.as_posix())
# PATH is updated by configure-env.sh for access to mongocryptd.
if is_set("TEST_ENCRYPTION"):
if not DRIVERS_TOOLS:
raise RuntimeError("Missing DRIVERS_TOOLS")
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh")
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh")
if is_set("TEST_CRYPT_SHARED"):
CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
if os.name == "nt":
write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH")
else:
write_env(
"DYLD_FALLBACK_LIBRARY_PATH",
f"{CRYPT_SHARED_DIR}:${{DYLD_FALLBACK_LIBRARY_PATH:-}}",
)
write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}")
if is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
if "SUCCESS" not in os.environ:
raise RuntimeError("Must define SUCCESS")
write_env("SUCCESS", os.environ["SUCCESS"])
MONGODB_URI = os.environ.get("MONGODB_URI", "")
if "@" in MONGODB_URI:
raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!")
if is_set("TEST_OCSP"):
write_env("CA_FILE", os.environ["CA_FILE"])
write_env("OCSP_TLS_SHOULD_SUCCEED", os.environ["OCSP_TLS_SHOULD_SUCCEED"])
if is_set("PERF_TEST"):
UV_ARGS.append("--group perf")
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
# affects the benchmark results.
TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}"
# Add coverage if requested.
# Only cover CPython. PyPy reports suspiciously low coverage.
if is_set("COVERAGE") and platform.python_implementation() == "CPython":
# Keep in sync with combine-coverage.sh.
# coverage >=5 is needed for relative_files=true.
UV_ARGS.append("--group coverage")
TEST_ARGS = f"{TEST_ARGS} --cov"
if is_set("GREEN_FRAMEWORK"):
framework = os.environ["GREEN_FRAMEWORK"]
UV_ARGS.append(f"--group {framework}")
else:
# Use --capture=tee-sys so pytest prints test output inline:
# https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
TEST_ARGS = f"-v --capture=tee-sys --durations=5 {TEST_ARGS}"
if TEST_SUITES:
TEST_ARGS = f"-m {TEST_SUITES} {TEST_ARGS}"
write_env("TEST_ARGS", TEST_ARGS)
write_env("UV_ARGS", " ".join(UV_ARGS))
if __name__ == "__main__":
handle_test_env()

View File

@ -1,27 +1,61 @@
#!/bin/bash -eux
#!/bin/bash
set -eu
PROJECT_DIRECTORY="$(pwd)"
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
# Supported/used environment variables:
# AUTH Set to enable authentication. Defaults to "noauth"
# SSL Set to enable SSL. Defaults to "nossl"
# GREEN_FRAMEWORK The green framework to test with, if any.
# COVERAGE If non-empty, run the test suite with coverage.
# COMPRESSORS If non-empty, install appropriate compressor.
# LIBMONGOCRYPT_URL The URL to download libmongocrypt.
# TEST_DATA_LAKE If non-empty, run data lake tests.
# TEST_ENCRYPTION If non-empty, run encryption tests.
# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib.
# TEST_SERVERLESS If non-empy, test on serverless.
# TEST_LOADBALANCER If non-empy, test load balancing.
# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure
# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP
# TEST_PYOPENSSL If non-empy, test with PyOpenSSL
# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth
# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism
# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism
# TEST_PERF If non-empty, run performance tests
# TEST_OCSP If non-empty, run OCSP tests
# TEST_ATLAS If non-empty, test Atlas connections
# TEST_INDEX_MANAGEMENT If non-empty, run index management tests
# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL
# PERF_TEST If non-empty, run the performance tests.
# MONGODB_URI If non-empty, use as the MONGODB_URI in tests.
# PYTHON_BINARY The python binary to use in tests.
if [ -f "$SCRIPT_DIR/test-env.sh" ]; then
echo "Reading $SCRIPT_DIR/test-env.sh file"
. "$SCRIPT_DIR/test-env.sh"
exit 0
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
# Try to source the env file.
if [ -f $SCRIPT_DIR/env.sh ]; then
source $SCRIPT_DIR/env.sh
fi
cat <<EOT > "$SCRIPT_DIR"/test-env.sh
export test_encryption="${test_encryption:-}"
export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}"
export test_crypt_shared="${test_crypt_shared:-}"
export test_pyopenssl="${test_pyopenssl:-}"
export test_loadbalancer="${test_loadbalancer:-}"
export test_serverless="${test_serverless:-}"
export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}"
export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}"
export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}"
export AUTH="${AUTH:-noauth}"
export SSL="${SSL:-nossl}"
export PYTHON_BINARY="${PYTHON_BINARY:-}"
EOT
# Source serverless secrets if applicable.
if [ -n "${TEST_SERVERLESS:-}" ]; then
source $DRIVERS_TOOLS/.evergreen/serverless/secrets-export.sh
fi
chmod +x "$SCRIPT_DIR"/test-env.sh
# Source atlas secrets if applicable.
if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh
fi
# Source ADL secrets if applicable.
if [ -n "${TEST_DATA_LAKE:-}" ]; then
source ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh
fi
# Source local secrets if applicable.
if [ -f "$ROOT_DIR/secrets-export.sh" ]; then
source "$ROOT_DIR/secrets-export.sh"
fi
. $ROOT_DIR/.evergreen/utils.sh
PYTHON=${PYTHON_BINARY:-$(find_python3)}
$PYTHON $SCRIPT_DIR/setup-tests.py

View File

@ -1,5 +0,0 @@
#!/bin/bash
cd "${DRIVERS_TOOLS}"/.evergreen || exit
DRIVERS_TOOLS=${DRIVERS_TOOLS}
bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop

View File

@ -0,0 +1,29 @@
#!/bin/bash
set -eu
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
# Remove temporary test files.
pushd $ROOT_DIR > /dev/null
rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid > /dev/null
popd > /dev/null
if [ ! -f $SCRIPT_DIR/test-env.sh ]; then
exit 0
fi
if [ -f $SCRIPT_DIR/env.sh ]; then
source $SCRIPT_DIR/env.sh
fi
source $SCRIPT_DIR/test-env.sh
# Shut down csfle servers if applicable
if [ -n "${TEST_ENCRYPTION:-}" ]; then
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
fi
# Shut down load balancer if applicable.
if [ -n "${TEST_LOADBALANCER:-}" ]; then
bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop
fi

View File

@ -1,11 +0,0 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
# shellcheck disable=SC2044
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
< "$i" tr -d '\r' >"$i".new
mv "$i".new "$i"
done
# Copy client certificate because symlinks do not work on Windows.
cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem

View File

@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_
echo "Copying files to $target... done"
ssh $target $remote_dir/.evergreen/scripts/setup-system.sh
ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} just install"
ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} .evergreen/scripts/setup-dev-env.sh"

View File

@ -7,9 +7,12 @@ fi
target=$1
user=${target%@*}
remote_dir=/home/$user/mongo-python-driver
echo "Copying files to $target..."
rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_dir
echo "Copying files to $target... done."
echo "Syncing files to $target..."
rsync -haz -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver
# shellcheck disable=SC2034
fswatch -o . | while read f; do rsync -hazv -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver; done
echo "Syncing files to $target... done."

View File

@ -1,10 +0,0 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
set -o xtrace
if [ -z "${DRIVERS_TOOLS}" ]; then
echo "Missing environment variable DRIVERS_TOOLS"
fi
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid

View File

@ -22,13 +22,13 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.9"
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install just
uses: extractions/setup-just@v2
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
python-version: "3.9"
- name: Install Python dependencies
run: |
just install
@ -61,42 +61,21 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- if: ${{ matrix.python-version == '3.13t' }}
name: Setup free-threaded Python
uses: deadsnakes/action@v3.2.0
with:
python-version: 3.13
nogil: true
- if: ${{ matrix.python-version != '3.13t' }}
name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
allow-prereleases: true
- name: Install just
uses: extractions/setup-just@v2
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then
# Just can't be installed on 3.13t, use pytest directly.
pip install .
pip install -r requirements/test.txt
else
just install
fi
run: just install
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.12.0
with:
mongodb-version: 6.0
- name: Run tests
run: |
if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then
pytest -v --durations=5 --maxfail=10
else
just test
fi
run: just test
doctest:
runs-on: ubuntu-latest
@ -105,24 +84,21 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.9"
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install just
uses: extractions/setup-just@v2
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
python-version: "3.9"
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.12.0
with:
mongodb-version: '8.0.0-rc4'
- name: Install dependencies
run: |
just install
run: just install
- name: Run tests
run: |
just docs-test
run: just docs-test
docs:
name: Docs Checks
@ -131,20 +107,17 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
# Build docs on lowest supported Python for furo
python-version: '3.9'
enable-cache: true
python-version: "3.9"
- name: Install just
uses: extractions/setup-just@v2
- name: Install dependencies
run: |
just install
run: just install
- name: Build docs
run: |
just docs
run: just docs
linkcheck:
name: Link Check
@ -153,20 +126,17 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
# Build docs on lowest supported Python for furo
python-version: '3.9'
enable-cache: true
python-version: "3.9"
- name: Install just
uses: extractions/setup-just@v2
- name: Install dependencies
run: |
just install
run: just install
- name: Build docs
run: |
just docs-linkcheck
run: just docs-linkcheck
typing:
name: Typing Tests
@ -178,11 +148,11 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
python-version: "${{matrix.python}}"
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install just
uses: extractions/setup-just@v2
- name: Install dependencies

1
.gitignore vendored
View File

@ -26,6 +26,7 @@ libmongocrypt/
expansion.yml
*expansions.yml
.evergreen/scripts/env.sh
.evergreen/scripts/test-env.sh
# Lambda temp files
test/lambda/.aws-sam

View File

@ -211,18 +211,18 @@ the pages will re-render and the browser will automatically refresh.
`git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`.
- Start the servers using
`LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh`.
- Start the load balancer using:
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`.
- Set up the test using:
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' TEST_LOADBALANCER=1 just setup-test`.
- Run the tests from the `pymongo` checkout directory using:
`TEST_LOADBALANCER=1 just test-eg`.
`just test-eg`.
## Running Encryption Tests Locally
- Clone `drivers-evergreen-tools`:
`git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`.
- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`
- Run `AWS_PROFILE=<profile> just setup-encryption` after setting up your AWS profile with `aws configure sso`.
- Run the tests with `TEST_ENCRYPTION=1 just test-eg`.
- When done, run `just teardown-encryption` to clean up.
- Run `TEST_ENCRYPTION=1 AWS_PROFILE=<profile> just setup-test` after setting up your AWS profile with `aws configure sso`.
- Run the tests with `just test-eg`.
- When done, run `just teardown-test` to clean up.
## Re-sync Spec Tests

View File

@ -1,7 +1,5 @@
# See https://just.systems/man/en/ for instructions
set shell := ["bash", "-c"]
set dotenv-load
set dotenv-filename := "./.evergreen/scripts/env.sh"
# Commonly used command segments.
uv_run := "uv run --isolated --frozen "
@ -70,10 +68,10 @@ test-mockupdb *args:
test-eg *args:
bash ./.evergreen/run-tests.sh {{args}}
[group('encryption')]
setup-encryption:
bash .evergreen/setup-encryption.sh
[group('test')]
setup-test:
bash .evergreen/scripts/setup-tests.sh
[group('encryption')]
teardown-encryption:
bash .evergreen/teardown-encryption.sh
[group('test')]
teardown-test:
bash .evergreen/scripts/teardown-tests.sh

View File

@ -0,0 +1,595 @@
# Copyright 2009-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test built in connection-pooling with threads."""
from __future__ import annotations
import asyncio
import gc
import random
import socket
import sys
import time
from bson.codec_options import DEFAULT_CODEC_OPTIONS
from bson.son import SON
from pymongo import AsyncMongoClient, message, timeout
from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError
from pymongo.hello import HelloCompat
from pymongo.lock import _async_create_lock
sys.path[0:0] = [""]
from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest
from test.asynchronous.helpers import ConcurrentRunner
from test.utils import async_get_pool, async_joinall, delay
from pymongo.asynchronous.pool import Pool, PoolOptions
from pymongo.socket_checker import SocketChecker
_IS_SYNC = False
N = 10
DB = "pymongo-pooling-tests"
async def gc_collect_until_done(tasks, timeout=60):
start = time.time()
running = list(tasks)
while running:
assert (time.time() - start) < timeout, "Tasks timed out"
for t in running:
await t.join(0.1)
if not t.is_alive():
running.remove(t)
gc.collect()
class MongoTask(ConcurrentRunner):
"""A thread/Task that uses a AsyncMongoClient."""
def __init__(self, client):
super().__init__()
self.daemon = True # Don't hang whole test if task hangs.
self.client = client
self.db = self.client[DB]
self.passed = False
async def run(self):
await self.run_mongo_thread()
self.passed = True
async def run_mongo_thread(self):
raise NotImplementedError
class InsertOneAndFind(MongoTask):
async def run_mongo_thread(self):
for _ in range(N):
rand = random.randint(0, N)
_id = (await self.db.sf.insert_one({"x": rand})).inserted_id
assert rand == (await self.db.sf.find_one(_id))["x"]
class Unique(MongoTask):
async def run_mongo_thread(self):
for _ in range(N):
await self.db.unique.insert_one({}) # no error
class NonUnique(MongoTask):
async def run_mongo_thread(self):
for _ in range(N):
try:
await self.db.unique.insert_one({"_id": "jesse"})
except DuplicateKeyError:
pass
else:
raise AssertionError("Should have raised DuplicateKeyError")
class SocketGetter(MongoTask):
"""Utility for TestPooling.
Checks out a socket and holds it forever. Used in
test_no_wait_queue_timeout.
"""
def __init__(self, client, pool):
super().__init__(client)
self.state = "init"
self.pool = pool
self.sock = None
async def run_mongo_thread(self):
self.state = "get_socket"
# Call 'pin_cursor' so we can hold the socket.
async with self.pool.checkout() as sock:
sock.pin_cursor()
self.sock = sock
self.state = "connection"
def __del__(self):
if self.sock:
self.sock.close_conn(None)
async def run_cases(client, cases):
tasks = []
n_runs = 5
for case in cases:
for _i in range(n_runs):
t = case(client)
await t.start()
tasks.append(t)
for t in tasks:
await t.join()
for t in tasks:
assert t.passed, "%s.run() threw an exception" % repr(t)
class _TestPoolingBase(AsyncIntegrationTest):
"""Base class for all connection-pool tests."""
@async_client_context.require_connection
async def asyncSetUp(self):
await super().asyncSetUp()
self.c = await self.async_rs_or_single_client()
db = self.c[DB]
await db.unique.drop()
await db.test.drop()
await db.unique.insert_one({"_id": "jesse"})
await db.test.insert_many([{} for _ in range(10)])
async def create_pool(self, pair=None, *args, **kwargs):
if pair is None:
pair = (await async_client_context.host, await async_client_context.port)
# Start the pool with the correct ssl options.
pool_options = async_client_context.client._topology_settings.pool_options
kwargs["ssl_context"] = pool_options._ssl_context
kwargs["tls_allow_invalid_hostnames"] = pool_options.tls_allow_invalid_hostnames
kwargs["server_api"] = pool_options.server_api
pool = Pool(pair, PoolOptions(*args, **kwargs))
await pool.ready()
return pool
class TestPooling(_TestPoolingBase):
async def test_max_pool_size_validation(self):
host, port = await async_client_context.host, await async_client_context.port
self.assertRaises(ValueError, AsyncMongoClient, host=host, port=port, maxPoolSize=-1)
self.assertRaises(ValueError, AsyncMongoClient, host=host, port=port, maxPoolSize="foo")
c = AsyncMongoClient(host=host, port=port, maxPoolSize=100, connect=False)
self.assertEqual(c.options.pool_options.max_pool_size, 100)
async def test_no_disconnect(self):
await run_cases(self.c, [NonUnique, Unique, InsertOneAndFind])
async def test_pool_reuses_open_socket(self):
# Test Pool's _check_closed() method doesn't close a healthy socket.
cx_pool = await self.create_pool(max_pool_size=10)
cx_pool._check_interval_seconds = 0 # Always check.
async with cx_pool.checkout() as conn:
pass
async with cx_pool.checkout() as new_connection:
self.assertEqual(conn, new_connection)
self.assertEqual(1, len(cx_pool.conns))
async def test_get_socket_and_exception(self):
# get_socket() returns socket after a non-network error.
cx_pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=1)
with self.assertRaises(ZeroDivisionError):
async with cx_pool.checkout() as conn:
1 / 0
# Socket was returned, not closed.
async with cx_pool.checkout() as new_connection:
self.assertEqual(conn, new_connection)
self.assertEqual(1, len(cx_pool.conns))
async def test_pool_removes_closed_socket(self):
# Test that Pool removes explicitly closed socket.
cx_pool = await self.create_pool()
async with cx_pool.checkout() as conn:
# Use Connection's API to close the socket.
conn.close_conn(None)
self.assertEqual(0, len(cx_pool.conns))
async def test_pool_removes_dead_socket(self):
# Test that Pool removes dead socket and the socket doesn't return
# itself PYTHON-344
cx_pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=1)
cx_pool._check_interval_seconds = 0 # Always check.
async with cx_pool.checkout() as conn:
# Simulate a closed socket without telling the Connection it's
# closed.
conn.conn.close()
self.assertTrue(conn.conn_closed())
async with cx_pool.checkout() as new_connection:
self.assertEqual(0, len(cx_pool.conns))
self.assertNotEqual(conn, new_connection)
self.assertEqual(1, len(cx_pool.conns))
# Semaphore was released.
async with cx_pool.checkout():
pass
async def test_socket_closed(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((await async_client_context.host, await async_client_context.port))
socket_checker = SocketChecker()
self.assertFalse(socket_checker.socket_closed(s))
s.close()
self.assertTrue(socket_checker.socket_closed(s))
async def test_socket_checker(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((await async_client_context.host, await async_client_context.port))
socket_checker = SocketChecker()
# Socket has nothing to read.
self.assertFalse(socket_checker.select(s, read=True))
self.assertFalse(socket_checker.select(s, read=True, timeout=0))
self.assertFalse(socket_checker.select(s, read=True, timeout=0.05))
# Socket is writable.
self.assertTrue(socket_checker.select(s, write=True, timeout=None))
self.assertTrue(socket_checker.select(s, write=True))
self.assertTrue(socket_checker.select(s, write=True, timeout=0))
self.assertTrue(socket_checker.select(s, write=True, timeout=0.05))
# Make the socket readable
_, msg, _ = message._query(
0, "admin.$cmd", 0, -1, SON([("ping", 1)]), None, DEFAULT_CODEC_OPTIONS
)
s.sendall(msg)
# Block until the socket is readable.
self.assertTrue(socket_checker.select(s, read=True, timeout=None))
self.assertTrue(socket_checker.select(s, read=True))
self.assertTrue(socket_checker.select(s, read=True, timeout=0))
self.assertTrue(socket_checker.select(s, read=True, timeout=0.05))
# Socket is still writable.
self.assertTrue(socket_checker.select(s, write=True, timeout=None))
self.assertTrue(socket_checker.select(s, write=True))
self.assertTrue(socket_checker.select(s, write=True, timeout=0))
self.assertTrue(socket_checker.select(s, write=True, timeout=0.05))
s.close()
self.assertTrue(socket_checker.socket_closed(s))
async def test_return_socket_after_reset(self):
pool = await self.create_pool()
async with pool.checkout() as sock:
self.assertEqual(pool.active_sockets, 1)
self.assertEqual(pool.operation_count, 1)
await pool.reset()
self.assertTrue(sock.closed)
self.assertEqual(0, len(pool.conns))
self.assertEqual(pool.active_sockets, 0)
self.assertEqual(pool.operation_count, 0)
async def test_pool_check(self):
# Test that Pool recovers from two connection failures in a row.
# This exercises code at the end of Pool._check().
cx_pool = await self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1)
cx_pool._check_interval_seconds = 0 # Always check.
self.addAsyncCleanup(cx_pool.close)
async with cx_pool.checkout() as conn:
# Simulate a closed socket without telling the Connection it's
# closed.
conn.conn.close()
# Swap pool's address with a bad one.
address, cx_pool.address = cx_pool.address, ("foo.com", 1234)
with self.assertRaises(AutoReconnect):
async with cx_pool.checkout():
pass
# Back to normal, semaphore was correctly released.
cx_pool.address = address
async with cx_pool.checkout():
pass
async def test_wait_queue_timeout(self):
wait_queue_timeout = 2 # Seconds
pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=wait_queue_timeout)
self.addAsyncCleanup(pool.close)
async with pool.checkout():
start = time.time()
with self.assertRaises(ConnectionFailure):
async with pool.checkout():
pass
duration = time.time() - start
self.assertTrue(
abs(wait_queue_timeout - duration) < 1,
f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}",
)
async def test_no_wait_queue_timeout(self):
# Verify get_socket() with no wait_queue_timeout blocks forever.
pool = await self.create_pool(max_pool_size=1)
self.addAsyncCleanup(pool.close)
# Reach max_size.
async with pool.checkout() as s1:
t = SocketGetter(self.c, pool)
await t.start()
while t.state != "get_socket":
await asyncio.sleep(0.1)
await asyncio.sleep(1)
self.assertEqual(t.state, "get_socket")
while t.state != "connection":
await asyncio.sleep(0.1)
self.assertEqual(t.state, "connection")
self.assertEqual(t.sock, s1)
async def test_checkout_more_than_max_pool_size(self):
pool = await self.create_pool(max_pool_size=2)
socks = []
for _ in range(2):
# Call 'pin_cursor' so we can hold the socket.
async with pool.checkout() as sock:
sock.pin_cursor()
socks.append(sock)
tasks = []
for _ in range(30):
t = SocketGetter(self.c, pool)
await t.start()
tasks.append(t)
await asyncio.sleep(1)
for t in tasks:
self.assertEqual(t.state, "get_socket")
for socket_info in socks:
socket_info.close_conn(None)
async def test_maxConnecting(self):
client = await self.async_rs_or_single_client()
await self.client.test.test.insert_one({})
self.addAsyncCleanup(self.client.test.test.delete_many, {})
pool = await async_get_pool(client)
docs = []
# Run 50 short running operations
async def find_one():
docs.append(await client.test.test.find_one({}))
tasks = [ConcurrentRunner(target=find_one) for _ in range(50)]
for task in tasks:
await task.start()
for task in tasks:
await task.join(10)
self.assertEqual(len(docs), 50)
self.assertLessEqual(len(pool.conns), 50)
# TLS and auth make connection establishment more expensive than
# the query which leads to more threads hitting maxConnecting.
# The end result is fewer total connections and better latency.
if async_client_context.tls and async_client_context.auth_enabled:
self.assertLessEqual(len(pool.conns), 30)
else:
self.assertLessEqual(len(pool.conns), 50)
# MongoDB 4.4.1 with auth + ssl:
# maxConnecting = 2: 6 connections in ~0.231+ seconds
# maxConnecting = unbounded: 50 connections in ~0.642+ seconds
#
# MongoDB 4.4.1 with no-auth no-ssl Python 3.8:
# maxConnecting = 2: 15-22 connections in ~0.108+ seconds
# maxConnecting = unbounded: 30+ connections in ~0.140+ seconds
print(len(pool.conns))
@async_client_context.require_failCommand_appName
async def test_csot_timeout_message(self):
client = await self.async_rs_or_single_client(appName="connectionTimeoutApp")
# Mock an operation failing due to pymongo.timeout().
mock_connection_timeout = {
"configureFailPoint": "failCommand",
"mode": "alwaysOn",
"data": {
"blockConnection": True,
"blockTimeMS": 1000,
"failCommands": ["find"],
"appName": "connectionTimeoutApp",
},
}
await client.db.t.insert_one({"x": 1})
async with self.fail_point(mock_connection_timeout):
with self.assertRaises(Exception) as error:
with timeout(0.5):
await client.db.t.find_one({"$where": delay(2)})
self.assertTrue("(configured timeouts: timeoutMS: 500.0ms" in str(error.exception))
@async_client_context.require_failCommand_appName
async def test_socket_timeout_message(self):
client = await self.async_rs_or_single_client(
socketTimeoutMS=500, appName="connectionTimeoutApp"
)
# Mock an operation failing due to socketTimeoutMS.
mock_connection_timeout = {
"configureFailPoint": "failCommand",
"mode": "alwaysOn",
"data": {
"blockConnection": True,
"blockTimeMS": 1000,
"failCommands": ["find"],
"appName": "connectionTimeoutApp",
},
}
await client.db.t.insert_one({"x": 1})
async with self.fail_point(mock_connection_timeout):
with self.assertRaises(Exception) as error:
await client.db.t.find_one({"$where": delay(2)})
self.assertTrue(
"(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 20000.0ms)"
in str(error.exception)
)
@async_client_context.require_failCommand_appName
async def test_connection_timeout_message(self):
# Mock a connection creation failing due to timeout.
mock_connection_timeout = {
"configureFailPoint": "failCommand",
"mode": "alwaysOn",
"data": {
"blockConnection": True,
"blockTimeMS": 1000,
"failCommands": [HelloCompat.LEGACY_CMD, "hello"],
"appName": "connectionTimeoutApp",
},
}
client = await self.async_rs_or_single_client(
connectTimeoutMS=500,
socketTimeoutMS=500,
appName="connectionTimeoutApp",
heartbeatFrequencyMS=1000000,
)
await client.admin.command("ping")
pool = await async_get_pool(client)
await pool.reset_without_pause()
async with self.fail_point(mock_connection_timeout):
with self.assertRaises(Exception) as error:
await client.admin.command("ping")
self.assertTrue(
"(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)"
in str(error.exception)
)
class TestPoolMaxSize(_TestPoolingBase):
async def test_max_pool_size(self):
max_pool_size = 4
c = await self.async_rs_or_single_client(maxPoolSize=max_pool_size)
collection = c[DB].test
# Need one document.
await collection.drop()
await collection.insert_one({})
# ntasks had better be much larger than max_pool_size to ensure that
# max_pool_size connections are actually required at some point in this
# test's execution.
cx_pool = await async_get_pool(c)
ntasks = 10
tasks = []
lock = _async_create_lock()
self.n_passed = 0
async def f():
for _ in range(5):
await collection.find_one({"$where": delay(0.1)})
assert len(cx_pool.conns) <= max_pool_size
async with lock:
self.n_passed += 1
for _i in range(ntasks):
t = ConcurrentRunner(target=f)
tasks.append(t)
await t.start()
await async_joinall(tasks)
self.assertEqual(ntasks, self.n_passed)
self.assertTrue(len(cx_pool.conns) > 1)
self.assertEqual(0, cx_pool.requests)
async def test_max_pool_size_none(self):
c = await self.async_rs_or_single_client(maxPoolSize=None)
collection = c[DB].test
# Need one document.
await collection.drop()
await collection.insert_one({})
cx_pool = await async_get_pool(c)
ntasks = 10
tasks = []
lock = _async_create_lock()
self.n_passed = 0
async def f():
for _ in range(5):
await collection.find_one({"$where": delay(0.1)})
async with lock:
self.n_passed += 1
for _i in range(ntasks):
t = ConcurrentRunner(target=f)
tasks.append(t)
await t.start()
await async_joinall(tasks)
self.assertEqual(ntasks, self.n_passed)
self.assertTrue(len(cx_pool.conns) > 1)
self.assertEqual(cx_pool.max_pool_size, float("inf"))
async def test_max_pool_size_zero(self):
c = await self.async_rs_or_single_client(maxPoolSize=0)
pool = await async_get_pool(c)
self.assertEqual(pool.max_pool_size, float("inf"))
async def test_max_pool_size_with_connection_failure(self):
# The pool acquires its semaphore before attempting to connect; ensure
# it releases the semaphore on connection failure.
test_pool = Pool(
("somedomainthatdoesntexist.org", 27017),
PoolOptions(max_pool_size=1, connect_timeout=1, socket_timeout=1, wait_queue_timeout=1),
)
await test_pool.ready()
# First call to get_socket fails; if pool doesn't release its semaphore
# then the second call raises "ConnectionFailure: Timed out waiting for
# socket from pool" instead of AutoReconnect.
for _i in range(2):
with self.assertRaises(AutoReconnect) as context:
async with test_pool.checkout():
pass
# Testing for AutoReconnect instead of ConnectionFailure, above,
# is sufficient right *now* to catch a semaphore leak. But that
# seems error-prone, so check the message too.
self.assertNotIn("waiting for socket from pool", str(context.exception))
if __name__ == "__main__":
unittest.main()

View File

@ -15,11 +15,11 @@
"""Test built in connection-pooling with threads."""
from __future__ import annotations
import asyncio
import gc
import random
import socket
import sys
import threading
import time
from bson.codec_options import DEFAULT_CODEC_OPTIONS
@ -27,30 +27,29 @@ from bson.son import SON
from pymongo import MongoClient, message, timeout
from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError
from pymongo.hello import HelloCompat
from pymongo.lock import _create_lock
sys.path[0:0] = [""]
from test import IntegrationTest, client_context, unittest
from test.helpers import ConcurrentRunner
from test.utils import delay, get_pool, joinall
from pymongo.socket_checker import SocketChecker
from pymongo.synchronous.pool import Pool, PoolOptions
@client_context.require_connection
def setUpModule():
pass
_IS_SYNC = True
N = 10
DB = "pymongo-pooling-tests"
def gc_collect_until_done(threads, timeout=60):
def gc_collect_until_done(tasks, timeout=60):
start = time.time()
running = list(threads)
running = list(tasks)
while running:
assert (time.time() - start) < timeout, "Threads timed out"
assert (time.time() - start) < timeout, "Tasks timed out"
for t in running:
t.join(0.1)
if not t.is_alive():
@ -58,12 +57,12 @@ def gc_collect_until_done(threads, timeout=60):
gc.collect()
class MongoThread(threading.Thread):
"""A thread that uses a MongoClient."""
class MongoTask(ConcurrentRunner):
"""A thread/Task that uses a MongoClient."""
def __init__(self, client):
super().__init__()
self.daemon = True # Don't hang whole test if thread hangs.
self.daemon = True # Don't hang whole test if task hangs.
self.client = client
self.db = self.client[DB]
self.passed = False
@ -76,21 +75,21 @@ class MongoThread(threading.Thread):
raise NotImplementedError
class InsertOneAndFind(MongoThread):
class InsertOneAndFind(MongoTask):
def run_mongo_thread(self):
for _ in range(N):
rand = random.randint(0, N)
_id = self.db.sf.insert_one({"x": rand}).inserted_id
assert rand == self.db.sf.find_one(_id)["x"]
_id = (self.db.sf.insert_one({"x": rand})).inserted_id
assert rand == (self.db.sf.find_one(_id))["x"]
class Unique(MongoThread):
class Unique(MongoTask):
def run_mongo_thread(self):
for _ in range(N):
self.db.unique.insert_one({}) # no error
class NonUnique(MongoThread):
class NonUnique(MongoTask):
def run_mongo_thread(self):
for _ in range(N):
try:
@ -101,7 +100,7 @@ class NonUnique(MongoThread):
raise AssertionError("Should have raised DuplicateKeyError")
class SocketGetter(MongoThread):
class SocketGetter(MongoTask):
"""Utility for TestPooling.
Checks out a socket and holds it forever. Used in
@ -130,25 +129,26 @@ class SocketGetter(MongoThread):
def run_cases(client, cases):
threads = []
tasks = []
n_runs = 5
for case in cases:
for _i in range(n_runs):
t = case(client)
t.start()
threads.append(t)
tasks.append(t)
for t in threads:
for t in tasks:
t.join()
for t in threads:
for t in tasks:
assert t.passed, "%s.run() threw an exception" % repr(t)
class _TestPoolingBase(IntegrationTest):
"""Base class for all connection-pool tests."""
@client_context.require_connection
def setUp(self):
super().setUp()
self.c = self.rs_or_single_client()
@ -158,11 +158,9 @@ class _TestPoolingBase(IntegrationTest):
db.unique.insert_one({"_id": "jesse"})
db.test.insert_many([{} for _ in range(10)])
def tearDown(self):
self.c.close()
super().tearDown()
def create_pool(self, pair=(client_context.host, client_context.port), *args, **kwargs):
def create_pool(self, pair=None, *args, **kwargs):
if pair is None:
pair = (client_context.host, client_context.port)
# Start the pool with the correct ssl options.
pool_options = client_context.client._topology_settings.pool_options
kwargs["ssl_context"] = pool_options._ssl_context
@ -365,13 +363,13 @@ class TestPooling(_TestPoolingBase):
sock.pin_cursor()
socks.append(sock)
threads = []
tasks = []
for _ in range(30):
t = SocketGetter(self.c, pool)
t.start()
threads.append(t)
tasks.append(t)
time.sleep(1)
for t in threads:
for t in tasks:
self.assertEqual(t.state, "get_socket")
for socket_info in socks:
@ -379,7 +377,6 @@ class TestPooling(_TestPoolingBase):
def test_maxConnecting(self):
client = self.rs_or_single_client()
self.addCleanup(client.close)
self.client.test.test.insert_one({})
self.addCleanup(self.client.test.test.delete_many, {})
pool = get_pool(client)
@ -389,11 +386,11 @@ class TestPooling(_TestPoolingBase):
def find_one():
docs.append(client.test.test.find_one({}))
threads = [threading.Thread(target=find_one) for _ in range(50)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(10)
tasks = [ConcurrentRunner(target=find_one) for _ in range(50)]
for task in tasks:
task.start()
for task in tasks:
task.join(10)
self.assertEqual(len(docs), 50)
self.assertLessEqual(len(pool.conns), 50)
@ -416,7 +413,6 @@ class TestPooling(_TestPoolingBase):
@client_context.require_failCommand_appName
def test_csot_timeout_message(self):
client = self.rs_or_single_client(appName="connectionTimeoutApp")
self.addCleanup(client.close)
# Mock an operation failing due to pymongo.timeout().
mock_connection_timeout = {
"configureFailPoint": "failCommand",
@ -441,7 +437,6 @@ class TestPooling(_TestPoolingBase):
@client_context.require_failCommand_appName
def test_socket_timeout_message(self):
client = self.rs_or_single_client(socketTimeoutMS=500, appName="connectionTimeoutApp")
self.addCleanup(client.close)
# Mock an operation failing due to socketTimeoutMS.
mock_connection_timeout = {
"configureFailPoint": "failCommand",
@ -485,7 +480,6 @@ class TestPooling(_TestPoolingBase):
appName="connectionTimeoutApp",
heartbeatFrequencyMS=1000000,
)
self.addCleanup(client.close)
client.admin.command("ping")
pool = get_pool(client)
pool.reset_without_pause()
@ -503,20 +497,19 @@ class TestPoolMaxSize(_TestPoolingBase):
def test_max_pool_size(self):
max_pool_size = 4
c = self.rs_or_single_client(maxPoolSize=max_pool_size)
self.addCleanup(c.close)
collection = c[DB].test
# Need one document.
collection.drop()
collection.insert_one({})
# nthreads had better be much larger than max_pool_size to ensure that
# ntasks had better be much larger than max_pool_size to ensure that
# max_pool_size connections are actually required at some point in this
# test's execution.
cx_pool = get_pool(c)
nthreads = 10
threads = []
lock = threading.Lock()
ntasks = 10
tasks = []
lock = _create_lock()
self.n_passed = 0
def f():
@ -527,19 +520,18 @@ class TestPoolMaxSize(_TestPoolingBase):
with lock:
self.n_passed += 1
for _i in range(nthreads):
t = threading.Thread(target=f)
threads.append(t)
for _i in range(ntasks):
t = ConcurrentRunner(target=f)
tasks.append(t)
t.start()
joinall(threads)
self.assertEqual(nthreads, self.n_passed)
joinall(tasks)
self.assertEqual(ntasks, self.n_passed)
self.assertTrue(len(cx_pool.conns) > 1)
self.assertEqual(0, cx_pool.requests)
def test_max_pool_size_none(self):
c = self.rs_or_single_client(maxPoolSize=None)
self.addCleanup(c.close)
collection = c[DB].test
# Need one document.
@ -547,9 +539,9 @@ class TestPoolMaxSize(_TestPoolingBase):
collection.insert_one({})
cx_pool = get_pool(c)
nthreads = 10
threads = []
lock = threading.Lock()
ntasks = 10
tasks = []
lock = _create_lock()
self.n_passed = 0
def f():
@ -559,19 +551,18 @@ class TestPoolMaxSize(_TestPoolingBase):
with lock:
self.n_passed += 1
for _i in range(nthreads):
t = threading.Thread(target=f)
threads.append(t)
for _i in range(ntasks):
t = ConcurrentRunner(target=f)
tasks.append(t)
t.start()
joinall(threads)
self.assertEqual(nthreads, self.n_passed)
joinall(tasks)
self.assertEqual(ntasks, self.n_passed)
self.assertTrue(len(cx_pool.conns) > 1)
self.assertEqual(cx_pool.max_pool_size, float("inf"))
def test_max_pool_size_zero(self):
c = self.rs_or_single_client(maxPoolSize=0)
self.addCleanup(c.close)
pool = get_pool(c)
self.assertEqual(pool.max_pool_size, float("inf"))

View File

@ -228,6 +228,7 @@ converted_tests = [
"test_monitoring.py",
"test_mongos_load_balancing.py",
"test_on_demand_csfle.py",
"test_pooling.py",
"test_raw_bson.py",
"test_read_concern.py",
"test_read_preferences.py",