Merge branch 'master' of github.com:mongodb/mongo-python-driver
This commit is contained in:
commit
230dd08965
@ -42,7 +42,7 @@ functions:
|
||||
# Make an evergreen expansion file with dynamic values
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"]
|
||||
include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "TEST_ENCRYPTION", "TEST_ENCRYPTION_PYOPENSSL", "TEST_CRYPT_SHARED", "TEST_PYOPENSSL", "SETDEFAULTENCODING", "TEST_LOADBALANCER", "TEST_SEVERLESS", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
@ -274,39 +274,22 @@ functions:
|
||||
|
||||
"run tests":
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
include_expansions_in_env: ["TEST_DATA_LAKE", "PYTHON_BINARY", "AUTH", "SSL", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"]
|
||||
include_expansions_in_env: ["TEST_DATA_LAKE", "PYTHON_BINARY", "AUTH", "SSL",
|
||||
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_SUITES",
|
||||
"TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "TEST_ENCRYPTION", "TEST_ENCRYPTION_PYOPENSSL",
|
||||
"TEST_CRYPT_SHARED", "TEST_PYOPENSSL", "TEST_LOADBALANCER", "TEST_SEVERLESS", "MONGODB_URI"]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
working_dir: "src"
|
||||
binary: bash
|
||||
background: true
|
||||
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
|
||||
args:
|
||||
- .evergreen/scripts/run-with-env.sh
|
||||
- .evergreen/scripts/setup-encryption.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
working_dir: "src"
|
||||
binary: bash
|
||||
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"]
|
||||
args:
|
||||
- .evergreen/scripts/run-with-env.sh
|
||||
- .evergreen/scripts/run-tests.sh
|
||||
|
||||
"run direct tests":
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
working_dir: "src"
|
||||
binary: bash
|
||||
include_expansions_in_env: ["PYTHON_BINARY"]
|
||||
args: [ .evergreen/scripts/run-direct-tests.sh ]
|
||||
args: [.evergreen/just.sh, test-eg]
|
||||
|
||||
"run enterprise auth tests":
|
||||
- command: subprocess.exec
|
||||
@ -340,13 +323,6 @@ functions:
|
||||
- ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh
|
||||
|
||||
"run aws auth test with regular aws credentials":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -359,13 +335,6 @@ functions:
|
||||
- regular
|
||||
|
||||
"run aws auth test with assume role credentials":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -378,13 +347,6 @@ functions:
|
||||
- assume-role
|
||||
|
||||
"run aws auth test with aws EC2 credentials":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -397,13 +359,6 @@ functions:
|
||||
- ec2
|
||||
|
||||
"run aws auth test with aws web identity credentials":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- # Test with and without AWS_ROLE_SESSION_NAME set.
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
@ -429,13 +384,6 @@ functions:
|
||||
- web-identity
|
||||
|
||||
"run aws auth test with aws credentials as environment variables":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -448,13 +396,6 @@ functions:
|
||||
- env-creds
|
||||
|
||||
"run aws auth test with aws credentials and session token as environment variables":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -467,13 +408,6 @@ functions:
|
||||
- session-creds
|
||||
|
||||
"run oidc auth test with test credentials":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -561,13 +495,6 @@ functions:
|
||||
file: atlas-expansion.yml
|
||||
|
||||
"run-ocsp-test":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
|
||||
binary: bash
|
||||
working_dir: "src"
|
||||
args:
|
||||
- .evergreen/scripts/setup-tests.sh
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
@ -587,25 +514,6 @@ functions:
|
||||
args:
|
||||
- ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh
|
||||
|
||||
"run load-balancer":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
binary: bash
|
||||
include_expansions_in_env: ["MONGODB_URI"]
|
||||
args:
|
||||
- src/.evergreen/scripts/run-with-env.sh
|
||||
- src/.evergreen/scripts/run-load-balancer.sh
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: lb-expansion.yml
|
||||
|
||||
"stop load-balancer":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
binary: bash
|
||||
args:
|
||||
- src/.evergreen/scripts/stop-load-balancer.sh
|
||||
|
||||
"teardown atlas":
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
@ -882,6 +790,7 @@ tasks:
|
||||
- func: "run tests"
|
||||
vars:
|
||||
TEST_INDEX_MANAGEMENT: "1"
|
||||
AUTH: "auth"
|
||||
|
||||
- name: "mod-wsgi-standalone"
|
||||
tags: ["mod_wsgi"]
|
||||
@ -935,7 +844,7 @@ tasks:
|
||||
vars:
|
||||
VERSION: "8.0"
|
||||
TOPOLOGY: "replica_set"
|
||||
- func: "run direct tests"
|
||||
- func: "run tests"
|
||||
|
||||
- name: "atlas-connect"
|
||||
tags: ["atlas-connect"]
|
||||
@ -1503,7 +1412,7 @@ tasks:
|
||||
- name: "testgcpkms-task"
|
||||
commands:
|
||||
- command: subprocess.exec
|
||||
type: setup
|
||||
type: test
|
||||
params:
|
||||
working_dir: "src"
|
||||
binary: bash
|
||||
@ -1531,6 +1440,7 @@ tasks:
|
||||
- name: testazurekms-task
|
||||
commands:
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
binary: bash
|
||||
working_dir: src
|
||||
|
||||
@ -8,12 +8,11 @@ tasks:
|
||||
AUTH: auth
|
||||
SSL: ssl
|
||||
LOAD_BALANCER: "true"
|
||||
- func: run load-balancer
|
||||
- func: run tests
|
||||
vars:
|
||||
AUTH: auth
|
||||
SSL: ssl
|
||||
test_loadbalancer: "true"
|
||||
TEST_LOADBALANCER: "true"
|
||||
tags: [load-balancer, auth, ssl]
|
||||
- name: test-load-balancer-noauth-ssl
|
||||
commands:
|
||||
@ -23,12 +22,11 @@ tasks:
|
||||
AUTH: noauth
|
||||
SSL: ssl
|
||||
LOAD_BALANCER: "true"
|
||||
- func: run load-balancer
|
||||
- func: run tests
|
||||
vars:
|
||||
AUTH: noauth
|
||||
SSL: ssl
|
||||
test_loadbalancer: "true"
|
||||
TEST_LOADBALANCER: "true"
|
||||
tags: [load-balancer, noauth, ssl]
|
||||
- name: test-load-balancer-noauth-nossl
|
||||
commands:
|
||||
@ -38,12 +36,11 @@ tasks:
|
||||
AUTH: noauth
|
||||
SSL: nossl
|
||||
LOAD_BALANCER: "true"
|
||||
- func: run load-balancer
|
||||
- func: run tests
|
||||
vars:
|
||||
AUTH: noauth
|
||||
SSL: nossl
|
||||
test_loadbalancer: "true"
|
||||
TEST_LOADBALANCER: "true"
|
||||
tags: [load-balancer, noauth, nossl]
|
||||
|
||||
# Server tests
|
||||
|
||||
@ -318,7 +318,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: /opt/python/3.9/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-rhel8-python3.13
|
||||
@ -331,7 +331,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: /opt/python/3.13/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-rhel8-pypy3.10
|
||||
@ -344,7 +344,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-rhel8-python3.9
|
||||
@ -357,8 +357,8 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: /opt/python/3.9/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-rhel8-python3.13
|
||||
@ -371,8 +371,8 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: /opt/python/3.13/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-rhel8-pypy3.10
|
||||
@ -385,8 +385,8 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-pyopenssl-rhel8-python3.9
|
||||
@ -399,8 +399,8 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_encryption_pyopenssl: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_ENCRYPTION_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/3.9/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-pyopenssl-rhel8-python3.13
|
||||
@ -413,8 +413,8 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_encryption_pyopenssl: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_ENCRYPTION_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/3.13/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-pyopenssl-rhel8-pypy3.10
|
||||
@ -427,8 +427,8 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_encryption_pyopenssl: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_ENCRYPTION_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-rhel8-python3.10
|
||||
@ -438,7 +438,7 @@ buildvariants:
|
||||
run_on:
|
||||
- rhel87-small
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: /opt/python/3.10/bin/python3
|
||||
- name: encryption-crypt_shared-rhel8-python3.11
|
||||
tasks:
|
||||
@ -447,8 +447,8 @@ buildvariants:
|
||||
run_on:
|
||||
- rhel87-small
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: /opt/python/3.11/bin/python3
|
||||
- name: encryption-pyopenssl-rhel8-python3.12
|
||||
tasks:
|
||||
@ -457,8 +457,8 @@ buildvariants:
|
||||
run_on:
|
||||
- rhel87-small
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_encryption_pyopenssl: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_ENCRYPTION_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/3.12/bin/python3
|
||||
- name: encryption-macos-python3.9
|
||||
tasks:
|
||||
@ -468,7 +468,7 @@ buildvariants:
|
||||
- macos-14
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-macos-python3.13
|
||||
@ -479,7 +479,7 @@ buildvariants:
|
||||
- macos-14
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-macos-python3.9
|
||||
@ -490,8 +490,8 @@ buildvariants:
|
||||
- macos-14
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-macos-python3.13
|
||||
@ -502,8 +502,8 @@ buildvariants:
|
||||
- macos-14
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-win64-python3.9
|
||||
@ -514,7 +514,7 @@ buildvariants:
|
||||
- windows-64-vsMulti-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: C:/python/Python39/python.exe
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-win64-python3.13
|
||||
@ -525,7 +525,7 @@ buildvariants:
|
||||
- windows-64-vsMulti-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
PYTHON_BINARY: C:/python/Python313/python.exe
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-win64-python3.9
|
||||
@ -536,8 +536,8 @@ buildvariants:
|
||||
- windows-64-vsMulti-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: C:/python/Python39/python.exe
|
||||
tags: [encryption_tag]
|
||||
- name: encryption-crypt_shared-win64-python3.13
|
||||
@ -548,8 +548,8 @@ buildvariants:
|
||||
- windows-64-vsMulti-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_encryption: "true"
|
||||
test_crypt_shared: "true"
|
||||
TEST_ENCRYPTION: "true"
|
||||
TEST_CRYPT_SHARED: "true"
|
||||
PYTHON_BINARY: C:/python/Python313/python.exe
|
||||
tags: [encryption_tag]
|
||||
|
||||
@ -1010,7 +1010,7 @@ buildvariants:
|
||||
- macos-14
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_pyopenssl: "true"
|
||||
TEST_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
|
||||
- name: pyopenssl-rhel8-python3.10
|
||||
tasks:
|
||||
@ -1021,7 +1021,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_pyopenssl: "true"
|
||||
TEST_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/3.10/bin/python3
|
||||
- name: pyopenssl-rhel8-python3.11
|
||||
tasks:
|
||||
@ -1032,7 +1032,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_pyopenssl: "true"
|
||||
TEST_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/3.11/bin/python3
|
||||
- name: pyopenssl-rhel8-python3.12
|
||||
tasks:
|
||||
@ -1043,7 +1043,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_pyopenssl: "true"
|
||||
TEST_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/3.12/bin/python3
|
||||
- name: pyopenssl-win64-python3.13
|
||||
tasks:
|
||||
@ -1054,7 +1054,7 @@ buildvariants:
|
||||
- windows-64-vsMulti-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_pyopenssl: "true"
|
||||
TEST_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: C:/python/Python313/python.exe
|
||||
- name: pyopenssl-rhel8-pypy3.10
|
||||
tasks:
|
||||
@ -1065,7 +1065,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_pyopenssl: "true"
|
||||
TEST_PYOPENSSL: "true"
|
||||
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
|
||||
|
||||
# Search index tests
|
||||
@ -1301,7 +1301,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_serverless: "true"
|
||||
TEST_SERVERLESS: "true"
|
||||
AUTH: auth
|
||||
SSL: ssl
|
||||
PYTHON_BINARY: /opt/python/3.9/bin/python3
|
||||
@ -1313,7 +1313,7 @@ buildvariants:
|
||||
- rhel87-small
|
||||
batchtime: 10080
|
||||
expansions:
|
||||
test_serverless: "true"
|
||||
TEST_SERVERLESS: "true"
|
||||
AUTH: auth
|
||||
SSL: ssl
|
||||
PYTHON_BINARY: /opt/python/3.13/bin/python3
|
||||
|
||||
@ -3,10 +3,9 @@ set -o errexit # Exit the script with error if any of the commands fail
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
|
||||
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
|
||||
SKIP_SERVERS=1 bash $HERE/setup-encryption.sh
|
||||
SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh
|
||||
PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
|
||||
KEY_NAME="${AZUREKMS_KEYNAME}" \
|
||||
KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
|
||||
SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
|
||||
$HERE/just.sh test-eg
|
||||
bash $HERE/teardown-encryption.sh
|
||||
bash $HERE/scripts/teardown-tests.sh
|
||||
|
||||
@ -6,24 +6,23 @@ echo "Copying files ... begin"
|
||||
export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
|
||||
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
|
||||
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
|
||||
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
|
||||
SKIP_SERVERS=1 bash $HERE/setup-encryption.sh
|
||||
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
|
||||
# Set up the remote files to test.
|
||||
git add .
|
||||
git commit -m "add files" || true
|
||||
git archive -o /tmp/mongo-python-driver.tar HEAD
|
||||
tar -rf /tmp/mongo-python-driver.tar libmongocrypt
|
||||
gzip -f /tmp/mongo-python-driver.tar
|
||||
git archive -o /tmp/mongo-python-driver.tgz HEAD
|
||||
# shellcheck disable=SC2088
|
||||
AZUREKMS_SRC="/tmp/mongo-python-driver.tar.gz" AZUREKMS_DST="~/" \
|
||||
AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \
|
||||
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
|
||||
echo "Copying files ... end"
|
||||
echo "Untarring file ... begin"
|
||||
AZUREKMS_CMD="tar xf mongo-python-driver.tar.gz" \
|
||||
AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
|
||||
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
|
||||
echo "Untarring file ... end"
|
||||
echo "Running test ... begin"
|
||||
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash ./.evergreen/just.sh test-eg" \
|
||||
AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \
|
||||
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
|
||||
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \
|
||||
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
|
||||
echo "Running test ... end"
|
||||
bash $HERE/teardown-encryption.sh
|
||||
bash $HERE/scripts/teardown-tests.sh
|
||||
|
||||
@ -8,20 +8,18 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
|
||||
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
|
||||
export GCPKMS_ZONE=${GCPKMS_ZONE}
|
||||
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
|
||||
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
|
||||
SKIP_SERVERS=1 bash $HERE/setup-encryption.sh
|
||||
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
|
||||
# Set up the remote files to test.
|
||||
git add .
|
||||
git commit -m "add files" || true
|
||||
git archive -o /tmp/mongo-python-driver.tar HEAD
|
||||
tar -rf /tmp/mongo-python-driver.tar libmongocrypt
|
||||
gzip -f /tmp/mongo-python-driver.tar
|
||||
GCPKMS_SRC=/tmp/mongo-python-driver.tar.gz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
|
||||
git archive -o /tmp/mongo-python-driver.tgz HEAD
|
||||
GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
|
||||
echo "Copying files ... end"
|
||||
echo "Untarring file ... begin"
|
||||
GCPKMS_CMD="tar xf mongo-python-driver.tar.gz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
|
||||
GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
|
||||
echo "Untarring file ... end"
|
||||
echo "Running test ... begin"
|
||||
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
|
||||
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
|
||||
GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
|
||||
echo "Running test ... end"
|
||||
bash $HERE/teardown-encryption.sh
|
||||
bash $HERE/scripts/teardown-tests.sh
|
||||
|
||||
@ -31,4 +31,5 @@ export AUTH="auth"
|
||||
export SET_XTRACE_ON=1
|
||||
cd src
|
||||
rm -rf .venv
|
||||
bash ./.evergreen/just.sh setup-test
|
||||
bash .evergreen/just.sh test-eg
|
||||
|
||||
@ -29,7 +29,5 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export TEST_AUTH_OIDC=1
|
||||
export COVERAGE=1
|
||||
export AUTH="auth"
|
||||
TEST_AUTH_OIDC=1 COVERAGE=1 AUTH="auth" bash ./.evergreen/just.sh setup-test
|
||||
bash ./.evergreen/just.sh test-eg "${@:1}"
|
||||
|
||||
@ -16,4 +16,5 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
|
||||
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
|
||||
export PERF_TEST=1
|
||||
|
||||
bash ./.evergreen/just.sh setup-test
|
||||
bash ./.evergreen/just.sh test-eg
|
||||
|
||||
@ -1,283 +1,81 @@
|
||||
#!/bin/bash
|
||||
set -o errexit # Exit the script with error if any of the commands fail
|
||||
set -o xtrace
|
||||
set -eu
|
||||
|
||||
# Note: It is assumed that you have already set up a virtual environment before running this file.
|
||||
|
||||
# Supported/used environment variables:
|
||||
# AUTH Set to enable authentication. Defaults to "noauth"
|
||||
# SSL Set to enable SSL. Defaults to "nossl"
|
||||
# GREEN_FRAMEWORK The green framework to test with, if any.
|
||||
# COVERAGE If non-empty, run the test suite with coverage.
|
||||
# COMPRESSORS If non-empty, install appropriate compressor.
|
||||
# LIBMONGOCRYPT_URL The URL to download libmongocrypt.
|
||||
# TEST_DATA_LAKE If non-empty, run data lake tests.
|
||||
# TEST_ENCRYPTION If non-empty, run encryption tests.
|
||||
# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib.
|
||||
# TEST_SERVERLESS If non-empy, test on serverless.
|
||||
# TEST_LOADBALANCER If non-empy, test load balancing.
|
||||
# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure
|
||||
# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP
|
||||
# TEST_PYOPENSSL If non-empy, test with PyOpenSSL
|
||||
# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth
|
||||
# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism
|
||||
# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism
|
||||
# TEST_PERF If non-empty, run performance tests
|
||||
# TEST_OCSP If non-empty, run OCSP tests
|
||||
# TEST_ATLAS If non-empty, test Atlas connections
|
||||
# TEST_INDEX_MANAGEMENT If non-empty, run index management tests
|
||||
# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL
|
||||
|
||||
AUTH=${AUTH:-noauth}
|
||||
SSL=${SSL:-nossl}
|
||||
TEST_SUITES=${TEST_SUITES:-}
|
||||
TEST_ARGS="${*:1}"
|
||||
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
|
||||
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
|
||||
|
||||
export PIP_QUIET=1 # Quiet by default
|
||||
export PIP_PREFER_BINARY=1 # Prefer binary dists by default
|
||||
export UV_FROZEN=1 # Do not modify lock files
|
||||
|
||||
set +x
|
||||
PYTHON_IMPL=$(uv run --frozen python -c "import platform; print(platform.python_implementation())")
|
||||
|
||||
# Try to source local Drivers Secrets
|
||||
if [ -f ./secrets-export.sh ]; then
|
||||
echo "Sourcing secrets"
|
||||
source ./secrets-export.sh
|
||||
# Try to source the env file.
|
||||
if [ -f $SCRIPT_DIR/scripts/env.sh ]; then
|
||||
echo "Sourcing env inputs"
|
||||
. $SCRIPT_DIR/scripts/env.sh
|
||||
else
|
||||
echo "Not sourcing secrets"
|
||||
echo "Not sourcing env inputs"
|
||||
fi
|
||||
|
||||
# Start compiling the args we'll pass to uv.
|
||||
# Run in an isolated environment so as not to pollute the base venv.
|
||||
UV_ARGS=("--isolated --frozen --extra test")
|
||||
# Ensure there are test inputs.
|
||||
if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then
|
||||
echo "Sourcing test inputs"
|
||||
. $SCRIPT_DIR/scripts/test-env.sh
|
||||
else
|
||||
echo "Missing test inputs, please run 'just setup-test'"
|
||||
fi
|
||||
|
||||
# Source the local secrets export file if available.
|
||||
if [ -f "$ROOT_DIR/secrets-export.sh" ]; then
|
||||
. "$ROOT_DIR/secrets-export.sh"
|
||||
fi
|
||||
|
||||
PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())")
|
||||
|
||||
# Ensure C extensions if applicable.
|
||||
if [ -z "${NO_EXT:-}" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
|
||||
uv run --frozen tools/fail_if_no_c.py
|
||||
fi
|
||||
|
||||
if [ "$AUTH" != "noauth" ]; then
|
||||
if [ -n "$TEST_DATA_LAKE" ]; then
|
||||
export DB_USER="mhuser"
|
||||
export DB_PASSWORD="pencil"
|
||||
elif [ -n "$TEST_SERVERLESS" ]; then
|
||||
source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh
|
||||
export DB_USER=$SERVERLESS_ATLAS_USER
|
||||
export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
|
||||
export MONGODB_URI="$SERVERLESS_URI"
|
||||
echo "MONGODB_URI=$MONGODB_URI"
|
||||
export SINGLE_MONGOS_LB_URI=$MONGODB_URI
|
||||
export MULTI_MONGOS_LB_URI=$MONGODB_URI
|
||||
elif [ -n "$TEST_AUTH_OIDC" ]; then
|
||||
export DB_USER=$OIDC_ADMIN_USER
|
||||
export DB_PASSWORD=$OIDC_ADMIN_PWD
|
||||
export DB_IP="$MONGODB_URI"
|
||||
else
|
||||
export DB_USER="bob"
|
||||
export DB_PASSWORD="pwd123"
|
||||
fi
|
||||
echo "Added auth, DB_USER: $DB_USER"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_ENTERPRISE_AUTH" ]; then
|
||||
UV_ARGS+=("--extra gssapi")
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
echo "Setting GSSAPI_PASS"
|
||||
export GSSAPI_PASS=${SASL_PASS}
|
||||
export GSSAPI_CANONICALIZE="true"
|
||||
else
|
||||
# BUILD-3830
|
||||
touch krb5.conf.empty
|
||||
export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty
|
||||
|
||||
echo "Writing keytab"
|
||||
echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab
|
||||
echo "Running kinit"
|
||||
kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL}
|
||||
fi
|
||||
echo "Setting GSSAPI variables"
|
||||
export GSSAPI_HOST=${SASL_HOST}
|
||||
export GSSAPI_PORT=${SASL_PORT}
|
||||
export GSSAPI_PRINCIPAL=${PRINCIPAL}
|
||||
|
||||
export TEST_SUITES="auth"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_LOADBALANCER" ]; then
|
||||
export LOAD_BALANCER=1
|
||||
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}"
|
||||
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}"
|
||||
export TEST_SUITES="load_balancer"
|
||||
fi
|
||||
|
||||
if [ "$SSL" != "nossl" ]; then
|
||||
export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
|
||||
export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
|
||||
|
||||
if [ -n "$TEST_LOADBALANCER" ]; then
|
||||
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}&tls=true"
|
||||
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}&tls=true"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$COMPRESSORS" = "snappy" ]; then
|
||||
UV_ARGS+=("--extra snappy")
|
||||
elif [ "$COMPRESSORS" = "zstd" ]; then
|
||||
UV_ARGS+=("--extra zstandard")
|
||||
fi
|
||||
|
||||
# PyOpenSSL test setup.
|
||||
if [ -n "$TEST_PYOPENSSL" ]; then
|
||||
UV_ARGS+=("--extra ocsp")
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
|
||||
# Check for libmongocrypt download.
|
||||
if [ ! -d "libmongocrypt" ]; then
|
||||
echo "Run encryption setup first!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
UV_ARGS+=("--extra encryption")
|
||||
# TODO: Test with 'pip install pymongocrypt'
|
||||
UV_ARGS+=("--group pymongocrypt_source")
|
||||
|
||||
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
|
||||
BASE=$(pwd)/libmongocrypt/nocrypto
|
||||
if [ -f "${BASE}/lib/libmongocrypt.so" ]; then
|
||||
PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so
|
||||
elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then
|
||||
PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib
|
||||
elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then
|
||||
PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll
|
||||
# libmongocrypt's windows dll is not marked executable.
|
||||
chmod +x $PYMONGOCRYPT_LIB
|
||||
PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB)
|
||||
elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then
|
||||
PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so
|
||||
else
|
||||
echo "Cannot find libmongocrypt shared object file"
|
||||
exit 1
|
||||
fi
|
||||
export PYMONGOCRYPT_LIB
|
||||
if [ -n "${PYMONGOCRYPT_LIB:-}" ]; then
|
||||
# Ensure pymongocrypt is working properly.
|
||||
# shellcheck disable=SC2048
|
||||
uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
|
||||
uv run ${UV_ARGS} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
|
||||
# shellcheck disable=SC2048
|
||||
uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
|
||||
uv run ${UV_ARGS} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
|
||||
# PATH is updated by configure-env.sh for access to mongocryptd.
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_ENCRYPTION" ]; then
|
||||
if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then
|
||||
UV_ARGS+=("--extra ocsp")
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_CRYPT_SHARED" ]; then
|
||||
CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH`
|
||||
echo "using crypt_shared_dir $CRYPT_SHARED_DIR"
|
||||
export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:$DYLD_FALLBACK_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:$LD_LIBRARY_PATH
|
||||
export PATH=$CRYPT_SHARED_DIR:$PATH
|
||||
fi
|
||||
# Only run the encryption tests.
|
||||
TEST_SUITES="encryption"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
|
||||
if [[ -z "$SUCCESS" ]]; then
|
||||
echo "Must define SUCCESS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if echo "$MONGODB_URI" | grep -q "@"; then
|
||||
echo "MONGODB_URI unexpectedly contains user credentials in FLE test!";
|
||||
exit 1
|
||||
fi
|
||||
TEST_SUITES="csfle"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
|
||||
source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh
|
||||
export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}"
|
||||
set +x
|
||||
export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
|
||||
set -x
|
||||
TEST_SUITES="index_management"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then
|
||||
TEST_SUITES="data_lake"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_ATLAS" ]; then
|
||||
TEST_SUITES="atlas"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_OCSP" ]; then
|
||||
UV_ARGS+=("--extra ocsp")
|
||||
TEST_SUITES="ocsp"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_AUTH_AWS" ]; then
|
||||
UV_ARGS+=("--extra aws")
|
||||
TEST_SUITES="auth_aws"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_AUTH_OIDC" ]; then
|
||||
UV_ARGS+=("--extra aws")
|
||||
TEST_SUITES="auth_oidc"
|
||||
fi
|
||||
|
||||
if [ -n "$PERF_TEST" ]; then
|
||||
UV_ARGS+=("--group perf")
|
||||
start_time=$(date +%s)
|
||||
TEST_SUITES="perf"
|
||||
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
|
||||
# affects the benchmark results.
|
||||
TEST_ARGS="test/performance/perf_test.py $TEST_ARGS"
|
||||
fi
|
||||
|
||||
echo "Running $AUTH tests over $SSL with python $(uv python find)"
|
||||
uv run --frozen python -c 'import sys; print(sys.version)'
|
||||
|
||||
|
||||
# Run the tests, and store the results in Evergreen compatible XUnit XML
|
||||
# files in the xunit-results/ directory.
|
||||
|
||||
# Run the tests with coverage if requested and coverage is installed.
|
||||
# Only cover CPython. PyPy reports suspiciously low coverage.
|
||||
if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
|
||||
# Keep in sync with combine-coverage.sh.
|
||||
# coverage >=5 is needed for relative_files=true.
|
||||
UV_ARGS+=("--group coverage")
|
||||
TEST_ARGS="$TEST_ARGS --cov"
|
||||
fi
|
||||
|
||||
if [ -n "$GREEN_FRAMEWORK" ]; then
|
||||
UV_ARGS+=("--group $GREEN_FRAMEWORK")
|
||||
fi
|
||||
PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())")
|
||||
echo "Running ${AUTH:-noauth} tests over ${SSL:-nossl} with python $(uv python find)"
|
||||
uv run python -c 'import sys; print(sys.version)'
|
||||
|
||||
# Show the installed packages
|
||||
# shellcheck disable=SC2048
|
||||
PIP_QUIET=0 uv run ${UV_ARGS[*]} --with pip pip list
|
||||
PIP_QUIET=0 uv run ${UV_ARGS} --with pip pip list
|
||||
|
||||
if [ -z "$GREEN_FRAMEWORK" ]; then
|
||||
# Use --capture=tee-sys so pytest prints test output inline:
|
||||
# https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
|
||||
PYTEST_ARGS="-v --capture=tee-sys --durations=5 $TEST_ARGS"
|
||||
if [ -n "$TEST_SUITES" ]; then
|
||||
PYTEST_ARGS="-m $TEST_SUITES $PYTEST_ARGS"
|
||||
fi
|
||||
# shellcheck disable=SC2048
|
||||
uv run ${UV_ARGS[*]} pytest $PYTEST_ARGS
|
||||
else
|
||||
# shellcheck disable=SC2048
|
||||
uv run ${UV_ARGS[*]} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
|
||||
# Record the start time for a perf test.
|
||||
if [ -n "${PERF_TEST:-}" ]; then
|
||||
start_time=$(date +%s)
|
||||
fi
|
||||
|
||||
# Run the tests, and store the results in Evergreen compatible XUnit XML
|
||||
# files in the xunit-results/ directory.
|
||||
TEST_ARGS=${TEST_ARGS}
|
||||
if [ "$#" -ne 0 ]; then
|
||||
TEST_ARGS="$*"
|
||||
fi
|
||||
echo "Running tests with $TEST_ARGS and uv args $UV_ARGS..."
|
||||
if [ -z "${GREEN_FRAMEWORK:-}" ]; then
|
||||
# shellcheck disable=SC2048
|
||||
uv run ${UV_ARGS} pytest $TEST_ARGS
|
||||
else
|
||||
# shellcheck disable=SC2048
|
||||
uv run ${UV_ARGS} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
|
||||
fi
|
||||
echo "Running tests with $TEST_ARGS... done."
|
||||
|
||||
# Handle perf test post actions.
|
||||
if [ -n "$PERF_TEST" ]; then
|
||||
if [ -n "${PERF_TEST:-}" ]; then
|
||||
end_time=$(date +%s)
|
||||
elapsed_secs=$((end_time-start_time))
|
||||
|
||||
@ -289,6 +87,6 @@ if [ -n "$PERF_TEST" ]; then
|
||||
fi
|
||||
|
||||
# Handle coverage post actions.
|
||||
if [ -n "$COVERAGE" ]; then
|
||||
if [ -n "${COVERAGE:-}" ]; then
|
||||
rm -rf .pytest_cache
|
||||
fi
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
mkdir out_dir
|
||||
# shellcheck disable=SC2156
|
||||
find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
|
||||
tar zcvf mongodb-logs.tar.gz -C out_dir/ .
|
||||
rm -rf out_dir
|
||||
@ -16,6 +16,18 @@ DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
|
||||
CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
|
||||
UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
|
||||
UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
|
||||
DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
|
||||
MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
|
||||
|
||||
# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
|
||||
if [ "${CI:-}" == "true" ]; then
|
||||
PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
|
||||
# We want to use a path that's already on PATH on spawn hosts.
|
||||
else
|
||||
PYMONGO_BIN_DIR=$HOME/cli_bin
|
||||
fi
|
||||
|
||||
PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH"
|
||||
|
||||
# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
|
||||
@ -24,6 +36,9 @@ if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
|
||||
CARGO_HOME=$(cygpath -m $CARGO_HOME)
|
||||
UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
|
||||
UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
|
||||
DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES")
|
||||
MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES")
|
||||
PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR")
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
|
||||
@ -36,7 +51,6 @@ fi
|
||||
|
||||
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
|
||||
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
|
||||
export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
|
||||
|
||||
cat <<EOT > "$SCRIPT_DIR"/env.sh
|
||||
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
|
||||
@ -63,16 +77,28 @@ export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
|
||||
export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
|
||||
|
||||
export CARGO_HOME="$CARGO_HOME"
|
||||
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
|
||||
export UV_TOOL_DIR="$UV_TOOL_DIR"
|
||||
export UV_CACHE_DIR="$UV_CACHE_DIR"
|
||||
export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
|
||||
export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH"
|
||||
export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR"
|
||||
export PATH="$PATH_EXT"
|
||||
# shellcheck disable=SC2154
|
||||
export PROJECT="${project:-mongo-python-driver}"
|
||||
export PIP_QUIET=1
|
||||
EOT
|
||||
|
||||
# Write the .env file for drivers-tools.
|
||||
rm -rf $DRIVERS_TOOLS
|
||||
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
|
||||
|
||||
cat <<EOT > ${DRIVERS_TOOLS}/.env
|
||||
SKIP_LEGACY_SHELL=1
|
||||
DRIVERS_TOOLS="$DRIVERS_TOOLS"
|
||||
MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
|
||||
MONGODB_BINARIES="$MONGODB_BINARIES"
|
||||
TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
|
||||
EOT
|
||||
|
||||
# Skip CSOT tests on non-linux platforms.
|
||||
if [ "$(uname -s)" != "Linux" ]; then
|
||||
echo "export SKIP_CSOT_TESTS=1" >> $SCRIPT_DIR/env.sh
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set +x
|
||||
. src/.evergreen/scripts/env.sh
|
||||
# shellcheck disable=SC2044
|
||||
for filename in $(find $DRIVERS_TOOLS -name \*.json); do
|
||||
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
|
||||
done
|
||||
@ -344,11 +344,11 @@ def create_encryption_variants() -> list[BuildVariant]:
|
||||
batchtime = BATCHTIME_WEEK
|
||||
|
||||
def get_encryption_expansions(encryption):
|
||||
expansions = dict(test_encryption="true")
|
||||
expansions = dict(TEST_ENCRYPTION="true")
|
||||
if "crypt_shared" in encryption:
|
||||
expansions["test_crypt_shared"] = "true"
|
||||
expansions["TEST_CRYPT_SHARED"] = "true"
|
||||
if "PyOpenSSL" in encryption:
|
||||
expansions["test_encryption_pyopenssl"] = "true"
|
||||
expansions["TEST_ENCRYPTION_PYOPENSSL"] = "true"
|
||||
return expansions
|
||||
|
||||
host = DEFAULT_HOST
|
||||
@ -487,7 +487,7 @@ def create_enterprise_auth_variants():
|
||||
def create_pyopenssl_variants():
|
||||
base_name = "PyOpenSSL"
|
||||
batchtime = BATCHTIME_WEEK
|
||||
expansions = dict(test_pyopenssl="true")
|
||||
expansions = dict(TEST_PYOPENSSL="true")
|
||||
variants = []
|
||||
|
||||
for python in ALL_PYTHONS:
|
||||
@ -645,7 +645,7 @@ def create_disable_test_commands_variants():
|
||||
def create_serverless_variants():
|
||||
host = DEFAULT_HOST
|
||||
batchtime = BATCHTIME_WEEK
|
||||
expansions = dict(test_serverless="true", AUTH="auth", SSL="ssl")
|
||||
expansions = dict(TEST_SERVERLESS="true", AUTH="auth", SSL="ssl")
|
||||
tasks = ["serverless_task_group"]
|
||||
base_name = "Serverless"
|
||||
return [
|
||||
@ -834,12 +834,9 @@ def create_load_balancer_tasks():
|
||||
tags = ["load-balancer", auth, ssl]
|
||||
bootstrap_vars = dict(TOPOLOGY="sharded_cluster", AUTH=auth, SSL=ssl, LOAD_BALANCER="true")
|
||||
bootstrap_func = FunctionCall(func="bootstrap mongo-orchestration", vars=bootstrap_vars)
|
||||
balancer_func = FunctionCall(func="run load-balancer")
|
||||
test_vars = dict(AUTH=auth, SSL=ssl, test_loadbalancer="true")
|
||||
test_vars = dict(AUTH=auth, SSL=ssl, TEST_LOADBALANCER="true")
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
tasks.append(
|
||||
EvgTask(name=name, tags=tags, commands=[bootstrap_func, balancer_func, test_func])
|
||||
)
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=[bootstrap_func, test_func]))
|
||||
return tasks
|
||||
|
||||
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set +x
|
||||
. src/.evergreen/scripts/env.sh
|
||||
echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json
|
||||
@ -2,23 +2,34 @@
|
||||
|
||||
set -eu
|
||||
|
||||
# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
|
||||
if [ "${CI:-}" == "true" ]; then
|
||||
_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
|
||||
else
|
||||
_BIN_DIR=$HOME/.local/bin
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
pushd "$(dirname "$(dirname $HERE)")" > /dev/null
|
||||
|
||||
# Source the env files to pick up common variables.
|
||||
if [ -f $HERE/env.sh ]; then
|
||||
. $HERE/env.sh
|
||||
fi
|
||||
|
||||
_BIN_DIR=${PYMONGO_BIN_DIR:-$HOME/.local/bin}
|
||||
export PATH="$PATH:${_BIN_DIR}"
|
||||
|
||||
# Helper function to pip install a dependency using a temporary python env.
|
||||
function _pip_install() {
|
||||
_HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
. $_HERE/../utils.sh
|
||||
_VENV_PATH=$(mktemp -d)
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
_VENV_PATH=$(cygpath -m $_VENV_PATH)
|
||||
fi
|
||||
echo "Installing $2 using pip..."
|
||||
createvirtualenv "$(find_python3)" $_VENV_PATH
|
||||
python -m pip install $1
|
||||
ln -s "$(which $2)" $_BIN_DIR/$2
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
ln -s "$(which $2)" $_BIN_DIR/$2.exe
|
||||
else
|
||||
ln -s "$(which $2)" $_BIN_DIR/$2
|
||||
fi
|
||||
echo "Installed to ${_BIN_DIR}"
|
||||
echo "Installing $2 using pip... done."
|
||||
}
|
||||
|
||||
@ -35,9 +46,6 @@ if ! command -v just 2>/dev/null; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $_TARGET --to "$_BIN_DIR" || {
|
||||
_pip_install rust-just just
|
||||
}
|
||||
if ! command -v just 2>/dev/null; then
|
||||
export PATH="$PATH:$_BIN_DIR"
|
||||
fi
|
||||
echo "Installing just... done."
|
||||
fi
|
||||
|
||||
@ -48,8 +56,10 @@ if ! command -v uv 2>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {
|
||||
_pip_install uv uv
|
||||
}
|
||||
if ! command -v uv 2>/dev/null; then
|
||||
export PATH="$PATH:$_BIN_DIR"
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
|
||||
fi
|
||||
echo "Installing uv... done."
|
||||
fi
|
||||
|
||||
popd > /dev/null
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set +x
|
||||
. src/.evergreen/scripts/env.sh
|
||||
# shellcheck disable=SC2044
|
||||
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
|
||||
chmod +x "$i"
|
||||
done
|
||||
@ -5,10 +5,6 @@ HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
pushd $HERE
|
||||
. env.sh
|
||||
|
||||
rm -rf $DRIVERS_TOOLS
|
||||
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
|
||||
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config
|
||||
|
||||
popd
|
||||
|
||||
# Copy PyMongo's test certificates over driver-evergreen-tools'
|
||||
|
||||
@ -4,4 +4,5 @@
|
||||
set +x
|
||||
set -o errexit
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
|
||||
TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
|
||||
TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test
|
||||
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
|
||||
|
||||
@ -5,4 +5,5 @@ set -eu
|
||||
set +x
|
||||
# Use the default python to bootstrap secrets.
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth
|
||||
TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
|
||||
TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test
|
||||
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
. .evergreen/scripts/env.sh
|
||||
set -eu
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
. $HERE/env.sh
|
||||
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
|
||||
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
|
||||
SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
|
||||
SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg
|
||||
SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh
|
||||
bash ./.evergreen/just.sh test-eg
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start
|
||||
@ -24,10 +24,5 @@ echo "Running MONGODB-AWS authentication tests for $1"
|
||||
# Handle credentials and environment setup.
|
||||
. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
|
||||
|
||||
# show test output
|
||||
set -x
|
||||
|
||||
export TEST_AUTH_AWS=1
|
||||
export AUTH="auth"
|
||||
export SET_XTRACE_ON=1
|
||||
TEST_AUTH_AWS=1 AUTH="auth" bash ./.evergreen/just.sh setup-test
|
||||
bash ./.evergreen/just.sh test-eg
|
||||
|
||||
@ -4,5 +4,6 @@ TEST_OCSP=1 \
|
||||
PYTHON_BINARY="${PYTHON_BINARY}" \
|
||||
CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
|
||||
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
|
||||
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test
|
||||
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh
|
||||
|
||||
@ -1,54 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Disable xtrace
|
||||
set +x
|
||||
if [ -n "${MONGODB_STARTED}" ]; then
|
||||
export PYMONGO_MUST_CONNECT=true
|
||||
fi
|
||||
if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
|
||||
export PYMONGO_DISABLE_TEST_COMMANDS=1
|
||||
fi
|
||||
if [ -n "${test_encryption}" ]; then
|
||||
# Disable xtrace (just in case it was accidentally set).
|
||||
set +x
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh
|
||||
export TEST_ENCRYPTION=1
|
||||
if [ -n "${test_encryption_pyopenssl}" ]; then
|
||||
export TEST_ENCRYPTION_PYOPENSSL=1
|
||||
fi
|
||||
fi
|
||||
if [ -n "${test_crypt_shared}" ]; then
|
||||
export TEST_CRYPT_SHARED=1
|
||||
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
|
||||
fi
|
||||
if [ -n "${test_pyopenssl}" ]; then
|
||||
export TEST_PYOPENSSL=1
|
||||
fi
|
||||
if [ -n "${SETDEFAULTENCODING}" ]; then
|
||||
export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
|
||||
fi
|
||||
if [ -n "${test_loadbalancer}" ]; then
|
||||
export TEST_LOADBALANCER=1
|
||||
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
|
||||
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
|
||||
fi
|
||||
if [ -n "${test_serverless}" ]; then
|
||||
export TEST_SERVERLESS=1
|
||||
fi
|
||||
if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
|
||||
export TEST_INDEX_MANAGEMENT=1
|
||||
fi
|
||||
if [ -n "${SKIP_CSOT_TESTS}" ]; then
|
||||
export SKIP_CSOT_TESTS=1
|
||||
fi
|
||||
GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
|
||||
PYTHON_BINARY=${PYTHON_BINARY} \
|
||||
NO_EXT=${NO_EXT} \
|
||||
COVERAGE=${COVERAGE} \
|
||||
COMPRESSORS=${COMPRESSORS} \
|
||||
AUTH=${AUTH} \
|
||||
SSL=${SSL} \
|
||||
TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \
|
||||
TEST_SUITES=${TEST_SUITES:-} \
|
||||
MONGODB_API_VERSION=${MONGODB_API_VERSION} \
|
||||
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
|
||||
@ -1,18 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eu
|
||||
set -eux
|
||||
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
pushd "$(dirname "$(dirname $HERE)")" > /dev/null
|
||||
|
||||
# Source the env file to pick up common variables.
|
||||
# Source the env files to pick up common variables.
|
||||
if [ -f $HERE/env.sh ]; then
|
||||
source $HERE/env.sh
|
||||
. $HERE/env.sh
|
||||
fi
|
||||
# PYTHON_BINARY may be defined in test-env.sh.
|
||||
if [ -f $HERE/test-env.sh ]; then
|
||||
. $HERE/test-env.sh
|
||||
fi
|
||||
|
||||
# Ensure dependencies are installed.
|
||||
. $HERE/install-dependencies.sh
|
||||
|
||||
bash $HERE/install-dependencies.sh
|
||||
|
||||
# Set the location of the python bin dir.
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
@ -32,6 +35,12 @@ if [ ! -d $BIN_DIR ]; then
|
||||
echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
|
||||
echo "Using python $UV_PYTHON"
|
||||
fi
|
||||
|
||||
# Add the default install path to the path if needed.
|
||||
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
|
||||
export PATH="$PATH:$HOME/.local/bin"
|
||||
fi
|
||||
|
||||
uv sync --frozen
|
||||
uv run --frozen --with pip pip install -e .
|
||||
echo "Setting up python environment... done."
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ -n "${test_encryption}" ]; then
|
||||
bash .evergreen/setup-encryption.sh
|
||||
fi
|
||||
@ -2,11 +2,6 @@
|
||||
set -o errexit # Exit the script with error if any of the commands fail
|
||||
set -o xtrace
|
||||
|
||||
if [ -z "${DRIVERS_TOOLS}" ]; then
|
||||
echo "Missing environment variable DRIVERS_TOOLS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET=""
|
||||
|
||||
if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
|
||||
@ -51,10 +46,7 @@ tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
|
||||
ls -la libmongocrypt
|
||||
ls -la libmongocrypt/nocrypto
|
||||
|
||||
if [ -z "${SKIP_SERVERS:-}" ]; then
|
||||
PYTHON_BINARY_OLD=${PYTHON_BINARY}
|
||||
export PYTHON_BINARY=""
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/setup-secrets.sh
|
||||
export PYTHON_BINARY=$PYTHON_BINARY_OLD
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/start-servers.sh
|
||||
if [ "Windows_NT" = "${OS:-''}" ]; then
|
||||
# libmongocrypt's windows dll is not marked executable.
|
||||
chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll
|
||||
fi
|
||||
299
.evergreen/scripts/setup-tests.py
Normal file
299
.evergreen/scripts/setup-tests.py
Normal file
@ -0,0 +1,299 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
HERE = Path(__file__).absolute().parent
|
||||
ROOT = HERE.parent.parent
|
||||
ENV_FILE = HERE / "test-env.sh"
|
||||
DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
EXPECTED_VARS = [
|
||||
"TEST_ENCRYPTION",
|
||||
"TEST_ENCRYPTION_PYOPENSSL",
|
||||
"TEST_CRYPT_SHARED",
|
||||
"TEST_PYOPENSSL",
|
||||
"TEST_LOAD_BALANCER",
|
||||
"TEST_SERVERLESS",
|
||||
"TEST_INDEX_MANAGEMENT",
|
||||
"TEST_ENTERPRISE_AUTH",
|
||||
"TEST_FLE_AZURE_AUTO",
|
||||
"TEST_FLE_GCP_AUTO",
|
||||
"TEST_LOADBALANCER",
|
||||
"TEST_DATA_LAKE",
|
||||
"TEST_ATLAS",
|
||||
"TEST_OCSP",
|
||||
"TEST_AUTH_AWS",
|
||||
"TEST_AUTH_OIDC",
|
||||
"COMPRESSORS",
|
||||
"MONGODB_URI",
|
||||
"PERF_TEST",
|
||||
"GREEN_FRAMEWORK",
|
||||
"PYTHON_BINARY",
|
||||
"LIBMONGOCRYPT_URL",
|
||||
]
|
||||
|
||||
# Handle the test suite based on the presence of env variables.
|
||||
TEST_SUITE_MAP = dict(
|
||||
TEST_DATA_LAKE="data_lake",
|
||||
TEST_AUTH_OIDC="auth_oidc",
|
||||
TEST_INDEX_MANAGEMENT="index_management",
|
||||
TEST_ENTERPRISE_AUTH="auth",
|
||||
TEST_LOADBALANCER="load_balancer",
|
||||
TEST_ENCRYPTION="encryption",
|
||||
TEST_FLE_AZURE_AUTO="csfle",
|
||||
TEST_FLE_GCP_AUTO="csfle",
|
||||
TEST_ATLAS="atlas",
|
||||
TEST_OCSP="ocsp",
|
||||
TEST_AUTH_AWS="auth_aws",
|
||||
PERF_TEST="perf",
|
||||
)
|
||||
|
||||
# Handle extras based on the presence of env variables.
|
||||
EXTRAS_MAP = dict(
|
||||
TEST_AUTH_OIDC="aws",
|
||||
TEST_AUTH_AWS="aws",
|
||||
TEST_OCSP="ocsp",
|
||||
TEST_PYOPENSSL="ocsp",
|
||||
TEST_ENTERPRISE_AUTH="gssapi",
|
||||
TEST_ENCRYPTION="encryption",
|
||||
TEST_FLE_AZURE_AUTO="encryption",
|
||||
TEST_FLE_GCP_AUTO="encryption",
|
||||
TEST_ENCRYPTION_PYOPENSSL="ocsp",
|
||||
)
|
||||
|
||||
|
||||
def write_env(name: str, value: Any) -> None:
|
||||
with ENV_FILE.open("a", newline="\n") as fid:
|
||||
# Remove any existing quote chars.
|
||||
value = str(value).replace('"', "")
|
||||
fid.write(f'export {name}="{value}"\n')
|
||||
|
||||
|
||||
def is_set(var: str) -> bool:
|
||||
value = os.environ.get(var, "")
|
||||
return len(value.strip()) > 0
|
||||
|
||||
|
||||
def run_command(cmd: str) -> None:
|
||||
LOGGER.info("Running command %s...", cmd)
|
||||
subprocess.check_call(shlex.split(cmd)) # noqa: S603
|
||||
LOGGER.info("Running command %s... done.", cmd)
|
||||
|
||||
|
||||
def handle_test_env() -> None:
|
||||
AUTH = os.environ.get("AUTH", "noauth")
|
||||
SSL = os.environ.get("SSL", "nossl")
|
||||
TEST_SUITES = os.environ.get("TEST_SUITES", "")
|
||||
TEST_ARGS = ""
|
||||
# Start compiling the args we'll pass to uv.
|
||||
# Run in an isolated environment so as not to pollute the base venv.
|
||||
UV_ARGS = ["--isolated --extra test"]
|
||||
|
||||
# Save variables in EXPECTED_VARS that have values.
|
||||
with ENV_FILE.open("w", newline="\n") as fid:
|
||||
fid.write("#!/usr/bin/env bash\n")
|
||||
fid.write("set +x\n")
|
||||
fid.write(f"export AUTH={AUTH}\n")
|
||||
fid.write(f"export SSL={SSL}\n")
|
||||
for var in EXPECTED_VARS:
|
||||
value = os.environ.get(var, "")
|
||||
# Remove any existing quote chars.
|
||||
value = value.replace('"', "")
|
||||
if value:
|
||||
fid.write(f'export {var}="{value}"\n')
|
||||
ENV_FILE.chmod(ENV_FILE.stat().st_mode | stat.S_IEXEC)
|
||||
|
||||
for env_var, extra in EXTRAS_MAP.items():
|
||||
if env_var in os.environ:
|
||||
UV_ARGS.append(f"--extra {extra}")
|
||||
|
||||
for env_var, suite in TEST_SUITE_MAP.items():
|
||||
if TEST_SUITES:
|
||||
break
|
||||
if env_var in os.environ:
|
||||
TEST_SUITES = suite
|
||||
|
||||
if AUTH != "noauth":
|
||||
if is_set("TEST_DATA_LAKE"):
|
||||
DB_USER = os.environ["ADL_USERNAME"]
|
||||
DB_PASSWORD = os.environ["ADL_PASSWORD"]
|
||||
elif is_set("TEST_SERVERLESS"):
|
||||
DB_USER = os.environ("SERVERLESS_ATLAS_USER")
|
||||
DB_PASSWORD = os.environ("SERVERLESS_ATLAS_PASSWORD")
|
||||
write_env("MONGODB_URI", os.environ("SERVERLESS_URI"))
|
||||
write_env("SINGLE_MONGOS_LB_URI", os.environ("SERVERLESS_URI"))
|
||||
write_env("MULTI_MONGOS_LB_URI", os.environ("SERVERLESS_URI"))
|
||||
elif is_set("TEST_AUTH_OIDC"):
|
||||
DB_USER = os.environ["OIDC_ADMIN_USER"]
|
||||
DB_PASSWORD = os.environ["OIDC_ADMIN_PWD"]
|
||||
write_env("DB_IP", os.environ["MONGODB_URI"])
|
||||
elif is_set("TEST_INDEX_MANAGEMENT"):
|
||||
DB_USER = os.environ["DRIVERS_ATLAS_LAMBDA_USER"]
|
||||
DB_PASSWORD = os.environ["DRIVERS_ATLAS_LAMBDA_PASSWORD"]
|
||||
else:
|
||||
DB_USER = "bob"
|
||||
DB_PASSWORD = "pwd123" # noqa: S105
|
||||
write_env("DB_USER", DB_USER)
|
||||
write_env("DB_PASSWORD", DB_PASSWORD)
|
||||
LOGGER.info("Added auth, DB_USER: %s", DB_USER)
|
||||
|
||||
if is_set("MONGODB_STARTED"):
|
||||
write_env("PYMONGO_MUST_CONNECT", "true")
|
||||
|
||||
if is_set("DISABLE_TEST_COMMANDS"):
|
||||
write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
|
||||
|
||||
if is_set("TEST_ENTERPRISE_AUTH"):
|
||||
if os.name == "nt":
|
||||
LOGGER.info("Setting GSSAPI_PASS")
|
||||
write_env("GSSAPI_PASS", os.environ["SASL_PASS"])
|
||||
write_env("GSSAPI_CANONICALIZE", "true")
|
||||
else:
|
||||
# BUILD-3830
|
||||
krb_conf = ROOT / ".evergreen/krb5.conf.empty"
|
||||
krb_conf.touch()
|
||||
write_env("KRB5_CONFIG", krb_conf)
|
||||
LOGGER.info("Writing keytab")
|
||||
keytab = base64.b64decode(os.environ["KEYTAB_BASE64"])
|
||||
keytab_file = ROOT / ".evergreen/drivers.keytab"
|
||||
with keytab_file.open("wb") as fid:
|
||||
fid.write(keytab)
|
||||
principal = os.environ["PRINCIPAL"]
|
||||
LOGGER.info("Running kinit")
|
||||
os.environ["KRB5_CONFIG"] = str(krb_conf)
|
||||
cmd = f"kinit -k -t {keytab_file} -p {principal}"
|
||||
run_command(cmd)
|
||||
|
||||
LOGGER.info("Setting GSSAPI variables")
|
||||
write_env("GSSAPI_HOST", os.environ["SASL_HOST"])
|
||||
write_env("GSSAPI_PORT", os.environ["SASL_PORT"])
|
||||
write_env("GSSAPI_PRINCIPAL", os.environ["PRINCIPAL"])
|
||||
|
||||
if is_set("TEST_LOADBALANCER"):
|
||||
write_env("LOAD_BALANCER", "1")
|
||||
SINGLE_MONGOS_LB_URI = os.environ.get(
|
||||
"SINGLE_MONGOS_LB_URI", "mongodb://127.0.0.1:8000/?loadBalanced=true"
|
||||
)
|
||||
MULTI_MONGOS_LB_URI = os.environ.get(
|
||||
"MULTI_MONGOS_LB_URI", "mongodb://127.0.0.1:8001/?loadBalanced=true"
|
||||
)
|
||||
if SSL != "nossl":
|
||||
SINGLE_MONGOS_LB_URI += "&tls=true"
|
||||
MULTI_MONGOS_LB_URI += "&tls=true"
|
||||
write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI)
|
||||
write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI)
|
||||
if not DRIVERS_TOOLS:
|
||||
raise RuntimeError("Missing DRIVERS_TOOLS")
|
||||
cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start'
|
||||
run_command(cmd)
|
||||
|
||||
if SSL != "nossl":
|
||||
if not DRIVERS_TOOLS:
|
||||
raise RuntimeError("Missing DRIVERS_TOOLS")
|
||||
write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
|
||||
write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
|
||||
|
||||
compressors = os.environ.get("COMPRESSORS")
|
||||
if compressors == "snappy":
|
||||
UV_ARGS.append("--extra snappy")
|
||||
elif compressors == "zstd":
|
||||
UV_ARGS.append("--extra zstandard")
|
||||
|
||||
if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
|
||||
# Check for libmongocrypt download.
|
||||
if not (ROOT / "libmongocrypt").exists():
|
||||
run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh")
|
||||
|
||||
# TODO: Test with 'pip install pymongocrypt'
|
||||
UV_ARGS.append("--group pymongocrypt_source")
|
||||
|
||||
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
|
||||
BASE = ROOT / "libmongocrypt/nocrypto"
|
||||
if sys.platform == "linux":
|
||||
if (BASE / "lib/libmongocrypt.so").exists():
|
||||
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so"
|
||||
else:
|
||||
PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so"
|
||||
elif sys.platform == "darwin":
|
||||
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib"
|
||||
else:
|
||||
PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll"
|
||||
if not PYMONGOCRYPT_LIB.exists():
|
||||
raise RuntimeError("Cannot find libmongocrypt shared object file")
|
||||
write_env("PYMONGOCRYPT_LIB", PYMONGOCRYPT_LIB.as_posix())
|
||||
# PATH is updated by configure-env.sh for access to mongocryptd.
|
||||
|
||||
if is_set("TEST_ENCRYPTION"):
|
||||
if not DRIVERS_TOOLS:
|
||||
raise RuntimeError("Missing DRIVERS_TOOLS")
|
||||
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh")
|
||||
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh")
|
||||
|
||||
if is_set("TEST_CRYPT_SHARED"):
|
||||
CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
|
||||
LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
|
||||
if os.name == "nt":
|
||||
write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH")
|
||||
else:
|
||||
write_env(
|
||||
"DYLD_FALLBACK_LIBRARY_PATH",
|
||||
f"{CRYPT_SHARED_DIR}:${{DYLD_FALLBACK_LIBRARY_PATH:-}}",
|
||||
)
|
||||
write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}")
|
||||
|
||||
if is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
|
||||
if "SUCCESS" not in os.environ:
|
||||
raise RuntimeError("Must define SUCCESS")
|
||||
|
||||
write_env("SUCCESS", os.environ["SUCCESS"])
|
||||
MONGODB_URI = os.environ.get("MONGODB_URI", "")
|
||||
if "@" in MONGODB_URI:
|
||||
raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!")
|
||||
|
||||
if is_set("TEST_OCSP"):
|
||||
write_env("CA_FILE", os.environ["CA_FILE"])
|
||||
write_env("OCSP_TLS_SHOULD_SUCCEED", os.environ["OCSP_TLS_SHOULD_SUCCEED"])
|
||||
|
||||
if is_set("PERF_TEST"):
|
||||
UV_ARGS.append("--group perf")
|
||||
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
|
||||
# affects the benchmark results.
|
||||
TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}"
|
||||
|
||||
# Add coverage if requested.
|
||||
# Only cover CPython. PyPy reports suspiciously low coverage.
|
||||
if is_set("COVERAGE") and platform.python_implementation() == "CPython":
|
||||
# Keep in sync with combine-coverage.sh.
|
||||
# coverage >=5 is needed for relative_files=true.
|
||||
UV_ARGS.append("--group coverage")
|
||||
TEST_ARGS = f"{TEST_ARGS} --cov"
|
||||
|
||||
if is_set("GREEN_FRAMEWORK"):
|
||||
framework = os.environ["GREEN_FRAMEWORK"]
|
||||
UV_ARGS.append(f"--group {framework}")
|
||||
|
||||
else:
|
||||
# Use --capture=tee-sys so pytest prints test output inline:
|
||||
# https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
|
||||
TEST_ARGS = f"-v --capture=tee-sys --durations=5 {TEST_ARGS}"
|
||||
if TEST_SUITES:
|
||||
TEST_ARGS = f"-m {TEST_SUITES} {TEST_ARGS}"
|
||||
|
||||
write_env("TEST_ARGS", TEST_ARGS)
|
||||
write_env("UV_ARGS", " ".join(UV_ARGS))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
handle_test_env()
|
||||
@ -1,27 +1,61 @@
|
||||
#!/bin/bash -eux
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
|
||||
PROJECT_DIRECTORY="$(pwd)"
|
||||
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
|
||||
# Supported/used environment variables:
|
||||
# AUTH Set to enable authentication. Defaults to "noauth"
|
||||
# SSL Set to enable SSL. Defaults to "nossl"
|
||||
# GREEN_FRAMEWORK The green framework to test with, if any.
|
||||
# COVERAGE If non-empty, run the test suite with coverage.
|
||||
# COMPRESSORS If non-empty, install appropriate compressor.
|
||||
# LIBMONGOCRYPT_URL The URL to download libmongocrypt.
|
||||
# TEST_DATA_LAKE If non-empty, run data lake tests.
|
||||
# TEST_ENCRYPTION If non-empty, run encryption tests.
|
||||
# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib.
|
||||
# TEST_SERVERLESS If non-empy, test on serverless.
|
||||
# TEST_LOADBALANCER If non-empy, test load balancing.
|
||||
# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure
|
||||
# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP
|
||||
# TEST_PYOPENSSL If non-empy, test with PyOpenSSL
|
||||
# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth
|
||||
# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism
|
||||
# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism
|
||||
# TEST_PERF If non-empty, run performance tests
|
||||
# TEST_OCSP If non-empty, run OCSP tests
|
||||
# TEST_ATLAS If non-empty, test Atlas connections
|
||||
# TEST_INDEX_MANAGEMENT If non-empty, run index management tests
|
||||
# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL
|
||||
# PERF_TEST If non-empty, run the performance tests.
|
||||
# MONGODB_URI If non-empty, use as the MONGODB_URI in tests.
|
||||
# PYTHON_BINARY The python binary to use in tests.
|
||||
|
||||
if [ -f "$SCRIPT_DIR/test-env.sh" ]; then
|
||||
echo "Reading $SCRIPT_DIR/test-env.sh file"
|
||||
. "$SCRIPT_DIR/test-env.sh"
|
||||
exit 0
|
||||
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
|
||||
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
|
||||
|
||||
# Try to source the env file.
|
||||
if [ -f $SCRIPT_DIR/env.sh ]; then
|
||||
source $SCRIPT_DIR/env.sh
|
||||
fi
|
||||
|
||||
cat <<EOT > "$SCRIPT_DIR"/test-env.sh
|
||||
export test_encryption="${test_encryption:-}"
|
||||
export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}"
|
||||
export test_crypt_shared="${test_crypt_shared:-}"
|
||||
export test_pyopenssl="${test_pyopenssl:-}"
|
||||
export test_loadbalancer="${test_loadbalancer:-}"
|
||||
export test_serverless="${test_serverless:-}"
|
||||
export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}"
|
||||
export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}"
|
||||
export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}"
|
||||
export AUTH="${AUTH:-noauth}"
|
||||
export SSL="${SSL:-nossl}"
|
||||
export PYTHON_BINARY="${PYTHON_BINARY:-}"
|
||||
EOT
|
||||
# Source serverless secrets if applicable.
|
||||
if [ -n "${TEST_SERVERLESS:-}" ]; then
|
||||
source $DRIVERS_TOOLS/.evergreen/serverless/secrets-export.sh
|
||||
fi
|
||||
|
||||
chmod +x "$SCRIPT_DIR"/test-env.sh
|
||||
# Source atlas secrets if applicable.
|
||||
if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
|
||||
source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh
|
||||
fi
|
||||
|
||||
# Source ADL secrets if applicable.
|
||||
if [ -n "${TEST_DATA_LAKE:-}" ]; then
|
||||
source ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh
|
||||
fi
|
||||
|
||||
# Source local secrets if applicable.
|
||||
if [ -f "$ROOT_DIR/secrets-export.sh" ]; then
|
||||
source "$ROOT_DIR/secrets-export.sh"
|
||||
fi
|
||||
|
||||
. $ROOT_DIR/.evergreen/utils.sh
|
||||
PYTHON=${PYTHON_BINARY:-$(find_python3)}
|
||||
$PYTHON $SCRIPT_DIR/setup-tests.py
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd "${DRIVERS_TOOLS}"/.evergreen || exit
|
||||
DRIVERS_TOOLS=${DRIVERS_TOOLS}
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop
|
||||
29
.evergreen/scripts/teardown-tests.sh
Executable file
29
.evergreen/scripts/teardown-tests.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
|
||||
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
|
||||
|
||||
# Remove temporary test files.
|
||||
pushd $ROOT_DIR > /dev/null
|
||||
rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid > /dev/null
|
||||
popd > /dev/null
|
||||
|
||||
if [ ! -f $SCRIPT_DIR/test-env.sh ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ -f $SCRIPT_DIR/env.sh ]; then
|
||||
source $SCRIPT_DIR/env.sh
|
||||
fi
|
||||
|
||||
source $SCRIPT_DIR/test-env.sh
|
||||
|
||||
# Shut down csfle servers if applicable
|
||||
if [ -n "${TEST_ENCRYPTION:-}" ]; then
|
||||
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
|
||||
fi
|
||||
|
||||
# Shut down load balancer if applicable.
|
||||
if [ -n "${TEST_LOADBALANCER:-}" ]; then
|
||||
bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop
|
||||
fi
|
||||
@ -1,11 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set +x
|
||||
. src/.evergreen/scripts/env.sh
|
||||
# shellcheck disable=SC2044
|
||||
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
|
||||
< "$i" tr -d '\r' >"$i".new
|
||||
mv "$i".new "$i"
|
||||
done
|
||||
# Copy client certificate because symlinks do not work on Windows.
|
||||
cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem
|
||||
@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_
|
||||
echo "Copying files to $target... done"
|
||||
|
||||
ssh $target $remote_dir/.evergreen/scripts/setup-system.sh
|
||||
ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} just install"
|
||||
ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} .evergreen/scripts/setup-dev-env.sh"
|
||||
|
||||
@ -7,9 +7,12 @@ fi
|
||||
|
||||
target=$1
|
||||
user=${target%@*}
|
||||
remote_dir=/home/$user/mongo-python-driver
|
||||
|
||||
echo "Copying files to $target..."
|
||||
rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_dir
|
||||
echo "Copying files to $target... done."
|
||||
echo "Syncing files to $target..."
|
||||
rsync -haz -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver
|
||||
# shellcheck disable=SC2034
|
||||
fswatch -o . | while read f; do rsync -hazv -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver; done
|
||||
echo "Syncing files to $target... done."
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -o errexit # Exit the script with error if any of the commands fail
|
||||
set -o xtrace
|
||||
|
||||
if [ -z "${DRIVERS_TOOLS}" ]; then
|
||||
echo "Missing environment variable DRIVERS_TOOLS"
|
||||
fi
|
||||
|
||||
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
|
||||
rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid
|
||||
98
.github/workflows/test-python.yml
vendored
98
.github/workflows/test-python.yml
vendored
@ -22,13 +22,13 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.9"
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'pyproject.toml'
|
||||
- name: Install just
|
||||
uses: extractions/setup-just@v2
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.9"
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
just install
|
||||
@ -61,42 +61,21 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- if: ${{ matrix.python-version == '3.13t' }}
|
||||
name: Setup free-threaded Python
|
||||
uses: deadsnakes/action@v3.2.0
|
||||
with:
|
||||
python-version: 3.13
|
||||
nogil: true
|
||||
- if: ${{ matrix.python-version != '3.13t' }}
|
||||
name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'pyproject.toml'
|
||||
allow-prereleases: true
|
||||
- name: Install just
|
||||
uses: extractions/setup-just@v2
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then
|
||||
# Just can't be installed on 3.13t, use pytest directly.
|
||||
pip install .
|
||||
pip install -r requirements/test.txt
|
||||
else
|
||||
just install
|
||||
fi
|
||||
run: just install
|
||||
- name: Start MongoDB
|
||||
uses: supercharge/mongodb-github-action@1.12.0
|
||||
with:
|
||||
mongodb-version: 6.0
|
||||
- name: Run tests
|
||||
run: |
|
||||
if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then
|
||||
pytest -v --durations=5 --maxfail=10
|
||||
else
|
||||
just test
|
||||
fi
|
||||
run: just test
|
||||
|
||||
doctest:
|
||||
runs-on: ubuntu-latest
|
||||
@ -105,24 +84,21 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.9"
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'pyproject.toml'
|
||||
- name: Install just
|
||||
uses: extractions/setup-just@v2
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.9"
|
||||
- name: Start MongoDB
|
||||
uses: supercharge/mongodb-github-action@1.12.0
|
||||
with:
|
||||
mongodb-version: '8.0.0-rc4'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
just install
|
||||
run: just install
|
||||
- name: Run tests
|
||||
run: |
|
||||
just docs-test
|
||||
run: just docs-test
|
||||
|
||||
docs:
|
||||
name: Docs Checks
|
||||
@ -131,20 +107,17 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'pyproject.toml'
|
||||
# Build docs on lowest supported Python for furo
|
||||
python-version: '3.9'
|
||||
enable-cache: true
|
||||
python-version: "3.9"
|
||||
- name: Install just
|
||||
uses: extractions/setup-just@v2
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
just install
|
||||
run: just install
|
||||
- name: Build docs
|
||||
run: |
|
||||
just docs
|
||||
run: just docs
|
||||
|
||||
linkcheck:
|
||||
name: Link Check
|
||||
@ -153,20 +126,17 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'pyproject.toml'
|
||||
# Build docs on lowest supported Python for furo
|
||||
python-version: '3.9'
|
||||
enable-cache: true
|
||||
python-version: "3.9"
|
||||
- name: Install just
|
||||
uses: extractions/setup-just@v2
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
just install
|
||||
run: just install
|
||||
- name: Build docs
|
||||
run: |
|
||||
just docs-linkcheck
|
||||
run: just docs-linkcheck
|
||||
|
||||
typing:
|
||||
name: Typing Tests
|
||||
@ -178,11 +148,11 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "${{matrix.python}}"
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'pyproject.toml'
|
||||
- name: Install just
|
||||
uses: extractions/setup-just@v2
|
||||
- name: Install dependencies
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -26,6 +26,7 @@ libmongocrypt/
|
||||
expansion.yml
|
||||
*expansions.yml
|
||||
.evergreen/scripts/env.sh
|
||||
.evergreen/scripts/test-env.sh
|
||||
|
||||
# Lambda temp files
|
||||
test/lambda/.aws-sam
|
||||
|
||||
@ -211,18 +211,18 @@ the pages will re-render and the browser will automatically refresh.
|
||||
`git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`.
|
||||
- Start the servers using
|
||||
`LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh`.
|
||||
- Start the load balancer using:
|
||||
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`.
|
||||
- Set up the test using:
|
||||
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' TEST_LOADBALANCER=1 just setup-test`.
|
||||
- Run the tests from the `pymongo` checkout directory using:
|
||||
`TEST_LOADBALANCER=1 just test-eg`.
|
||||
`just test-eg`.
|
||||
|
||||
## Running Encryption Tests Locally
|
||||
- Clone `drivers-evergreen-tools`:
|
||||
`git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`.
|
||||
- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`
|
||||
- Run `AWS_PROFILE=<profile> just setup-encryption` after setting up your AWS profile with `aws configure sso`.
|
||||
- Run the tests with `TEST_ENCRYPTION=1 just test-eg`.
|
||||
- When done, run `just teardown-encryption` to clean up.
|
||||
- Run `TEST_ENCRYPTION=1 AWS_PROFILE=<profile> just setup-test` after setting up your AWS profile with `aws configure sso`.
|
||||
- Run the tests with `just test-eg`.
|
||||
- When done, run `just teardown-test` to clean up.
|
||||
|
||||
## Re-sync Spec Tests
|
||||
|
||||
|
||||
14
justfile
14
justfile
@ -1,7 +1,5 @@
|
||||
# See https://just.systems/man/en/ for instructions
|
||||
set shell := ["bash", "-c"]
|
||||
set dotenv-load
|
||||
set dotenv-filename := "./.evergreen/scripts/env.sh"
|
||||
|
||||
# Commonly used command segments.
|
||||
uv_run := "uv run --isolated --frozen "
|
||||
@ -70,10 +68,10 @@ test-mockupdb *args:
|
||||
test-eg *args:
|
||||
bash ./.evergreen/run-tests.sh {{args}}
|
||||
|
||||
[group('encryption')]
|
||||
setup-encryption:
|
||||
bash .evergreen/setup-encryption.sh
|
||||
[group('test')]
|
||||
setup-test:
|
||||
bash .evergreen/scripts/setup-tests.sh
|
||||
|
||||
[group('encryption')]
|
||||
teardown-encryption:
|
||||
bash .evergreen/teardown-encryption.sh
|
||||
[group('test')]
|
||||
teardown-test:
|
||||
bash .evergreen/scripts/teardown-tests.sh
|
||||
|
||||
595
test/asynchronous/test_pooling.py
Normal file
595
test/asynchronous/test_pooling.py
Normal file
@ -0,0 +1,595 @@
|
||||
# Copyright 2009-present MongoDB, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Test built in connection-pooling with threads."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
from bson.codec_options import DEFAULT_CODEC_OPTIONS
|
||||
from bson.son import SON
|
||||
from pymongo import AsyncMongoClient, message, timeout
|
||||
from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError
|
||||
from pymongo.hello import HelloCompat
|
||||
from pymongo.lock import _async_create_lock
|
||||
|
||||
sys.path[0:0] = [""]
|
||||
|
||||
from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest
|
||||
from test.asynchronous.helpers import ConcurrentRunner
|
||||
from test.utils import async_get_pool, async_joinall, delay
|
||||
|
||||
from pymongo.asynchronous.pool import Pool, PoolOptions
|
||||
from pymongo.socket_checker import SocketChecker
|
||||
|
||||
_IS_SYNC = False
|
||||
|
||||
|
||||
N = 10
|
||||
DB = "pymongo-pooling-tests"
|
||||
|
||||
|
||||
async def gc_collect_until_done(tasks, timeout=60):
|
||||
start = time.time()
|
||||
running = list(tasks)
|
||||
while running:
|
||||
assert (time.time() - start) < timeout, "Tasks timed out"
|
||||
for t in running:
|
||||
await t.join(0.1)
|
||||
if not t.is_alive():
|
||||
running.remove(t)
|
||||
gc.collect()
|
||||
|
||||
|
||||
class MongoTask(ConcurrentRunner):
|
||||
"""A thread/Task that uses a AsyncMongoClient."""
|
||||
|
||||
def __init__(self, client):
|
||||
super().__init__()
|
||||
self.daemon = True # Don't hang whole test if task hangs.
|
||||
self.client = client
|
||||
self.db = self.client[DB]
|
||||
self.passed = False
|
||||
|
||||
async def run(self):
|
||||
await self.run_mongo_thread()
|
||||
self.passed = True
|
||||
|
||||
async def run_mongo_thread(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class InsertOneAndFind(MongoTask):
|
||||
async def run_mongo_thread(self):
|
||||
for _ in range(N):
|
||||
rand = random.randint(0, N)
|
||||
_id = (await self.db.sf.insert_one({"x": rand})).inserted_id
|
||||
assert rand == (await self.db.sf.find_one(_id))["x"]
|
||||
|
||||
|
||||
class Unique(MongoTask):
|
||||
async def run_mongo_thread(self):
|
||||
for _ in range(N):
|
||||
await self.db.unique.insert_one({}) # no error
|
||||
|
||||
|
||||
class NonUnique(MongoTask):
|
||||
async def run_mongo_thread(self):
|
||||
for _ in range(N):
|
||||
try:
|
||||
await self.db.unique.insert_one({"_id": "jesse"})
|
||||
except DuplicateKeyError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("Should have raised DuplicateKeyError")
|
||||
|
||||
|
||||
class SocketGetter(MongoTask):
|
||||
"""Utility for TestPooling.
|
||||
|
||||
Checks out a socket and holds it forever. Used in
|
||||
test_no_wait_queue_timeout.
|
||||
"""
|
||||
|
||||
def __init__(self, client, pool):
|
||||
super().__init__(client)
|
||||
self.state = "init"
|
||||
self.pool = pool
|
||||
self.sock = None
|
||||
|
||||
async def run_mongo_thread(self):
|
||||
self.state = "get_socket"
|
||||
|
||||
# Call 'pin_cursor' so we can hold the socket.
|
||||
async with self.pool.checkout() as sock:
|
||||
sock.pin_cursor()
|
||||
self.sock = sock
|
||||
|
||||
self.state = "connection"
|
||||
|
||||
def __del__(self):
|
||||
if self.sock:
|
||||
self.sock.close_conn(None)
|
||||
|
||||
|
||||
async def run_cases(client, cases):
|
||||
tasks = []
|
||||
n_runs = 5
|
||||
|
||||
for case in cases:
|
||||
for _i in range(n_runs):
|
||||
t = case(client)
|
||||
await t.start()
|
||||
tasks.append(t)
|
||||
|
||||
for t in tasks:
|
||||
await t.join()
|
||||
|
||||
for t in tasks:
|
||||
assert t.passed, "%s.run() threw an exception" % repr(t)
|
||||
|
||||
|
||||
class _TestPoolingBase(AsyncIntegrationTest):
|
||||
"""Base class for all connection-pool tests."""
|
||||
|
||||
@async_client_context.require_connection
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.c = await self.async_rs_or_single_client()
|
||||
db = self.c[DB]
|
||||
await db.unique.drop()
|
||||
await db.test.drop()
|
||||
await db.unique.insert_one({"_id": "jesse"})
|
||||
await db.test.insert_many([{} for _ in range(10)])
|
||||
|
||||
async def create_pool(self, pair=None, *args, **kwargs):
|
||||
if pair is None:
|
||||
pair = (await async_client_context.host, await async_client_context.port)
|
||||
# Start the pool with the correct ssl options.
|
||||
pool_options = async_client_context.client._topology_settings.pool_options
|
||||
kwargs["ssl_context"] = pool_options._ssl_context
|
||||
kwargs["tls_allow_invalid_hostnames"] = pool_options.tls_allow_invalid_hostnames
|
||||
kwargs["server_api"] = pool_options.server_api
|
||||
pool = Pool(pair, PoolOptions(*args, **kwargs))
|
||||
await pool.ready()
|
||||
return pool
|
||||
|
||||
|
||||
class TestPooling(_TestPoolingBase):
|
||||
async def test_max_pool_size_validation(self):
|
||||
host, port = await async_client_context.host, await async_client_context.port
|
||||
self.assertRaises(ValueError, AsyncMongoClient, host=host, port=port, maxPoolSize=-1)
|
||||
|
||||
self.assertRaises(ValueError, AsyncMongoClient, host=host, port=port, maxPoolSize="foo")
|
||||
|
||||
c = AsyncMongoClient(host=host, port=port, maxPoolSize=100, connect=False)
|
||||
self.assertEqual(c.options.pool_options.max_pool_size, 100)
|
||||
|
||||
async def test_no_disconnect(self):
|
||||
await run_cases(self.c, [NonUnique, Unique, InsertOneAndFind])
|
||||
|
||||
async def test_pool_reuses_open_socket(self):
|
||||
# Test Pool's _check_closed() method doesn't close a healthy socket.
|
||||
cx_pool = await self.create_pool(max_pool_size=10)
|
||||
cx_pool._check_interval_seconds = 0 # Always check.
|
||||
async with cx_pool.checkout() as conn:
|
||||
pass
|
||||
|
||||
async with cx_pool.checkout() as new_connection:
|
||||
self.assertEqual(conn, new_connection)
|
||||
|
||||
self.assertEqual(1, len(cx_pool.conns))
|
||||
|
||||
async def test_get_socket_and_exception(self):
|
||||
# get_socket() returns socket after a non-network error.
|
||||
cx_pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=1)
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
async with cx_pool.checkout() as conn:
|
||||
1 / 0
|
||||
|
||||
# Socket was returned, not closed.
|
||||
async with cx_pool.checkout() as new_connection:
|
||||
self.assertEqual(conn, new_connection)
|
||||
|
||||
self.assertEqual(1, len(cx_pool.conns))
|
||||
|
||||
async def test_pool_removes_closed_socket(self):
|
||||
# Test that Pool removes explicitly closed socket.
|
||||
cx_pool = await self.create_pool()
|
||||
|
||||
async with cx_pool.checkout() as conn:
|
||||
# Use Connection's API to close the socket.
|
||||
conn.close_conn(None)
|
||||
|
||||
self.assertEqual(0, len(cx_pool.conns))
|
||||
|
||||
async def test_pool_removes_dead_socket(self):
|
||||
# Test that Pool removes dead socket and the socket doesn't return
|
||||
# itself PYTHON-344
|
||||
cx_pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=1)
|
||||
cx_pool._check_interval_seconds = 0 # Always check.
|
||||
|
||||
async with cx_pool.checkout() as conn:
|
||||
# Simulate a closed socket without telling the Connection it's
|
||||
# closed.
|
||||
conn.conn.close()
|
||||
self.assertTrue(conn.conn_closed())
|
||||
|
||||
async with cx_pool.checkout() as new_connection:
|
||||
self.assertEqual(0, len(cx_pool.conns))
|
||||
self.assertNotEqual(conn, new_connection)
|
||||
|
||||
self.assertEqual(1, len(cx_pool.conns))
|
||||
|
||||
# Semaphore was released.
|
||||
async with cx_pool.checkout():
|
||||
pass
|
||||
|
||||
async def test_socket_closed(self):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.connect((await async_client_context.host, await async_client_context.port))
|
||||
socket_checker = SocketChecker()
|
||||
self.assertFalse(socket_checker.socket_closed(s))
|
||||
s.close()
|
||||
self.assertTrue(socket_checker.socket_closed(s))
|
||||
|
||||
async def test_socket_checker(self):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.connect((await async_client_context.host, await async_client_context.port))
|
||||
socket_checker = SocketChecker()
|
||||
# Socket has nothing to read.
|
||||
self.assertFalse(socket_checker.select(s, read=True))
|
||||
self.assertFalse(socket_checker.select(s, read=True, timeout=0))
|
||||
self.assertFalse(socket_checker.select(s, read=True, timeout=0.05))
|
||||
# Socket is writable.
|
||||
self.assertTrue(socket_checker.select(s, write=True, timeout=None))
|
||||
self.assertTrue(socket_checker.select(s, write=True))
|
||||
self.assertTrue(socket_checker.select(s, write=True, timeout=0))
|
||||
self.assertTrue(socket_checker.select(s, write=True, timeout=0.05))
|
||||
# Make the socket readable
|
||||
_, msg, _ = message._query(
|
||||
0, "admin.$cmd", 0, -1, SON([("ping", 1)]), None, DEFAULT_CODEC_OPTIONS
|
||||
)
|
||||
s.sendall(msg)
|
||||
# Block until the socket is readable.
|
||||
self.assertTrue(socket_checker.select(s, read=True, timeout=None))
|
||||
self.assertTrue(socket_checker.select(s, read=True))
|
||||
self.assertTrue(socket_checker.select(s, read=True, timeout=0))
|
||||
self.assertTrue(socket_checker.select(s, read=True, timeout=0.05))
|
||||
# Socket is still writable.
|
||||
self.assertTrue(socket_checker.select(s, write=True, timeout=None))
|
||||
self.assertTrue(socket_checker.select(s, write=True))
|
||||
self.assertTrue(socket_checker.select(s, write=True, timeout=0))
|
||||
self.assertTrue(socket_checker.select(s, write=True, timeout=0.05))
|
||||
s.close()
|
||||
self.assertTrue(socket_checker.socket_closed(s))
|
||||
|
||||
async def test_return_socket_after_reset(self):
|
||||
pool = await self.create_pool()
|
||||
async with pool.checkout() as sock:
|
||||
self.assertEqual(pool.active_sockets, 1)
|
||||
self.assertEqual(pool.operation_count, 1)
|
||||
await pool.reset()
|
||||
|
||||
self.assertTrue(sock.closed)
|
||||
self.assertEqual(0, len(pool.conns))
|
||||
self.assertEqual(pool.active_sockets, 0)
|
||||
self.assertEqual(pool.operation_count, 0)
|
||||
|
||||
async def test_pool_check(self):
|
||||
# Test that Pool recovers from two connection failures in a row.
|
||||
# This exercises code at the end of Pool._check().
|
||||
cx_pool = await self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1)
|
||||
cx_pool._check_interval_seconds = 0 # Always check.
|
||||
self.addAsyncCleanup(cx_pool.close)
|
||||
|
||||
async with cx_pool.checkout() as conn:
|
||||
# Simulate a closed socket without telling the Connection it's
|
||||
# closed.
|
||||
conn.conn.close()
|
||||
|
||||
# Swap pool's address with a bad one.
|
||||
address, cx_pool.address = cx_pool.address, ("foo.com", 1234)
|
||||
with self.assertRaises(AutoReconnect):
|
||||
async with cx_pool.checkout():
|
||||
pass
|
||||
|
||||
# Back to normal, semaphore was correctly released.
|
||||
cx_pool.address = address
|
||||
async with cx_pool.checkout():
|
||||
pass
|
||||
|
||||
async def test_wait_queue_timeout(self):
|
||||
wait_queue_timeout = 2 # Seconds
|
||||
pool = await self.create_pool(max_pool_size=1, wait_queue_timeout=wait_queue_timeout)
|
||||
self.addAsyncCleanup(pool.close)
|
||||
|
||||
async with pool.checkout():
|
||||
start = time.time()
|
||||
with self.assertRaises(ConnectionFailure):
|
||||
async with pool.checkout():
|
||||
pass
|
||||
|
||||
duration = time.time() - start
|
||||
self.assertTrue(
|
||||
abs(wait_queue_timeout - duration) < 1,
|
||||
f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}",
|
||||
)
|
||||
|
||||
async def test_no_wait_queue_timeout(self):
|
||||
# Verify get_socket() with no wait_queue_timeout blocks forever.
|
||||
pool = await self.create_pool(max_pool_size=1)
|
||||
self.addAsyncCleanup(pool.close)
|
||||
|
||||
# Reach max_size.
|
||||
async with pool.checkout() as s1:
|
||||
t = SocketGetter(self.c, pool)
|
||||
await t.start()
|
||||
while t.state != "get_socket":
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
await asyncio.sleep(1)
|
||||
self.assertEqual(t.state, "get_socket")
|
||||
|
||||
while t.state != "connection":
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
self.assertEqual(t.state, "connection")
|
||||
self.assertEqual(t.sock, s1)
|
||||
|
||||
async def test_checkout_more_than_max_pool_size(self):
|
||||
pool = await self.create_pool(max_pool_size=2)
|
||||
|
||||
socks = []
|
||||
for _ in range(2):
|
||||
# Call 'pin_cursor' so we can hold the socket.
|
||||
async with pool.checkout() as sock:
|
||||
sock.pin_cursor()
|
||||
socks.append(sock)
|
||||
|
||||
tasks = []
|
||||
for _ in range(30):
|
||||
t = SocketGetter(self.c, pool)
|
||||
await t.start()
|
||||
tasks.append(t)
|
||||
await asyncio.sleep(1)
|
||||
for t in tasks:
|
||||
self.assertEqual(t.state, "get_socket")
|
||||
|
||||
for socket_info in socks:
|
||||
socket_info.close_conn(None)
|
||||
|
||||
async def test_maxConnecting(self):
|
||||
client = await self.async_rs_or_single_client()
|
||||
await self.client.test.test.insert_one({})
|
||||
self.addAsyncCleanup(self.client.test.test.delete_many, {})
|
||||
pool = await async_get_pool(client)
|
||||
docs = []
|
||||
|
||||
# Run 50 short running operations
|
||||
async def find_one():
|
||||
docs.append(await client.test.test.find_one({}))
|
||||
|
||||
tasks = [ConcurrentRunner(target=find_one) for _ in range(50)]
|
||||
for task in tasks:
|
||||
await task.start()
|
||||
for task in tasks:
|
||||
await task.join(10)
|
||||
|
||||
self.assertEqual(len(docs), 50)
|
||||
self.assertLessEqual(len(pool.conns), 50)
|
||||
# TLS and auth make connection establishment more expensive than
|
||||
# the query which leads to more threads hitting maxConnecting.
|
||||
# The end result is fewer total connections and better latency.
|
||||
if async_client_context.tls and async_client_context.auth_enabled:
|
||||
self.assertLessEqual(len(pool.conns), 30)
|
||||
else:
|
||||
self.assertLessEqual(len(pool.conns), 50)
|
||||
# MongoDB 4.4.1 with auth + ssl:
|
||||
# maxConnecting = 2: 6 connections in ~0.231+ seconds
|
||||
# maxConnecting = unbounded: 50 connections in ~0.642+ seconds
|
||||
#
|
||||
# MongoDB 4.4.1 with no-auth no-ssl Python 3.8:
|
||||
# maxConnecting = 2: 15-22 connections in ~0.108+ seconds
|
||||
# maxConnecting = unbounded: 30+ connections in ~0.140+ seconds
|
||||
print(len(pool.conns))
|
||||
|
||||
@async_client_context.require_failCommand_appName
|
||||
async def test_csot_timeout_message(self):
|
||||
client = await self.async_rs_or_single_client(appName="connectionTimeoutApp")
|
||||
# Mock an operation failing due to pymongo.timeout().
|
||||
mock_connection_timeout = {
|
||||
"configureFailPoint": "failCommand",
|
||||
"mode": "alwaysOn",
|
||||
"data": {
|
||||
"blockConnection": True,
|
||||
"blockTimeMS": 1000,
|
||||
"failCommands": ["find"],
|
||||
"appName": "connectionTimeoutApp",
|
||||
},
|
||||
}
|
||||
|
||||
await client.db.t.insert_one({"x": 1})
|
||||
|
||||
async with self.fail_point(mock_connection_timeout):
|
||||
with self.assertRaises(Exception) as error:
|
||||
with timeout(0.5):
|
||||
await client.db.t.find_one({"$where": delay(2)})
|
||||
|
||||
self.assertTrue("(configured timeouts: timeoutMS: 500.0ms" in str(error.exception))
|
||||
|
||||
@async_client_context.require_failCommand_appName
|
||||
async def test_socket_timeout_message(self):
|
||||
client = await self.async_rs_or_single_client(
|
||||
socketTimeoutMS=500, appName="connectionTimeoutApp"
|
||||
)
|
||||
# Mock an operation failing due to socketTimeoutMS.
|
||||
mock_connection_timeout = {
|
||||
"configureFailPoint": "failCommand",
|
||||
"mode": "alwaysOn",
|
||||
"data": {
|
||||
"blockConnection": True,
|
||||
"blockTimeMS": 1000,
|
||||
"failCommands": ["find"],
|
||||
"appName": "connectionTimeoutApp",
|
||||
},
|
||||
}
|
||||
|
||||
await client.db.t.insert_one({"x": 1})
|
||||
|
||||
async with self.fail_point(mock_connection_timeout):
|
||||
with self.assertRaises(Exception) as error:
|
||||
await client.db.t.find_one({"$where": delay(2)})
|
||||
|
||||
self.assertTrue(
|
||||
"(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 20000.0ms)"
|
||||
in str(error.exception)
|
||||
)
|
||||
|
||||
@async_client_context.require_failCommand_appName
|
||||
async def test_connection_timeout_message(self):
|
||||
# Mock a connection creation failing due to timeout.
|
||||
mock_connection_timeout = {
|
||||
"configureFailPoint": "failCommand",
|
||||
"mode": "alwaysOn",
|
||||
"data": {
|
||||
"blockConnection": True,
|
||||
"blockTimeMS": 1000,
|
||||
"failCommands": [HelloCompat.LEGACY_CMD, "hello"],
|
||||
"appName": "connectionTimeoutApp",
|
||||
},
|
||||
}
|
||||
|
||||
client = await self.async_rs_or_single_client(
|
||||
connectTimeoutMS=500,
|
||||
socketTimeoutMS=500,
|
||||
appName="connectionTimeoutApp",
|
||||
heartbeatFrequencyMS=1000000,
|
||||
)
|
||||
await client.admin.command("ping")
|
||||
pool = await async_get_pool(client)
|
||||
await pool.reset_without_pause()
|
||||
async with self.fail_point(mock_connection_timeout):
|
||||
with self.assertRaises(Exception) as error:
|
||||
await client.admin.command("ping")
|
||||
|
||||
self.assertTrue(
|
||||
"(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)"
|
||||
in str(error.exception)
|
||||
)
|
||||
|
||||
|
||||
class TestPoolMaxSize(_TestPoolingBase):
|
||||
async def test_max_pool_size(self):
|
||||
max_pool_size = 4
|
||||
c = await self.async_rs_or_single_client(maxPoolSize=max_pool_size)
|
||||
collection = c[DB].test
|
||||
|
||||
# Need one document.
|
||||
await collection.drop()
|
||||
await collection.insert_one({})
|
||||
|
||||
# ntasks had better be much larger than max_pool_size to ensure that
|
||||
# max_pool_size connections are actually required at some point in this
|
||||
# test's execution.
|
||||
cx_pool = await async_get_pool(c)
|
||||
ntasks = 10
|
||||
tasks = []
|
||||
lock = _async_create_lock()
|
||||
self.n_passed = 0
|
||||
|
||||
async def f():
|
||||
for _ in range(5):
|
||||
await collection.find_one({"$where": delay(0.1)})
|
||||
assert len(cx_pool.conns) <= max_pool_size
|
||||
|
||||
async with lock:
|
||||
self.n_passed += 1
|
||||
|
||||
for _i in range(ntasks):
|
||||
t = ConcurrentRunner(target=f)
|
||||
tasks.append(t)
|
||||
await t.start()
|
||||
|
||||
await async_joinall(tasks)
|
||||
self.assertEqual(ntasks, self.n_passed)
|
||||
self.assertTrue(len(cx_pool.conns) > 1)
|
||||
self.assertEqual(0, cx_pool.requests)
|
||||
|
||||
async def test_max_pool_size_none(self):
|
||||
c = await self.async_rs_or_single_client(maxPoolSize=None)
|
||||
collection = c[DB].test
|
||||
|
||||
# Need one document.
|
||||
await collection.drop()
|
||||
await collection.insert_one({})
|
||||
|
||||
cx_pool = await async_get_pool(c)
|
||||
ntasks = 10
|
||||
tasks = []
|
||||
lock = _async_create_lock()
|
||||
self.n_passed = 0
|
||||
|
||||
async def f():
|
||||
for _ in range(5):
|
||||
await collection.find_one({"$where": delay(0.1)})
|
||||
|
||||
async with lock:
|
||||
self.n_passed += 1
|
||||
|
||||
for _i in range(ntasks):
|
||||
t = ConcurrentRunner(target=f)
|
||||
tasks.append(t)
|
||||
await t.start()
|
||||
|
||||
await async_joinall(tasks)
|
||||
self.assertEqual(ntasks, self.n_passed)
|
||||
self.assertTrue(len(cx_pool.conns) > 1)
|
||||
self.assertEqual(cx_pool.max_pool_size, float("inf"))
|
||||
|
||||
async def test_max_pool_size_zero(self):
|
||||
c = await self.async_rs_or_single_client(maxPoolSize=0)
|
||||
pool = await async_get_pool(c)
|
||||
self.assertEqual(pool.max_pool_size, float("inf"))
|
||||
|
||||
async def test_max_pool_size_with_connection_failure(self):
|
||||
# The pool acquires its semaphore before attempting to connect; ensure
|
||||
# it releases the semaphore on connection failure.
|
||||
test_pool = Pool(
|
||||
("somedomainthatdoesntexist.org", 27017),
|
||||
PoolOptions(max_pool_size=1, connect_timeout=1, socket_timeout=1, wait_queue_timeout=1),
|
||||
)
|
||||
await test_pool.ready()
|
||||
|
||||
# First call to get_socket fails; if pool doesn't release its semaphore
|
||||
# then the second call raises "ConnectionFailure: Timed out waiting for
|
||||
# socket from pool" instead of AutoReconnect.
|
||||
for _i in range(2):
|
||||
with self.assertRaises(AutoReconnect) as context:
|
||||
async with test_pool.checkout():
|
||||
pass
|
||||
|
||||
# Testing for AutoReconnect instead of ConnectionFailure, above,
|
||||
# is sufficient right *now* to catch a semaphore leak. But that
|
||||
# seems error-prone, so check the message too.
|
||||
self.assertNotIn("waiting for socket from pool", str(context.exception))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@ -15,11 +15,11 @@
|
||||
"""Test built in connection-pooling with threads."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from bson.codec_options import DEFAULT_CODEC_OPTIONS
|
||||
@ -27,30 +27,29 @@ from bson.son import SON
|
||||
from pymongo import MongoClient, message, timeout
|
||||
from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError
|
||||
from pymongo.hello import HelloCompat
|
||||
from pymongo.lock import _create_lock
|
||||
|
||||
sys.path[0:0] = [""]
|
||||
|
||||
from test import IntegrationTest, client_context, unittest
|
||||
from test.helpers import ConcurrentRunner
|
||||
from test.utils import delay, get_pool, joinall
|
||||
|
||||
from pymongo.socket_checker import SocketChecker
|
||||
from pymongo.synchronous.pool import Pool, PoolOptions
|
||||
|
||||
|
||||
@client_context.require_connection
|
||||
def setUpModule():
|
||||
pass
|
||||
_IS_SYNC = True
|
||||
|
||||
|
||||
N = 10
|
||||
DB = "pymongo-pooling-tests"
|
||||
|
||||
|
||||
def gc_collect_until_done(threads, timeout=60):
|
||||
def gc_collect_until_done(tasks, timeout=60):
|
||||
start = time.time()
|
||||
running = list(threads)
|
||||
running = list(tasks)
|
||||
while running:
|
||||
assert (time.time() - start) < timeout, "Threads timed out"
|
||||
assert (time.time() - start) < timeout, "Tasks timed out"
|
||||
for t in running:
|
||||
t.join(0.1)
|
||||
if not t.is_alive():
|
||||
@ -58,12 +57,12 @@ def gc_collect_until_done(threads, timeout=60):
|
||||
gc.collect()
|
||||
|
||||
|
||||
class MongoThread(threading.Thread):
|
||||
"""A thread that uses a MongoClient."""
|
||||
class MongoTask(ConcurrentRunner):
|
||||
"""A thread/Task that uses a MongoClient."""
|
||||
|
||||
def __init__(self, client):
|
||||
super().__init__()
|
||||
self.daemon = True # Don't hang whole test if thread hangs.
|
||||
self.daemon = True # Don't hang whole test if task hangs.
|
||||
self.client = client
|
||||
self.db = self.client[DB]
|
||||
self.passed = False
|
||||
@ -76,21 +75,21 @@ class MongoThread(threading.Thread):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class InsertOneAndFind(MongoThread):
|
||||
class InsertOneAndFind(MongoTask):
|
||||
def run_mongo_thread(self):
|
||||
for _ in range(N):
|
||||
rand = random.randint(0, N)
|
||||
_id = self.db.sf.insert_one({"x": rand}).inserted_id
|
||||
assert rand == self.db.sf.find_one(_id)["x"]
|
||||
_id = (self.db.sf.insert_one({"x": rand})).inserted_id
|
||||
assert rand == (self.db.sf.find_one(_id))["x"]
|
||||
|
||||
|
||||
class Unique(MongoThread):
|
||||
class Unique(MongoTask):
|
||||
def run_mongo_thread(self):
|
||||
for _ in range(N):
|
||||
self.db.unique.insert_one({}) # no error
|
||||
|
||||
|
||||
class NonUnique(MongoThread):
|
||||
class NonUnique(MongoTask):
|
||||
def run_mongo_thread(self):
|
||||
for _ in range(N):
|
||||
try:
|
||||
@ -101,7 +100,7 @@ class NonUnique(MongoThread):
|
||||
raise AssertionError("Should have raised DuplicateKeyError")
|
||||
|
||||
|
||||
class SocketGetter(MongoThread):
|
||||
class SocketGetter(MongoTask):
|
||||
"""Utility for TestPooling.
|
||||
|
||||
Checks out a socket and holds it forever. Used in
|
||||
@ -130,25 +129,26 @@ class SocketGetter(MongoThread):
|
||||
|
||||
|
||||
def run_cases(client, cases):
|
||||
threads = []
|
||||
tasks = []
|
||||
n_runs = 5
|
||||
|
||||
for case in cases:
|
||||
for _i in range(n_runs):
|
||||
t = case(client)
|
||||
t.start()
|
||||
threads.append(t)
|
||||
tasks.append(t)
|
||||
|
||||
for t in threads:
|
||||
for t in tasks:
|
||||
t.join()
|
||||
|
||||
for t in threads:
|
||||
for t in tasks:
|
||||
assert t.passed, "%s.run() threw an exception" % repr(t)
|
||||
|
||||
|
||||
class _TestPoolingBase(IntegrationTest):
|
||||
"""Base class for all connection-pool tests."""
|
||||
|
||||
@client_context.require_connection
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.c = self.rs_or_single_client()
|
||||
@ -158,11 +158,9 @@ class _TestPoolingBase(IntegrationTest):
|
||||
db.unique.insert_one({"_id": "jesse"})
|
||||
db.test.insert_many([{} for _ in range(10)])
|
||||
|
||||
def tearDown(self):
|
||||
self.c.close()
|
||||
super().tearDown()
|
||||
|
||||
def create_pool(self, pair=(client_context.host, client_context.port), *args, **kwargs):
|
||||
def create_pool(self, pair=None, *args, **kwargs):
|
||||
if pair is None:
|
||||
pair = (client_context.host, client_context.port)
|
||||
# Start the pool with the correct ssl options.
|
||||
pool_options = client_context.client._topology_settings.pool_options
|
||||
kwargs["ssl_context"] = pool_options._ssl_context
|
||||
@ -365,13 +363,13 @@ class TestPooling(_TestPoolingBase):
|
||||
sock.pin_cursor()
|
||||
socks.append(sock)
|
||||
|
||||
threads = []
|
||||
tasks = []
|
||||
for _ in range(30):
|
||||
t = SocketGetter(self.c, pool)
|
||||
t.start()
|
||||
threads.append(t)
|
||||
tasks.append(t)
|
||||
time.sleep(1)
|
||||
for t in threads:
|
||||
for t in tasks:
|
||||
self.assertEqual(t.state, "get_socket")
|
||||
|
||||
for socket_info in socks:
|
||||
@ -379,7 +377,6 @@ class TestPooling(_TestPoolingBase):
|
||||
|
||||
def test_maxConnecting(self):
|
||||
client = self.rs_or_single_client()
|
||||
self.addCleanup(client.close)
|
||||
self.client.test.test.insert_one({})
|
||||
self.addCleanup(self.client.test.test.delete_many, {})
|
||||
pool = get_pool(client)
|
||||
@ -389,11 +386,11 @@ class TestPooling(_TestPoolingBase):
|
||||
def find_one():
|
||||
docs.append(client.test.test.find_one({}))
|
||||
|
||||
threads = [threading.Thread(target=find_one) for _ in range(50)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(10)
|
||||
tasks = [ConcurrentRunner(target=find_one) for _ in range(50)]
|
||||
for task in tasks:
|
||||
task.start()
|
||||
for task in tasks:
|
||||
task.join(10)
|
||||
|
||||
self.assertEqual(len(docs), 50)
|
||||
self.assertLessEqual(len(pool.conns), 50)
|
||||
@ -416,7 +413,6 @@ class TestPooling(_TestPoolingBase):
|
||||
@client_context.require_failCommand_appName
|
||||
def test_csot_timeout_message(self):
|
||||
client = self.rs_or_single_client(appName="connectionTimeoutApp")
|
||||
self.addCleanup(client.close)
|
||||
# Mock an operation failing due to pymongo.timeout().
|
||||
mock_connection_timeout = {
|
||||
"configureFailPoint": "failCommand",
|
||||
@ -441,7 +437,6 @@ class TestPooling(_TestPoolingBase):
|
||||
@client_context.require_failCommand_appName
|
||||
def test_socket_timeout_message(self):
|
||||
client = self.rs_or_single_client(socketTimeoutMS=500, appName="connectionTimeoutApp")
|
||||
self.addCleanup(client.close)
|
||||
# Mock an operation failing due to socketTimeoutMS.
|
||||
mock_connection_timeout = {
|
||||
"configureFailPoint": "failCommand",
|
||||
@ -485,7 +480,6 @@ class TestPooling(_TestPoolingBase):
|
||||
appName="connectionTimeoutApp",
|
||||
heartbeatFrequencyMS=1000000,
|
||||
)
|
||||
self.addCleanup(client.close)
|
||||
client.admin.command("ping")
|
||||
pool = get_pool(client)
|
||||
pool.reset_without_pause()
|
||||
@ -503,20 +497,19 @@ class TestPoolMaxSize(_TestPoolingBase):
|
||||
def test_max_pool_size(self):
|
||||
max_pool_size = 4
|
||||
c = self.rs_or_single_client(maxPoolSize=max_pool_size)
|
||||
self.addCleanup(c.close)
|
||||
collection = c[DB].test
|
||||
|
||||
# Need one document.
|
||||
collection.drop()
|
||||
collection.insert_one({})
|
||||
|
||||
# nthreads had better be much larger than max_pool_size to ensure that
|
||||
# ntasks had better be much larger than max_pool_size to ensure that
|
||||
# max_pool_size connections are actually required at some point in this
|
||||
# test's execution.
|
||||
cx_pool = get_pool(c)
|
||||
nthreads = 10
|
||||
threads = []
|
||||
lock = threading.Lock()
|
||||
ntasks = 10
|
||||
tasks = []
|
||||
lock = _create_lock()
|
||||
self.n_passed = 0
|
||||
|
||||
def f():
|
||||
@ -527,19 +520,18 @@ class TestPoolMaxSize(_TestPoolingBase):
|
||||
with lock:
|
||||
self.n_passed += 1
|
||||
|
||||
for _i in range(nthreads):
|
||||
t = threading.Thread(target=f)
|
||||
threads.append(t)
|
||||
for _i in range(ntasks):
|
||||
t = ConcurrentRunner(target=f)
|
||||
tasks.append(t)
|
||||
t.start()
|
||||
|
||||
joinall(threads)
|
||||
self.assertEqual(nthreads, self.n_passed)
|
||||
joinall(tasks)
|
||||
self.assertEqual(ntasks, self.n_passed)
|
||||
self.assertTrue(len(cx_pool.conns) > 1)
|
||||
self.assertEqual(0, cx_pool.requests)
|
||||
|
||||
def test_max_pool_size_none(self):
|
||||
c = self.rs_or_single_client(maxPoolSize=None)
|
||||
self.addCleanup(c.close)
|
||||
collection = c[DB].test
|
||||
|
||||
# Need one document.
|
||||
@ -547,9 +539,9 @@ class TestPoolMaxSize(_TestPoolingBase):
|
||||
collection.insert_one({})
|
||||
|
||||
cx_pool = get_pool(c)
|
||||
nthreads = 10
|
||||
threads = []
|
||||
lock = threading.Lock()
|
||||
ntasks = 10
|
||||
tasks = []
|
||||
lock = _create_lock()
|
||||
self.n_passed = 0
|
||||
|
||||
def f():
|
||||
@ -559,19 +551,18 @@ class TestPoolMaxSize(_TestPoolingBase):
|
||||
with lock:
|
||||
self.n_passed += 1
|
||||
|
||||
for _i in range(nthreads):
|
||||
t = threading.Thread(target=f)
|
||||
threads.append(t)
|
||||
for _i in range(ntasks):
|
||||
t = ConcurrentRunner(target=f)
|
||||
tasks.append(t)
|
||||
t.start()
|
||||
|
||||
joinall(threads)
|
||||
self.assertEqual(nthreads, self.n_passed)
|
||||
joinall(tasks)
|
||||
self.assertEqual(ntasks, self.n_passed)
|
||||
self.assertTrue(len(cx_pool.conns) > 1)
|
||||
self.assertEqual(cx_pool.max_pool_size, float("inf"))
|
||||
|
||||
def test_max_pool_size_zero(self):
|
||||
c = self.rs_or_single_client(maxPoolSize=0)
|
||||
self.addCleanup(c.close)
|
||||
pool = get_pool(c)
|
||||
self.assertEqual(pool.max_pool_size, float("inf"))
|
||||
|
||||
|
||||
@ -228,6 +228,7 @@ converted_tests = [
|
||||
"test_monitoring.py",
|
||||
"test_mongos_load_balancing.py",
|
||||
"test_on_demand_csfle.py",
|
||||
"test_pooling.py",
|
||||
"test_raw_bson.py",
|
||||
"test_read_concern.py",
|
||||
"test_read_preferences.py",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user