diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..5cbc2d3f5 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,4 @@ +# do not notify until at least 100 builds have been uploaded from the CI pipeline +# you can also set after_n_builds on comments independently +comment: + after_n_builds: 100 diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 91fa44277..1af19857c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -38,6 +38,7 @@ post: # Disabled, causing timeouts # - func: "upload working dir" - func: "teardown system" + - func: "upload codecov" - func: "upload coverage" - func: "upload mo artifacts" - func: "upload test results" diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index bd983abb3..9fbc1f850 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -250,6 +250,27 @@ functions: working_dir: src include_expansions_in_env: - TOOLCHAIN_VERSION + - COVERAGE + type: test + + # Upload coverage codecov + upload codecov: + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/upload-codecov.sh + working_dir: src + include_expansions_in_env: + - CODECOV_TOKEN + - build_variant + - task_name + - github_commit + - github_pr_number + - github_pr_head_branch + - github_author + - requester + - branch_name type: test # Upload coverage diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 60ee6ed13..6a460ce48 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -75,7 +75,7 @@ tasks: SUB_TEST_NAME: session-creds TOOLCHAIN_VERSION: 3.14t tags: [auth-aws, auth-aws-session-creds, free-threaded] - - name: test-auth-aws-rapid-web-identity-python3.14 + - name: test-auth-aws-rapid-web-identity-python3.14-cov commands: - func: run server vars: @@ -87,7 +87,8 @@ tasks: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity TOOLCHAIN_VERSION: "3.14" - tags: [auth-aws, auth-aws-web-identity] + COVERAGE: "1" + tags: [auth-aws, auth-aws-web-identity, pr] - name: test-auth-aws-rapid-web-identity-session-name-python3.14 commands: - func: run server @@ -904,7 +905,7 @@ tasks: - ocsp-ecdsa - rapid - ocsp-staple - - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-python3.14 + - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-python3.14-cov commands: - func: run tests vars: @@ -913,11 +914,13 @@ tasks: TEST_NAME: ocsp TOOLCHAIN_VERSION: "3.14" VERSION: latest + COVERAGE: "1" tags: - ocsp - ocsp-ecdsa - latest - ocsp-staple + - pr - name: test-ocsp-ecdsa-invalid-cert-server-staples-v4.4-python3.10-min-deps commands: - func: run tests @@ -1928,7 +1931,7 @@ tasks: - ocsp-rsa - rapid - ocsp-staple - - name: test-ocsp-rsa-valid-cert-server-staples-latest-python3.14 + - name: test-ocsp-rsa-valid-cert-server-staples-latest-python3.14-cov commands: - func: run tests vars: @@ -1937,11 +1940,13 @@ tasks: TEST_NAME: ocsp TOOLCHAIN_VERSION: "3.14" VERSION: latest + COVERAGE: "1" tags: - ocsp - ocsp-rsa - latest - ocsp-staple + - pr - name: test-ocsp-rsa-invalid-cert-server-staples-v4.4-python3.10-min-deps commands: - func: run tests @@ -2615,20 +2620,18 @@ tasks: - replica_set-auth-nossl - async - free-threaded - - name: test-server-version-python3.13-sync-auth-nossl-replica-set-cov + - name: test-server-version-python3.13-sync-auth-nossl-replica-set commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_sync tags: @@ -2636,20 +2639,18 @@ tasks: - python-3.13 - replica_set-auth-nossl - sync - - name: test-server-version-python3.12-async-auth-ssl-replica-set-cov + - name: test-server-version-python3.12-async-auth-ssl-replica-set commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -2657,20 +2658,18 @@ tasks: - python-3.12 - replica_set-auth-ssl - async - - name: test-server-version-python3.11-sync-auth-ssl-replica-set-cov + - name: test-server-version-python3.11-sync-auth-ssl-replica-set commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_sync tags: @@ -2743,20 +2742,18 @@ tasks: - python-pypy3.11 - replica_set-noauth-ssl - async - - name: test-server-version-python3.14-sync-noauth-ssl-replica-set-cov + - name: test-server-version-python3.14-sync-noauth-ssl-replica-set commands: - func: run server vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_sync tags: @@ -2764,20 +2761,18 @@ tasks: - python-3.14 - replica_set-noauth-ssl - sync - - name: test-server-version-python3.14-async-auth-nossl-sharded-cluster-cov + - name: test-server-version-python3.14-async-auth-nossl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -2829,20 +2824,18 @@ tasks: - sharded_cluster-auth-ssl - async - pr - - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_async tags: @@ -2850,20 +2843,18 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - async - - name: test-server-version-python3.12-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.12-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -2871,20 +2862,18 @@ tasks: - python-3.12 - sharded_cluster-auth-ssl - async - - name: test-server-version-python3.13-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.13-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_async tags: @@ -2892,20 +2881,18 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - async - - name: test-server-version-python3.14-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.14-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -2976,20 +2963,18 @@ tasks: - sharded_cluster-auth-ssl - sync - pr - - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_sync tags: @@ -2997,20 +2982,18 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.12-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.12-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_sync tags: @@ -3018,20 +3001,18 @@ tasks: - python-3.12 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.13-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.13-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_sync tags: @@ -3039,20 +3020,18 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.14-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.14-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_sync tags: @@ -3099,20 +3078,18 @@ tasks: - python-pypy3.11 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.12-async-noauth-nossl-sharded-cluster-cov + - name: test-server-version-python3.12-async-noauth-nossl-sharded-cluster commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -3120,20 +3097,18 @@ tasks: - python-3.12 - sharded_cluster-noauth-nossl - async - - name: test-server-version-python3.11-sync-noauth-nossl-sharded-cluster-cov + - name: test-server-version-python3.11-sync-noauth-nossl-sharded-cluster commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_sync tags: @@ -3141,7 +3116,7 @@ tasks: - python-3.11 - sharded_cluster-noauth-nossl - sync - - name: test-server-version-python3.10-async-noauth-ssl-sharded-cluster-min-deps-cov + - name: test-server-version-python3.10-async-noauth-ssl-sharded-cluster-min-deps commands: - func: run server vars: @@ -3149,14 +3124,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" TOOLCHAIN_VERSION: "3.10" TEST_NAME: default_async tags: @@ -3183,20 +3156,18 @@ tasks: - python-pypy3.11 - sharded_cluster-noauth-ssl - sync - - name: test-server-version-python3.13-async-auth-nossl-standalone-cov + - name: test-server-version-python3.13-async-auth-nossl-standalone commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_async tags: @@ -3204,20 +3175,18 @@ tasks: - python-3.13 - standalone-auth-nossl - async - - name: test-server-version-python3.12-sync-auth-nossl-standalone-cov + - name: test-server-version-python3.12-sync-auth-nossl-standalone commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_sync tags: @@ -3225,20 +3194,18 @@ tasks: - python-3.12 - standalone-auth-nossl - sync - - name: test-server-version-python3.11-async-auth-ssl-standalone-cov + - name: test-server-version-python3.11-async-auth-ssl-standalone commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_async tags: @@ -3246,7 +3213,7 @@ tasks: - python-3.11 - standalone-auth-ssl - async - - name: test-server-version-python3.10-sync-auth-ssl-standalone-min-deps-cov + - name: test-server-version-python3.10-sync-auth-ssl-standalone-min-deps commands: - func: run server vars: @@ -3254,14 +3221,12 @@ tasks: SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - COVERAGE: "1" TOOLCHAIN_VERSION: "3.10" TEST_NAME: default_sync tags: @@ -3293,18 +3258,20 @@ tasks: - standalone-noauth-nossl - async - pr - - name: test-server-version-pypy3.11-sync-noauth-nossl-standalone + - name: test-server-version-pypy3.11-sync-noauth-nossl-standalone-cov commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone + COVERAGE: "1" TOOLCHAIN_VERSION: pypy3.11 TEST_NAME: default_sync tags: @@ -3313,20 +3280,18 @@ tasks: - standalone-noauth-nossl - sync - pr - - name: test-server-version-python3.14-async-noauth-ssl-standalone-cov + - name: test-server-version-python3.14-async-noauth-ssl-standalone commands: - func: run server vars: AUTH: noauth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -4082,7 +4047,7 @@ tasks: - standalone-noauth-nossl - async - pypy - - name: test-standard-latest-python3.12-async-noauth-ssl-replica-set + - name: test-standard-latest-python3.12-async-noauth-ssl-replica-set-cov commands: - func: run server vars: @@ -4090,12 +4055,14 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -4128,7 +4095,7 @@ tasks: - replica_set-noauth-ssl - async - pypy - - name: test-standard-latest-python3.13-async-auth-ssl-sharded-cluster + - name: test-standard-latest-python3.13-async-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -4136,12 +4103,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_async tags: @@ -4151,7 +4120,7 @@ tasks: - sharded_cluster-auth-ssl - async - pr - - name: test-standard-latest-python3.11-async-noauth-nossl-standalone + - name: test-standard-latest-python3.11-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4159,12 +4128,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_async tags: @@ -4174,7 +4145,7 @@ tasks: - standalone-noauth-nossl - async - pr - - name: test-standard-latest-python3.14-async-noauth-nossl-standalone + - name: test-standard-latest-python3.14-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4182,12 +4153,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -4829,7 +4802,7 @@ tasks: - python-3.13 - standalone-noauth-nossl - noauth - - name: test-non-standard-latest-python3.14t-noauth-ssl-replica-set + - name: test-non-standard-latest-python3.14t-noauth-ssl-replica-set-cov commands: - func: run server vars: @@ -4837,12 +4810,14 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: 3.14t tags: - test-non-standard @@ -4874,7 +4849,7 @@ tasks: - replica_set-noauth-ssl - noauth - pypy - - name: test-non-standard-latest-python3.14-auth-ssl-sharded-cluster + - name: test-non-standard-latest-python3.14-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -4882,12 +4857,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" tags: - test-non-standard @@ -4896,7 +4873,7 @@ tasks: - sharded_cluster-auth-ssl - auth - pr - - name: test-non-standard-latest-python3.13-noauth-nossl-standalone + - name: test-non-standard-latest-python3.13-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4904,12 +4881,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" tags: - test-non-standard @@ -5007,7 +4986,7 @@ tasks: - pypy # Test numpy tests - - name: test-numpy-python3.10 + - name: test-numpy-python3.10-python3.10 commands: - func: test numpy vars: @@ -5017,16 +4996,18 @@ tasks: - vector - python-3.10 - test-numpy - - name: test-numpy-python3.14 + - name: test-numpy-python3.14-python3.14-cov commands: - func: test numpy vars: TOOLCHAIN_VERSION: "3.14" + COVERAGE: "1" tags: - binary - vector - python-3.14 - test-numpy + - pr # Test standard auth tests - name: test-standard-auth-v4.2-python3.10-auth-ssl-sharded-cluster-min-deps @@ -5290,7 +5271,7 @@ tasks: - sharded_cluster-auth-ssl - auth - pypy - - name: test-standard-auth-latest-python3.11-auth-ssl-sharded-cluster + - name: test-standard-auth-latest-python3.11-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -5298,12 +5279,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" tags: - test-standard-auth diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 42a677609..4c9116d62 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -367,6 +367,8 @@ buildvariants: display_name: No C Ext RHEL8 run_on: - rhel87-small + expansions: + NO_EXT: "1" # No server tests - name: no-server-rhel8 @@ -417,6 +419,8 @@ buildvariants: run_on: - ubuntu2204-small batchtime: 1440 + expansions: + COVERAGE: "1" tags: [pr] - name: auth-oidc-macos tasks: diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 095b7938d..0785bcf01 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -38,6 +38,7 @@ trap "cleanup_tests" SIGINT ERR # Start the test runner. echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..." +echo "UV_ARGS=${UV_ARGS}" uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@" echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done." diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 04579c521..b4760eab9 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -321,7 +321,7 @@ def create_no_c_ext_variants(): expansions = dict() handle_c_ext(C_EXTS[0], expansions) display_name = get_variant_name("No C Ext", host) - return [create_variant(tasks, display_name, host=host)] + return [create_variant(tasks, display_name, host=host, expansions=expansions)] def create_mod_wsgi_variants(): @@ -344,8 +344,12 @@ def create_test_numpy_tasks(): tasks = [] for python in MIN_MAX_PYTHON: tags = ["binary", "vector", f"python-{python}", "test-numpy"] - task_name = get_task_name("test-numpy", python=python) - test_func = FunctionCall(func="test numpy", vars=dict(TOOLCHAIN_VERSION=python)) + vars = dict(TOOLCHAIN_VERSION=python) + if python == MIN_MAX_PYTHON[-1]: + tags.append("pr") + vars["COVERAGE"] = "1" + task_name = get_task_name("test-numpy", python=python, **vars) + test_func = FunctionCall(func="test numpy", vars=vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) return tasks @@ -397,6 +401,7 @@ def create_oidc_auth_variants(): tags=["pr"], host=host, batchtime=BATCHTIME_DAY, + expansions=dict(COVERAGE="1"), ) ) return variants @@ -596,7 +601,7 @@ def create_server_version_tasks(): expansions["TEST_MIN_DEPS"] = "1" if "t" in python: tags.append("free-threaded") - if python not in PYPYS and "t" not in python: + if "pr" in tags: expansions["COVERAGE"] = "1" name = get_task_name( "test-server-version", @@ -661,6 +666,8 @@ def create_test_non_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + elif pr: + expansions["COVERAGE"] = "1" name = get_task_name("test-non-standard", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() @@ -703,6 +710,8 @@ def create_test_standard_auth_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + elif pr: + expansions["COVERAGE"] = "1" name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() @@ -741,6 +750,8 @@ def create_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + elif pr: + expansions["COVERAGE"] = "1" name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() @@ -810,8 +821,11 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, TOOLCHAIN_VERSION=python) - if python == ALL_PYTHONS[0]: + if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" + elif python == MIN_MAX_PYTHON[-1]: + tags.append("pr") + test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) funcs = [server_func, assume_func, test_func] @@ -849,11 +863,11 @@ def create_oidc_tasks(): tasks = [] for sub_test in ["default", "azure", "gcp", "eks", "aks", "gke"]: vars = dict(TEST_NAME="auth_oidc", SUB_TEST_NAME=sub_test) - test_func = FunctionCall(func="run tests", vars=vars) - task_name = f"test-auth-oidc-{sub_test}" tags = ["auth_oidc"] if sub_test != "default": tags.append("auth_oidc_remote") + test_func = FunctionCall(func="run tests", vars=vars) + task_name = get_task_name(f"test-auth-oidc-{sub_test}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) return tasks @@ -903,14 +917,14 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): ) if python == ALL_PYTHONS[0]: vars["TEST_MIN_DEPS"] = "1" - test_func = FunctionCall(func="run tests", vars=vars) - tags = ["ocsp", f"ocsp-{algo}", version] if "disableStapling" not in variant: tags.append("ocsp-staple") - if algo == "valid-cert-server-staples" and version == "latest": + if base_task_name == "valid-cert-server-staples" and version == "latest": tags.append("pr") - + if "TEST_MIN_DEPS" not in vars: + vars["COVERAGE"] = "1" + test_func = FunctionCall(func="run tests", vars=vars) task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) @@ -1077,6 +1091,26 @@ def create_upload_coverage_func(): return "upload coverage", [get_assume_role(), cmd] +def create_upload_coverage_codecov_func(): + # Upload the coverage xml report to codecov. + include_expansions = [ + "CODECOV_TOKEN", + "build_variant", + "task_name", + "github_commit", + "github_pr_number", + "github_pr_head_branch", + "github_author", + "requester", + "branch_name", + ] + args = [ + ".evergreen/scripts/upload-codecov.sh", + ] + upload_cmd = get_subprocess_exec(include_expansions_in_env=include_expansions, args=args) + return "upload codecov", [upload_cmd] + + def create_download_and_merge_coverage_func(): include_expansions = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args = [ @@ -1210,7 +1244,7 @@ def create_run_tests_func(): def create_test_numpy_func(): - includes = ["TOOLCHAIN_VERSION"] + includes = ["TOOLCHAIN_VERSION", "COVERAGE"] test_cmd = get_subprocess_exec( include_expansions_in_env=includes, args=[".evergreen/just.sh", "test-numpy"] ) diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py index 1996d5d63..16782de9a 100644 --- a/.evergreen/scripts/resync-all-specs.py +++ b/.evergreen/scripts/resync-all-specs.py @@ -7,6 +7,8 @@ import subprocess from argparse import Namespace from subprocess import CalledProcessError +JIRA_FILTER = "https://jira.mongodb.org/issues/?jql=labels%20%3D%20automated-sync%20AND%20status%20!%3D%20Closed" + def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: """Actually sync the specs""" @@ -117,6 +119,7 @@ def write_summary(errored: dict[str, str], new: list[str], filename: str | None) pr_body += "\n -".join(new) pr_body += "\n" if pr_body != "": + pr_body = f"Jira tickets: {JIRA_FILTER}\n\n" + pr_body if filename is None: print(f"\n{pr_body}") else: diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index a35fbb57a..9757eb3a4 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -12,7 +12,7 @@ def set_env(name: str, value: Any = "1") -> None: def start_server(): opts, extra_opts = get_test_options( - "Run a MongoDB server. All given flags will be passed to run-orchestration.sh in DRIVERS_TOOLS.", + "Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.", require_sub_test_name=False, allow_extra_opts=True, ) @@ -51,7 +51,7 @@ def start_server(): elif opts.quiet: extra_opts.append("-q") - cmd = ["bash", f"{DRIVERS_TOOLS}/.evergreen/run-orchestration.sh", *extra_opts] + cmd = ["bash", f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh", "start", *extra_opts] run_command(cmd, cwd=DRIVERS_TOOLS) diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 9c8101c5b..fcf5fe76c 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -4,7 +4,9 @@ import json import logging import os import platform +import shlex import shutil +import subprocess import sys from datetime import datetime from pathlib import Path @@ -202,6 +204,16 @@ def run() -> None: if os.environ.get("DEBUG_LOG"): TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG}".split()) + if os.environ.get("COVERAGE"): + binary = sys.executable.replace(os.sep, "/") + cmd = f"{binary} -m coverage run -m pytest {' '.join(TEST_ARGS)} {' '.join(sys.argv[1:])}" + result = subprocess.run(shlex.split(cmd), check=False) # noqa: S603 + cmd = f"{binary} -m coverage report" + subprocess.run(shlex.split(cmd), check=False) # noqa: S603 + if result.returncode != 0: + print(result.stderr) + sys.exit(result.returncode) + # Run local tests. ret = pytest.main(TEST_ARGS + sys.argv[1:]) if ret != 0: diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 939423ffc..e188dcaa9 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -153,6 +153,10 @@ def handle_test_env() -> None: # Start compiling the args we'll pass to uv. UV_ARGS = ["--extra test --no-group dev"] + # If USE_ACTIVE_VENV is set, add --active to UV_ARGS so run-tests.sh uses the active venv. + if is_set("USE_ACTIVE_VENV"): + UV_ARGS.append("--active") + test_title = test_name if sub_test_name: test_title += f" {sub_test_name}" @@ -324,7 +328,8 @@ def handle_test_env() -> None: version = os.environ.get("VERSION", "latest") cmd = [ "bash", - f"{DRIVERS_TOOLS}/.evergreen/run-orchestration.sh", + f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh", + "start", "--ssl", "--version", version, @@ -431,6 +436,9 @@ def handle_test_env() -> None: # We do not want the default client_context to be initialized. write_env("DISABLE_CONTEXT") + if test_name == "numpy": + UV_ARGS.append("--with numpy") + if test_name == "perf": data_dir = ROOT / "specifications/source/benchmarking/data" if not data_dir.exists(): @@ -458,12 +466,14 @@ def handle_test_env() -> None: # Keep in sync with combine-coverage.sh. # coverage >=5 is needed for relative_files=true. UV_ARGS.append("--group coverage") - TEST_ARGS = f"{TEST_ARGS} --cov" write_env("COVERAGE") if opts.green_framework: framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"] UV_ARGS.append(f"--group {framework}") + if framework == "gevent" and opts.test_min_deps: + # PYTHON-5729. This can be removed when the min supported gevent is moved to 25.9.1. + UV_ARGS.append('--with "setuptools==81.0"') else: TEST_ARGS = f"-v --durations=5 {TEST_ARGS}" diff --git a/.evergreen/scripts/stop-server.sh b/.evergreen/scripts/stop-server.sh index 7599387f5..045a655cb 100755 --- a/.evergreen/scripts/stop-server.sh +++ b/.evergreen/scripts/stop-server.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Stop a server that was started using run-orchestration.sh in DRIVERS_TOOLS. +# Stop a server that was started using run-mongodb.sh in DRIVERS_TOOLS. set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) @@ -11,4 +11,4 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi -bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh +bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop diff --git a/.evergreen/scripts/upload-codecov.sh b/.evergreen/scripts/upload-codecov.sh new file mode 100755 index 000000000..5c1d84c55 --- /dev/null +++ b/.evergreen/scripts/upload-codecov.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# shellcheck disable=SC2154 +# Upload a coverate report to codecov. +set -eu + +HERE=$(dirname ${BASH_SOURCE:-$0}) +ROOT=$(dirname "$(dirname $HERE)") + +pushd $ROOT > /dev/null +export FNAME=coverage.xml +REQUESTER=${requester:-} + +if [ ! -f ".coverage" ]; then + echo "There are no coverage results, not running codecov" + exit 0 +fi + +if [[ "${REQUESTER}" == "github_pr" || "${REQUESTER}" == "commit" ]]; then + echo "Uploading codecov for $REQUESTER..." +else + echo "Error: requester must be 'github_pr' or 'commit', got '${REQUESTER}'" >&2 + exit 1 +fi + +printf 'sha: %s\n' "$github_commit" +printf 'flag: %s-%s\n' "$build_variant" "$task_name" +printf 'file: %s\n' "$FNAME" +uv tool run --with "coverage[toml]" coverage xml + +codecov_args=( + upload-process + --report-type coverage + --disable-search + --fail-on-error + --git-service github + --token "${CODECOV_TOKEN}" + --sha "${github_commit}" + --flag "${build_variant}-${task_name}" + --file "${FNAME}" +) + +if [ -n "${github_pr_number:-}" ]; then + printf 'branch: %s:%s\n' "$github_author" "$github_pr_head_branch" + printf 'pr: %s\n' "$github_pr_number" + uv tool run --from codecov-cli codecovcli \ + "${codecov_args[@]}" \ + --pr "${github_pr_number}" \ + --branch "${github_author}:${github_pr_head_branch}" +else + printf 'branch: %s\n' "$branch_name" + uv tool run --from codecov-cli codecovcli \ + "${codecov_args[@]}" \ + --branch "${branch_name}" +fi +echo "Uploading codecov for $REQUESTER... done." + +popd > /dev/null diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 2bc9c720d..914cd9ac6 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -44,6 +44,7 @@ TEST_SUITE_MAP = { "mockupdb": "mockupdb", "ocsp": "ocsp", "perf": "perf", + "numpy": "", } # Tests that require a sub test suite. @@ -51,7 +52,7 @@ SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"] EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "doctest"] -# Tests that do not use run-orchestration directly. +# Tests that do not use run-mongodb directly. NO_RUN_ORCHESTRATION = [ "auth_oidc", "atlas_connect", diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..b67cb49ac --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,44 @@ +When reviewing code, focus on: + +## Security Critical Issues +- Check for hardcoded secrets, API keys, or credentials. +- Check for instances of potential method call injection, dynamic code execution, symbol injection or other code injection vulnerabilities. + +## Performance Red Flags +- Spot inefficient loops and algorithmic issues. +- Check for memory leaks and resource cleanup. + +## Code Quality Essentials +- Methods should be focused and appropriately sized. If a method is doing too much, suggest refactorings to split it up. +- Use clear, descriptive naming conventions. +- Avoid encapsulation violations and ensure proper separation of concerns. +- All public classes, modules, and methods should have clear documentation in Sphinx format. + +## PyMongo-specific Concerns +- Do not review files within `pymongo/synchronous` or files in `test/` that also have a file of the same name in `test/asynchronous` unless the reviewed changes include a `_IS_SYNC` statement. PyMongo generates these files from `pymongo/asynchronous` and `test/asynchronous` using `tools/synchro.py`. +- All asynchronous functions must not call any blocking I/O. + +## Review Style +- Be specific and actionable in feedback. +- Explain the "why" behind recommendations. +- Acknowledge good patterns when you see them. +- Ask clarifying questions when code intent is unclear. + +Always prioritize security vulnerabilities and performance issues that could impact users. + +Always suggest changes to improve readability and testability. For example, this suggestion seeks to make the code more readable, reusable, and testable: + +```python +# Instead of: +if user.email and "@" in user.email and len(user.email) > 5: + submit_button.enabled = True +else: + submit_button.enabled = False + +# Consider: +def valid_email(email): + return email and "@" in email and len(email) > 5 + + +submit_button.enabled = valid_email(user.email) +``` diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 086e22fae..e8fb9b918 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -26,7 +26,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: "3.10" @@ -68,7 +68,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: ${{ matrix.python-version }} @@ -79,6 +79,37 @@ jobs: - name: Run tests run: uv run --extra test pytest -v + coverage: + # This enables a coverage report for a given PR, which will be augmented by + # the combined codecov report uploaded in Evergreen. + runs-on: ubuntu-latest + + name: Coverage + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 + with: + enable-cache: true + python-version: "3.10" + - id: setup-mongodb + uses: mongodb-labs/drivers-evergreen-tools@master + with: + version: "8.0" + - name: Install just + run: uv tool install rust-just + - name: Setup tests + run: COVERAGE=1 just setup-tests + - name: Run tests + run: just run-tests + - name: Generate xml report + run: uv tool run --with "coverage[toml]" coverage xml + - name: Upload test results to Codecov + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} doctest: runs-on: ubuntu-latest name: DocTest @@ -87,7 +118,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: "3.10" @@ -112,7 +143,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: "3.10" @@ -131,7 +162,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: "3.10" @@ -153,7 +184,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: "${{matrix.python}}" @@ -174,7 +205,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: enable-cache: true python-version: "3.10" @@ -264,7 +295,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7 with: python-version: "3.9" - id: setup-mongodb diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 26f75fa79..c64a9e32e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -18,4 +18,4 @@ jobs: with: persist-credentials: false - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@135698455da5c3b3e55f73f4419e481ab68cdd95 # v0.4.1 + uses: zizmorcore/zizmor-action@0dce2577a4760a2749d8cfb7a84b7d5585ebcb7d # v0.5.0 diff --git a/.gitignore b/.gitignore index 74ed0bbb7..f69f27404 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,6 @@ test/lambda/*.json # test results and logs xunit-results/ +coverage.xml server.log +.coverage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eaf05111d..86c5e6455 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -197,7 +197,7 @@ the pages will re-render and the browser will automatically refresh. version of Python, set `UV_PYTHON` before running `just install`. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to - [`run-orchestration.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh). Run `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h` + [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` for a full list of options. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run @@ -205,6 +205,7 @@ the pages will re-render and the browser will automatically refresh. and the `` to test a full module. For example: `just test test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`. - Use the `-k` argument to select tests by pattern. +- Run `just test-coverage` to run tests with coverage and display a report. After running tests with coverage, use `just coverage-html` to generate an HTML report in `htmlcov/index.html`. ## Running tests that require secrets, services, or other configuration @@ -396,7 +397,7 @@ To run any of the test suites with minimum supported dependencies, pass `--test- - If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add to the list of pytest markers in `pyproject.toml`. Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`. If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead. -- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in +- If the test uses Atlas or otherwise doesn't use `run-mongodb.sh`, add it to the `NO_RUN_ORCHESTRATION` list in `.evergreen/scripts/utils.py`. - If there is something special required to run the local server or there is an extra flag that should always be set like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`. diff --git a/bson/_cbsonmodule.c b/bson/_cbsonmodule.c index 7d184641c..034490f55 100644 --- a/bson/_cbsonmodule.c +++ b/bson/_cbsonmodule.c @@ -356,7 +356,8 @@ static PyObject* datetime_ms_from_millis(PyObject* self, long long millis){ if (!(ll_millis = PyLong_FromLongLong(millis))){ return NULL; } - dt = PyObject_CallFunctionObjArgs(state->DatetimeMS, ll_millis, NULL); + PyObject* args[1] = {ll_millis}; + dt = PyObject_Vectorcall(state->DatetimeMS, args, 1, NULL); Py_DECREF(ll_millis); return dt; } @@ -401,7 +402,9 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o int64_t min_millis_offset = 0; int64_t max_millis_offset = 0; if (options->tz_aware && options->tzinfo && options->tzinfo != Py_None) { - PyObject* utcoffset = PyObject_CallMethodObjArgs(options->tzinfo, state->_utcoffset_str, state->min_datetime, NULL); + PyObject* utcoffset_args[2] = {options->tzinfo, state->min_datetime}; + PyObject* utcoffset = PyObject_VectorcallMethod( + state->_utcoffset_str, utcoffset_args, 2, NULL); if (utcoffset == NULL) { return 0; } @@ -420,7 +423,9 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000); } Py_DECREF(utcoffset); - utcoffset = PyObject_CallMethodObjArgs(options->tzinfo, state->_utcoffset_str, state->max_datetime, NULL); + utcoffset_args[1] = state->max_datetime; + utcoffset = PyObject_VectorcallMethod( + state->_utcoffset_str, utcoffset_args, 2, NULL); if (utcoffset == NULL) { return 0; } @@ -481,7 +486,9 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o /* convert to local time */ if (options->tzinfo != Py_None) { - PyObject* temp = PyObject_CallMethodObjArgs(value, state->_astimezone_str, options->tzinfo, NULL); + PyObject* astimezone_args[2] = {value, options->tzinfo}; + PyObject* temp = PyObject_VectorcallMethod( + state->_astimezone_str, astimezone_args, 2, NULL); Py_DECREF(value); value = temp; } @@ -688,7 +695,8 @@ static int _load_python_objects(PyObject* module) { return 1; } - compiled = PyObject_CallFunction(re_compile, "O", empty_string); + PyObject* compile_args[1] = {empty_string}; + compiled = PyObject_Vectorcall(re_compile, compile_args, 1, NULL); Py_DECREF(re_compile); if (compiled == NULL) { state->REType = NULL; @@ -711,13 +719,19 @@ static long _type_marker(PyObject* object, PyObject* _type_marker_str) { PyObject* type_marker = NULL; long type = 0; - if (PyObject_HasAttr(object, _type_marker_str)) { - type_marker = PyObject_GetAttr(object, _type_marker_str); - if (type_marker == NULL) { + #if PY_VERSION_HEX >= 0x030D0000 + // 3.13 + if (PyObject_GetOptionalAttr(object, _type_marker_str, &type_marker) == -1) { return -1; } - } - + # else + if (PyObject_HasAttr(object, _type_marker_str)) { + type_marker = PyObject_GetAttr(object, _type_marker_str); + if (type_marker == NULL) { + return -1; + } + } + #endif /* * Python objects with broken __getattr__ implementations could return * arbitrary types for a call to PyObject_GetAttrString. For example @@ -814,6 +828,7 @@ int convert_codec_options(PyObject* self, PyObject* options_obj, codec_options_t } options->is_raw_bson = (101 == type_marker); + options->is_dict_class = (options->document_class == (PyObject*)&PyDict_Type); options->options_obj = options_obj; Py_INCREF(options->options_obj); @@ -1013,10 +1028,20 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, } /* * Use _type_marker attribute instead of PyObject_IsInstance for better perf. + * + * Skip _type_marker lookup for common built-in types + * that we know don't have a _type_marker attribute. This avoids the overhead + * of PyObject_HasAttr/PyObject_GetAttr calls for the most common cases. */ - type = _type_marker(value, state->_type_marker_str); - if (type < 0) { - return 0; + if (PyUnicode_CheckExact(value) || PyLong_CheckExact(value) || PyFloat_CheckExact(value) || + PyBool_Check(value) || PyDict_CheckExact(value) || PyList_CheckExact(value) || + PyTuple_CheckExact(value) || PyBytes_CheckExact(value) || value == Py_None) { + type = 0; + } else { + type = _type_marker(value, state->_type_marker_str); + if (type < 0) { + return 0; + } } switch (type) { @@ -1227,7 +1252,9 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, case 100: { /* DBRef */ - PyObject* as_doc = PyObject_CallMethodObjArgs(value, state->_as_doc_str, NULL); + PyObject* as_doc_args[1] = {value}; + PyObject* as_doc = PyObject_VectorcallMethod( + state->_as_doc_str, as_doc_args, 1, NULL); if (!as_doc) { return 0; } @@ -1383,7 +1410,9 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, return write_unicode(buffer, value); } else if (PyDateTime_Check(value)) { long long millis; - PyObject* utcoffset = PyObject_CallMethodObjArgs(value, state->_utcoffset_str , NULL); + PyObject* utcoffset_args[1] = {value}; + PyObject* utcoffset = PyObject_VectorcallMethod( + state->_utcoffset_str, utcoffset_args, 1, NULL); if (utcoffset == NULL) return 0; if (utcoffset != Py_None) { @@ -1422,7 +1451,9 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, if (!(uuid_rep_obj = PyLong_FromLong(options->uuid_rep))) { return 0; } - binary_value = PyObject_CallMethodObjArgs(state->Binary, state->_from_uuid_str, value, uuid_rep_obj, NULL); + PyObject* from_uuid_args[3] = {state->Binary, value, uuid_rep_obj}; + binary_value = PyObject_VectorcallMethod( + state->_from_uuid_str, from_uuid_args, 3, NULL); Py_DECREF(uuid_rep_obj); if (binary_value == NULL) { @@ -1452,7 +1483,8 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, if (converter != NULL) { /* Transform types that have a registered converter. * A new reference is created upon transformation. */ - new_value = PyObject_CallFunctionObjArgs(converter, value, NULL); + PyObject* converter_args[1] = {value}; + new_value = PyObject_Vectorcall(converter, converter_args, 1, NULL); if (new_value == NULL) { return 0; } @@ -1466,8 +1498,9 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, /* Try the fallback encoder if one is provided and we have not already * attempted to use the fallback encoder. */ if (!in_fallback_call && options->type_registry.has_fallback_encoder) { - new_value = PyObject_CallFunctionObjArgs( - options->type_registry.fallback_encoder, value, NULL); + PyObject* fallback_args[1] = {value}; + new_value = PyObject_Vectorcall( + options->type_registry.fallback_encoder, fallback_args, 1, NULL); if (new_value == NULL) { // propagate any exception raised by the callback return 0; @@ -1668,7 +1701,8 @@ void handle_invalid_doc_error(PyObject* dict) { goto cleanup; } // Add doc to the error instance as a property. - new_evalue = PyObject_CallFunctionObjArgs(InvalidDocument, new_msg, dict, NULL); + PyObject* exc_args[2] = {new_msg, dict}; + new_evalue = PyObject_Vectorcall(InvalidDocument, exc_args, 2, NULL); Py_DECREF(evalue); Py_DECREF(etype); etype = InvalidDocument; @@ -1944,7 +1978,8 @@ static PyObject *_dbref_hook(PyObject* self, PyObject* value) { PyMapping_DelItem(value, state->_dollar_db_str); } - ret = PyObject_CallFunctionObjArgs(state->DBRef, ref, id, database, value, NULL); + PyObject* dbref_args[4] = {ref, id, database, value}; + ret = PyObject_Vectorcall(state->DBRef, dbref_args, 4, NULL); Py_DECREF(value); } else { ret = value; @@ -2160,7 +2195,13 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto uuiderror; } - binary_value = PyObject_CallFunction(state->Binary, "(Oi)", data, subtype); + PyObject* subtype_obj = PyLong_FromLong(subtype); + if (!subtype_obj) { + goto uuiderror; + } + PyObject* binary_args[2] = {data, subtype_obj}; + binary_value = PyObject_Vectorcall(state->Binary, binary_args, 2, NULL); + Py_DECREF(subtype_obj); if (binary_value == NULL) { goto uuiderror; } @@ -2175,7 +2216,9 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, if (!uuid_rep_obj) { goto uuiderror; } - value = PyObject_CallMethodObjArgs(binary_value, state->_as_uuid_str, uuid_rep_obj, NULL); + PyObject* as_uuid_args[2] = {binary_value, uuid_rep_obj}; + value = PyObject_VectorcallMethod( + state->_as_uuid_str, as_uuid_args, 2, NULL); Py_DECREF(uuid_rep_obj); } @@ -2194,7 +2237,8 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, Py_DECREF(data); goto invalid; } - value = PyObject_CallFunctionObjArgs(state->Binary, data, st, NULL); + PyObject* binary_args[2] = {data, st}; + value = PyObject_Vectorcall(state->Binary, binary_args, 2, NULL); Py_DECREF(st); Py_DECREF(data); if (!value) { @@ -2215,7 +2259,13 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, if (max < 12) { goto invalid; } - value = PyObject_CallFunction(state->ObjectId, "y#", buffer + *position, (Py_ssize_t)12); + PyObject* oid_bytes = PyBytes_FromStringAndSize(buffer + *position, 12); + if (!oid_bytes) { + goto invalid; + } + PyObject* oid_args[1] = {oid_bytes}; + value = PyObject_Vectorcall(state->ObjectId, oid_args, 1, NULL); + Py_DECREF(oid_bytes); *position += 12; break; } @@ -2294,7 +2344,14 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } *position += (unsigned)flags_length + 1; - value = PyObject_CallFunction(state->Regex, "Oi", pattern, flags); + PyObject* flags_obj = PyLong_FromLong(flags); + if (!flags_obj) { + Py_DECREF(pattern); + goto invalid; + } + PyObject* regex_args[2] = {pattern, flags_obj}; + value = PyObject_Vectorcall(state->Regex, regex_args, 2, NULL); + Py_DECREF(flags_obj); Py_DECREF(pattern); break; } @@ -2327,13 +2384,21 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } *position += coll_length; - id = PyObject_CallFunction(state->ObjectId, "y#", buffer + *position, (Py_ssize_t)12); + PyObject* oid_bytes = PyBytes_FromStringAndSize(buffer + *position, 12); + if (!oid_bytes) { + Py_DECREF(collection); + goto invalid; + } + PyObject* oid_args[1] = {oid_bytes}; + id = PyObject_Vectorcall(state->ObjectId, oid_args, 1, NULL); + Py_DECREF(oid_bytes); if (!id) { Py_DECREF(collection); goto invalid; } *position += 12; - value = PyObject_CallFunctionObjArgs(state->DBRef, collection, id, NULL); + PyObject* dbref_args[2] = {collection, id}; + value = PyObject_Vectorcall(state->DBRef, dbref_args, 2, NULL); Py_DECREF(collection); Py_DECREF(id); break; @@ -2363,7 +2428,8 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } *position += value_length; - value = PyObject_CallFunctionObjArgs(state->Code, code, NULL, NULL); + PyObject* code_args[1] = {code}; + value = PyObject_Vectorcall(state->Code, code_args, 1, NULL); Py_DECREF(code); break; } @@ -2429,7 +2495,8 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } *position += scope_size; - value = PyObject_CallFunctionObjArgs(state->Code, code, scope, NULL); + PyObject* code_scope_args[2] = {code, scope}; + value = PyObject_Vectorcall(state->Code, code_scope_args, 2, NULL); Py_DECREF(code); Py_DECREF(scope); break; @@ -2459,7 +2526,19 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, memcpy(&time, buffer + *position + 4, 4); inc = BSON_UINT32_FROM_LE(inc); time = BSON_UINT32_FROM_LE(time); - value = PyObject_CallFunction(state->Timestamp, "II", time, inc); + PyObject* time_obj = PyLong_FromUnsignedLong(time); + if (!time_obj) { + goto invalid; + } + PyObject* inc_obj = PyLong_FromUnsignedLong(inc); + if (!inc_obj) { + Py_DECREF(time_obj); + goto invalid; + } + PyObject* ts_args[2] = {time_obj, inc_obj}; + value = PyObject_Vectorcall(state->Timestamp, ts_args, 2, NULL); + Py_DECREF(time_obj); + Py_DECREF(inc_obj); *position += 8; break; } @@ -2471,7 +2550,13 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } memcpy(&ll, buffer + *position, 8); ll = (int64_t)BSON_UINT64_FROM_LE(ll); - value = PyObject_CallFunction(state->BSONInt64, "L", ll); + PyObject* ll_obj = PyLong_FromLongLong(ll); + if (!ll_obj) { + goto invalid; + } + PyObject* int64_args[1] = {ll_obj}; + value = PyObject_Vectorcall(state->BSONInt64, int64_args, 1, NULL); + Py_DECREF(ll_obj); *position += 8; break; } @@ -2484,19 +2569,21 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, if (!_bytes_obj) { goto invalid; } - value = PyObject_CallMethodObjArgs(state->Decimal128, state->_from_bid_str, _bytes_obj, NULL); + PyObject* dec128_args[2] = {state->Decimal128, _bytes_obj}; + value = PyObject_VectorcallMethod( + state->_from_bid_str, dec128_args, 2, NULL); Py_DECREF(_bytes_obj); *position += 16; break; } case 255: { - value = PyObject_CallFunctionObjArgs(state->MinKey, NULL); + value = PyObject_Vectorcall(state->MinKey, NULL, 0, NULL); break; } case 127: { - value = PyObject_CallFunctionObjArgs(state->MaxKey, NULL); + value = PyObject_Vectorcall(state->MaxKey, NULL, 0, NULL); break; } default: @@ -2548,7 +2635,8 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } converter = PyDict_GetItem(options->type_registry.decoder_map, value_type); if (converter != NULL) { - PyObject* new_value = PyObject_CallFunctionObjArgs(converter, value, NULL); + PyObject* converter_args[1] = {value}; + PyObject* new_value = PyObject_Vectorcall(converter, converter_args, 1, NULL); Py_DECREF(value_type); Py_DECREF(value); return new_value; @@ -2716,11 +2804,20 @@ static PyObject* _elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options) { unsigned position = 0; - PyObject* dict = PyObject_CallObject(options->document_class, NULL); + PyObject* dict; + int raw_array = 0; + + /* Use PyDict_New() directly when document_class is dict. + * This avoids the overhead of PyObject_CallObject() for the common case. */ + if (options->is_dict_class) { + dict = PyDict_New(); + } else { + dict = PyObject_CallObject(options->document_class, NULL); + } if (!dict) { return NULL; } - int raw_array = 0; + while (position < max) { PyObject* name = NULL; PyObject* value = NULL; @@ -2735,7 +2832,24 @@ static PyObject* _elements_to_dict(PyObject* self, const char* string, position = (unsigned)new_position; } - PyObject_SetItem(dict, name, value); + /* Use PyDict_SetItem() when document_class is dict. + * PyDict_SetItem() is faster than PyObject_SetItem() because it + * avoids method lookup overhead. */ + if (options->is_dict_class) { + if (PyDict_SetItem(dict, name, value) < 0) { + Py_DECREF(name); + Py_DECREF(value); + Py_DECREF(dict); + return NULL; + } + } else { + if (PyObject_SetItem(dict, name, value) < 0) { + Py_DECREF(name); + Py_DECREF(value); + Py_DECREF(dict); + return NULL; + } + } Py_DECREF(name); Py_DECREF(value); } @@ -2747,9 +2861,14 @@ static PyObject* elements_to_dict(PyObject* self, const char* string, const codec_options_t* options) { PyObject* result; if (options->is_raw_bson) { - return PyObject_CallFunction( - options->document_class, "y#O", - string, max, options->options_obj); + PyObject* bson_bytes = PyBytes_FromStringAndSize(string, max); + if (!bson_bytes) { + return NULL; + } + PyObject* raw_args[2] = {bson_bytes, options->options_obj}; + result = PyObject_Vectorcall(options->document_class, raw_args, 2, NULL); + Py_DECREF(bson_bytes); + return result; } if (Py_EnterRecursiveCall(" while decoding a BSON document")) return NULL; diff --git a/bson/_cbsonmodule.h b/bson/_cbsonmodule.h index 3be2b7442..a9bee24b8 100644 --- a/bson/_cbsonmodule.h +++ b/bson/_cbsonmodule.h @@ -72,6 +72,7 @@ typedef struct codec_options_t { unsigned char datetime_conversion; PyObject* options_obj; unsigned char is_raw_bson; + unsigned char is_dict_class; } codec_options_t; /* C API functions */ diff --git a/doc/changelog.rst b/doc/changelog.rst index 571ce3b63..f38709203 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +Changes in Version 4.17.0 (2026/XX/XX) +-------------------------------------- + +PyMongo 4.17 brings a number of changes including: + +- Added the :meth:`~pymongo.asynchronous.client_session.AsyncClientSession.bind` and :meth:`~pymongo.client_session.ClientSession.bind` methods + that allow users to bind a session to all database operations within the scope of a context manager instead of having to explicitly pass the session to each individual operation. + See for examples and more information. + Changes in Version 4.16.0 (2026/01/07) -------------------------------------- diff --git a/justfile b/justfile index 082b6ea17..3a8e216db 100644 --- a/justfile +++ b/justfile @@ -57,11 +57,14 @@ lint-manual *args="": && resync [group('test')] test *args="-v --durations=5 --maxfail=10": && resync - uv run --extra test python -m pytest {{args}} + #!/usr/bin/env bash + set -euo pipefail + uv run ${USE_ACTIVE_VENV:+--active} --extra test python -m pytest {{args}} [group('test')] -test-numpy: && resync - uv run --extra test --with numpy python -m pytest test/test_bson.py +test-numpy *args="": && resync + just setup-tests numpy {{args}} + just run-tests test/test_bson.py [group('test')] run-tests *args: && resync @@ -79,6 +82,25 @@ teardown-tests: integration-tests: bash integration_tests/run.sh +[group('test')] +test-coverage *args="": + just setup-tests --cov + just run-tests {{args}} + +[group('coverage')] +coverage-report: + uv tool run --with "coverage[toml]" coverage report + +[group('coverage')] +coverage-html: + uv tool run --with "coverage[toml]" coverage html + @echo "Coverage report generated in htmlcov/index.html" + +[group('coverage')] +coverage-xml: + uv tool run --with "coverage[toml]" coverage xml + @echo "Coverage report generated in coverage.xml" + [group('server')] run-server *args="": bash .evergreen/scripts/run-server.sh {{args}} diff --git a/pymongo/asynchronous/client_session.py b/pymongo/asynchronous/client_session.py index c72e82884..8212d1396 100644 --- a/pymongo/asynchronous/client_session.py +++ b/pymongo/asynchronous/client_session.py @@ -141,6 +141,7 @@ import random import time import uuid from collections.abc import Mapping as _Mapping +from contextvars import ContextVar, Token from typing import ( TYPE_CHECKING, Any, @@ -185,6 +186,28 @@ if TYPE_CHECKING: _IS_SYNC = False +_SESSION: ContextVar[Optional[AsyncClientSession]] = ContextVar("SESSION", default=None) + + +class _AsyncBoundSessionContext: + """Context manager returned by AsyncClientSession.bind() that manages bound state.""" + + def __init__(self, session: AsyncClientSession, end_session: bool) -> None: + self._session = session + self._session_token: Optional[Token[AsyncClientSession]] = None + self._end_session = end_session + + async def __aenter__(self) -> AsyncClientSession: + self._session_token = _SESSION.set(self._session) # type: ignore[assignment] + return self._session + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + if self._session_token: + _SESSION.reset(self._session_token) # type: ignore[arg-type] + self._session_token = None + if self._end_session: + await self._session.end_session() + class SessionOptions: """Options for a new :class:`AsyncClientSession`. @@ -568,6 +591,24 @@ class AsyncClientSession: if self._server_session is None: raise InvalidOperation("Cannot use ended session") + def bind(self, end_session: bool = True) -> _AsyncBoundSessionContext: + """Bind this session so it is implicitly passed to all database operations within the returned context. + + .. code-block:: python + + async with client.start_session() as s: + async with s.bind(): + # session=s is passed implicitly + await client.db.collection.insert_one({"x": 1}) + + :param end_session: Whether to end the session on exiting the returned context. Defaults to True. + If set to False, :meth:`~pymongo.asynchronous.client_session.AsyncClientSession.end_session()` must be called + once the session is no longer used. + + .. versionadded:: 4.17 + """ + return _AsyncBoundSessionContext(self, end_session) + async def __aenter__(self) -> AsyncClientSession: return self diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 5c8251b84..8936068af 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -66,7 +66,7 @@ from pymongo import _csot, common, helpers_shared, periodic_executor from pymongo.asynchronous import client_session, database, uri_parser from pymongo.asynchronous.change_stream import AsyncChangeStream, AsyncClusterChangeStream from pymongo.asynchronous.client_bulk import _AsyncClientBulk -from pymongo.asynchronous.client_session import _EmptyServerSession +from pymongo.asynchronous.client_session import _SESSION, _EmptyServerSession from pymongo.asynchronous.command_cursor import AsyncCommandCursor from pymongo.asynchronous.helpers import ( _RetryPolicy, @@ -1428,7 +1428,8 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): def _ensure_session( self, session: Optional[AsyncClientSession] = None ) -> Optional[AsyncClientSession]: - """If provided session is None, lend a temporary session.""" + """If provided session and bound session are None, lend a temporary session.""" + session = session or self._get_bound_session() if session: return session @@ -2299,11 +2300,14 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): self, session: Optional[client_session.AsyncClientSession] ) -> AsyncGenerator[Optional[client_session.AsyncClientSession], None]: """If provided session is None, lend a temporary session.""" - if session is not None: - if not isinstance(session, client_session.AsyncClientSession): - raise ValueError( - f"'session' argument must be an AsyncClientSession or None, not {type(session)}" - ) + if session is not None and not isinstance(session, client_session.AsyncClientSession): + raise ValueError( + f"'session' argument must be an AsyncClientSession or None, not {type(session)}" + ) + + # Check for a bound session. If one exists, treat it as an explicitly passed session. + session = session or self._get_bound_session() + if session: # Don't call end_session. yield session return @@ -2333,6 +2337,18 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): if session is not None: session._process_response(reply) + def _get_bound_session(self) -> Optional[AsyncClientSession]: + bound_session = _SESSION.get() + if bound_session: + if bound_session.client is self: + return bound_session + else: + raise InvalidOperation( + "Only the client that created the bound session can perform operations within its context block. See for more information." + ) + else: + return None + async def server_info( self, session: Optional[client_session.AsyncClientSession] = None ) -> dict[str, Any]: @@ -2914,7 +2930,11 @@ class _ClientConnectionRetryable(Generic[T]): transaction.set_starting() transaction.attempt = 0 - if self._server is not None: + if ( + self._server is not None + and self._client.topology_description.topology_type_name == "Sharded" + or exc.has_error_label("SystemOverloadedError") + ): self._deprioritized_servers.append(self._server) self._always_retryable = always_retryable diff --git a/pymongo/pyopenssl_context.py b/pymongo/pyopenssl_context.py index 67456d553..941d91cd1 100644 --- a/pymongo/pyopenssl_context.py +++ b/pymongo/pyopenssl_context.py @@ -357,7 +357,7 @@ class SSLContext: try: for storename in ("CA", "ROOT"): self._load_wincerts(storename) - except PermissionError: + except Exception: # Fall back to certifi self._load_certifi() elif _sys.platform == "darwin": diff --git a/pymongo/synchronous/client_session.py b/pymongo/synchronous/client_session.py index 2467bc71b..f8211a60b 100644 --- a/pymongo/synchronous/client_session.py +++ b/pymongo/synchronous/client_session.py @@ -140,6 +140,7 @@ import random import time import uuid from collections.abc import Mapping as _Mapping +from contextvars import ContextVar, Token from typing import ( TYPE_CHECKING, Any, @@ -183,6 +184,28 @@ if TYPE_CHECKING: _IS_SYNC = True +_SESSION: ContextVar[Optional[ClientSession]] = ContextVar("SESSION", default=None) + + +class _BoundSessionContext: + """Context manager returned by ClientSession.bind() that manages bound state.""" + + def __init__(self, session: ClientSession, end_session: bool) -> None: + self._session = session + self._session_token: Optional[Token[ClientSession]] = None + self._end_session = end_session + + def __enter__(self) -> ClientSession: + self._session_token = _SESSION.set(self._session) # type: ignore[assignment] + return self._session + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + if self._session_token: + _SESSION.reset(self._session_token) # type: ignore[arg-type] + self._session_token = None + if self._end_session: + self._session.end_session() + class SessionOptions: """Options for a new :class:`ClientSession`. @@ -566,6 +589,24 @@ class ClientSession: if self._server_session is None: raise InvalidOperation("Cannot use ended session") + def bind(self, end_session: bool = True) -> _BoundSessionContext: + """Bind this session so it is implicitly passed to all database operations within the returned context. + + .. code-block:: python + + with client.start_session() as s: + with s.bind(): + # session=s is passed implicitly + client.db.collection.insert_one({"x": 1}) + + :param end_session: Whether to end the session on exiting the returned context. Defaults to True. + If set to False, :meth:`~pymongo.client_session.ClientSession.end_session()` must be called + once the session is no longer used. + + .. versionadded:: 4.17 + """ + return _BoundSessionContext(self, end_session) + def __enter__(self) -> ClientSession: return self diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 9cec36a75..ab7465078 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -109,7 +109,7 @@ from pymongo.server_type import SERVER_TYPE from pymongo.synchronous import client_session, database, uri_parser from pymongo.synchronous.change_stream import ChangeStream, ClusterChangeStream from pymongo.synchronous.client_bulk import _ClientBulk -from pymongo.synchronous.client_session import _EmptyServerSession +from pymongo.synchronous.client_session import _SESSION, _EmptyServerSession from pymongo.synchronous.command_cursor import CommandCursor from pymongo.synchronous.helpers import ( _RetryPolicy, @@ -1426,7 +1426,8 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): ) def _ensure_session(self, session: Optional[ClientSession] = None) -> Optional[ClientSession]: - """If provided session is None, lend a temporary session.""" + """If provided session and bound session are None, lend a temporary session.""" + session = session or self._get_bound_session() if session: return session @@ -2295,11 +2296,14 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): self, session: Optional[client_session.ClientSession] ) -> Generator[Optional[client_session.ClientSession], None]: """If provided session is None, lend a temporary session.""" - if session is not None: - if not isinstance(session, client_session.ClientSession): - raise ValueError( - f"'session' argument must be a ClientSession or None, not {type(session)}" - ) + if session is not None and not isinstance(session, client_session.ClientSession): + raise ValueError( + f"'session' argument must be a ClientSession or None, not {type(session)}" + ) + + # Check for a bound session. If one exists, treat it as an explicitly passed session. + session = session or self._get_bound_session() + if session: # Don't call end_session. yield session return @@ -2327,6 +2331,18 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): if session is not None: session._process_response(reply) + def _get_bound_session(self) -> Optional[ClientSession]: + bound_session = _SESSION.get() + if bound_session: + if bound_session.client is self: + return bound_session + else: + raise InvalidOperation( + "Only the client that created the bound session can perform operations within its context block. See for more information." + ) + else: + return None + def server_info(self, session: Optional[client_session.ClientSession] = None) -> dict[str, Any]: """Get information about the MongoDB server we're connected to. @@ -2904,7 +2920,11 @@ class _ClientConnectionRetryable(Generic[T]): transaction.set_starting() transaction.attempt = 0 - if self._server is not None: + if ( + self._server is not None + and self._client.topology_description.topology_type_name == "Sharded" + or exc.has_error_label("SystemOverloadedError") + ): self._deprioritized_servers.append(self._server) self._always_retryable = always_retryable diff --git a/pyproject.toml b/pyproject.toml index 65cbeca8b..9b3287834 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ dev = [] pip = ["pip>=20.2"] gevent = ["gevent>=21.12"] coverage = [ - "pytest-cov>=4.0.0", "coverage[toml]>=5,<=7.10.7" ] mockupdb = [ @@ -239,7 +238,11 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?)|dummy.*)$" [tool.coverage.run] branch = true -source = ["pymongo", "bson", "gridfs" ] +include = [ + "pymongo/*", + "bson/*", + "gridfs/*" +] relative_files = true [tool.coverage.report] diff --git a/test/asynchronous/test_auth_oidc.py b/test/asynchronous/test_auth_oidc.py index ff604f55a..3567d7706 100644 --- a/test/asynchronous/test_auth_oidc.py +++ b/test/asynchronous/test_auth_oidc.py @@ -104,7 +104,7 @@ class OIDCTestBase(AsyncPyMongoTestCase): @asynccontextmanager async def fail_point(self, command_args): - cmd_on = SON([("configureFailPoint", "failCommand")]) + cmd_on = dict(configureFailPoint="failCommand", appName="auth_oidc") cmd_on.update(command_args) client = AsyncMongoClient(self.uri_admin) await client.admin.command(cmd_on) @@ -112,7 +112,7 @@ class OIDCTestBase(AsyncPyMongoTestCase): yield finally: await client.admin.command( - "configureFailPoint", cmd_on["configureFailPoint"], mode="off" + "configureFailPoint", cmd_on["configureFailPoint"], mode="off", appName="auth_oidc" ) await client.close() diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index b1dbc73f3..9650f7043 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -876,6 +876,8 @@ class TestViews(AsyncEncryptionIntegrationTest): class TestCorpus(AsyncEncryptionIntegrationTest): + # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. + @async_client_context.require_version_max(6, 99) @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") async def asyncSetUp(self): await super().asyncSetUp() @@ -1052,6 +1054,8 @@ class TestBsonSizeBatches(AsyncEncryptionIntegrationTest): client_encrypted: AsyncMongoClient listener: OvertCommandListener + # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. + @async_client_context.require_version_max(6, 99) async def asyncSetUp(self): await super().asyncSetUp() db = async_client_context.client.db diff --git a/test/asynchronous/test_retryable_reads.py b/test/asynchronous/test_retryable_reads.py index 47ac91b0f..6adfaaae1 100644 --- a/test/asynchronous/test_retryable_reads.py +++ b/test/asynchronous/test_retryable_reads.py @@ -261,6 +261,84 @@ class TestRetryableReads(AsyncIntegrationTest): self.assertEqual(command_docs[0]["lsid"], command_docs[1]["lsid"]) self.assertIsNot(command_docs[0], command_docs[1]) + @async_client_context.require_replica_set + @async_client_context.require_secondaries_count(1) + @async_client_context.require_failCommand_fail_point + @async_client_context.require_version_min(4, 4, 0) + async def test_03_01_retryable_reads_caused_by_overload_errors_are_retried_on_a_different_replicaset_server_when_one_is_available( + self + ): + listener = OvertCommandListener() + + # 1. Create a client `client` with `retryReads=true`, `readPreference=primaryPreferred`, and command event monitoring enabled. + client = await self.async_rs_or_single_client( + event_listeners=[listener], retryReads=True, readPreference="primaryPreferred" + ) + + # 2. Configure a fail point with the RetryableError and SystemOverloadedError error labels. + command_args = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find"], + "errorLabels": ["RetryableError", "SystemOverloadedError"], + "errorCode": 6, + }, + } + await async_set_fail_point(client, command_args) + + # 3. Reset the command event monitor to clear the fail point command from its stored events. + listener.reset() + + # 4. Execute a `find` command with `client`. + await client.t.t.find_one({}) + + # 5. Assert that one failed command event and one successful command event occurred. + self.assertEqual(len(listener.failed_events), 1) + self.assertEqual(len(listener.succeeded_events), 1) + + # 6. Assert that both events occurred on different servers. + assert listener.failed_events[0].connection_id != listener.succeeded_events[0].connection_id + + @async_client_context.require_replica_set + @async_client_context.require_secondaries_count(1) + @async_client_context.require_failCommand_fail_point + @async_client_context.require_version_min(4, 4, 0) + async def test_03_02_retryable_reads_caused_by_non_overload_errors_are_retried_on_the_same_replicaset_server( + self + ): + listener = OvertCommandListener() + + # 1. Create a client `client` with `retryReads=true`, `readPreference=primaryPreferred`, and command event monitoring enabled. + client = await self.async_rs_or_single_client( + event_listeners=[listener], retryReads=True, readPreference="primaryPreferred" + ) + + # 2. Configure a fail point with the RetryableError error label. + command_args = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find"], + "errorLabels": ["RetryableError"], + "errorCode": 6, + }, + } + await async_set_fail_point(client, command_args) + + # 3. Reset the command event monitor to clear the fail point command from its stored events. + listener.reset() + + # 4. Execute a `find` command with `client`. + await client.t.t.find_one({}) + + # 5. Assert that one failed command event and one successful command event occurred. + self.assertEqual(len(listener.failed_events), 1) + self.assertEqual(len(listener.succeeded_events), 1) + + # 6. Assert that both events occurred the same server. + assert listener.failed_events[0].connection_id == listener.succeeded_events[0].connection_id + if __name__ == "__main__": unittest.main() diff --git a/test/asynchronous/test_session.py b/test/asynchronous/test_session.py index 19ce868c5..404a69fde 100644 --- a/test/asynchronous/test_session.py +++ b/test/asynchronous/test_session.py @@ -189,6 +189,52 @@ class TestSession(AsyncIntegrationTest): f"{f.__name__} did not return implicit session to pool", ) + # Explicit bound session + for f, args, kw in ops: + async with client.start_session() as s: + async with s.bind(): + listener.reset() + s._materialize() + last_use = s._server_session.last_use + start = time.monotonic() + self.assertLessEqual(last_use, start) + # In case "f" modifies its inputs. + args = copy.copy(args) + kw = copy.copy(kw) + await f(*args, **kw) + self.assertGreaterEqual(len(listener.started_events), 1) + for event in listener.started_events: + self.assertIn( + "lsid", + event.command, + f"{f.__name__} sent no lsid with {event.command_name}", + ) + + self.assertEqual( + s.session_id, + event.command["lsid"], + f"{f.__name__} sent wrong lsid with {event.command_name}", + ) + + self.assertFalse(s.has_ended) + + self.assertTrue(s.has_ended) + with self.assertRaisesRegex(InvalidOperation, "ended session"): + async with s.bind(): + await f(*args, **kw) + + # Test a session cannot be used on another client. + async with self.client2.start_session() as s: + async with s.bind(): + # In case "f" modifies its inputs. + args = copy.copy(args) + kw = copy.copy(kw) + with self.assertRaisesRegex( + InvalidOperation, + "Only the client that created the bound session can perform operations within its context block", + ): + await f(*args, **kw) + async def test_implicit_sessions_checkout(self): # "To confirm that implicit sessions only allocate their server session after a # successful connection checkout" test from Driver Sessions Spec. @@ -825,6 +871,73 @@ class TestSession(AsyncIntegrationTest): async with client.start_session() as s: self.assertRaises(TypeError, lambda: copy.copy(s)) + async def test_nested_session_binding(self): + coll = self.client.pymongo_test.test + await coll.insert_one({"x": 1}) + + session1 = self.client.start_session() + session2 = self.client.start_session() + session1._materialize() + session2._materialize() + try: + self.listener.reset() + # Uses implicit session + await coll.find_one() + implicit_lsid = self.listener.started_events[0].command.get("lsid") + self.assertIsNotNone(implicit_lsid) + self.assertNotEqual(implicit_lsid, session1.session_id) + self.assertNotEqual(implicit_lsid, session2.session_id) + + async with session1.bind(end_session=False): + self.listener.reset() + # Uses bound session1 + await coll.find_one() + session1_lsid = self.listener.started_events[0].command.get("lsid") + self.assertEqual(session1_lsid, session1.session_id) + + async with session2.bind(end_session=False): + self.listener.reset() + # Uses bound session2 + await coll.find_one() + session2_lsid = self.listener.started_events[0].command.get("lsid") + self.assertEqual(session2_lsid, session2.session_id) + self.assertNotEqual(session2_lsid, session1.session_id) + + self.listener.reset() + # Use bound session1 again + await coll.find_one() + session1_lsid = self.listener.started_events[0].command.get("lsid") + self.assertEqual(session1_lsid, session1.session_id) + self.assertNotEqual(session1_lsid, session2.session_id) + + self.listener.reset() + # Uses implicit session + await coll.find_one() + implicit_lsid = self.listener.started_events[0].command.get("lsid") + self.assertIsNotNone(implicit_lsid) + self.assertNotEqual(implicit_lsid, session1.session_id) + self.assertNotEqual(implicit_lsid, session2.session_id) + + finally: + await session1.end_session() + await session2.end_session() + + async def test_session_binding_end_session(self): + coll = self.client.pymongo_test.test + await coll.insert_one({"x": 1}) + + async with self.client.start_session().bind() as s1: + await coll.find_one() + + self.assertTrue(s1.has_ended) + + async with self.client.start_session().bind(end_session=False) as s2: + await coll.find_one() + + self.assertFalse(s2.has_ended) + + await s2.end_session() + class TestCausalConsistency(AsyncUnitTest): listener: SessionTestListener diff --git a/test/asynchronous/test_ssl.py b/test/asynchronous/test_ssl.py index 0ce3e8bba..7fe57e850 100644 --- a/test/asynchronous/test_ssl.py +++ b/test/asynchronous/test_ssl.py @@ -48,19 +48,11 @@ from pymongo.write_concern import WriteConcern _HAVE_PYOPENSSL = False try: - # All of these must be available to use PyOpenSSL - import OpenSSL - import requests - import service_identity - - # Ensure service_identity>=18.1 is installed - from service_identity.pyopenssl import verify_ip_address - - from pymongo.ocsp_support import _load_trusted_ca_certs + from pymongo import pyopenssl_context _HAVE_PYOPENSSL = True except ImportError: - _load_trusted_ca_certs = None # type: ignore + pass if HAVE_SSL: @@ -136,11 +128,6 @@ class TestClientSSL(AsyncPyMongoTestCase): def test_use_pyopenssl_when_available(self): self.assertTrue(HAVE_PYSSL) - @unittest.skipUnless(_HAVE_PYOPENSSL, "Cannot test without PyOpenSSL") - def test_load_trusted_ca_certs(self): - trusted_ca_certs = _load_trusted_ca_certs(CA_BUNDLE_PEM) - self.assertEqual(2, len(trusted_ca_certs)) - class TestSSL(AsyncIntegrationTest): saved_port: int diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 6ce8f852c..1fb93e7b8 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1464,11 +1464,6 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): self.assertListEqual(sorted_expected_documents, actual_documents) async def run_scenario(self, spec, uri=None): - # Kill all sessions before and after each test to prevent an open - # transaction (from a test failure) from blocking collection/database - # operations during test set up and tear down. - await self.kill_all_sessions() - # Handle flaky tests. flaky_tests = [ ("PYTHON-5170", ".*test_discovery_and_monitoring.*"), @@ -1504,6 +1499,15 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): if skip_reason is not None: raise unittest.SkipTest(f"{skip_reason}") + # Kill all sessions after each test with transactions to prevent an open + # transaction (from a test failure) from blocking collection/database + # operations during test set up and tear down. + for op in spec["operations"]: + name = op["name"] + if name == "startTransaction" or name == "withTransaction": + self.addAsyncCleanup(self.kill_all_sessions) + break + # process createEntities self._uri = uri self.entity_map = EntityMapUtil(self) diff --git a/test/asynchronous/utils_spec_runner.py b/test/asynchronous/utils_spec_runner.py index 63e7e9e15..ff5f61db0 100644 --- a/test/asynchronous/utils_spec_runner.py +++ b/test/asynchronous/utils_spec_runner.py @@ -16,43 +16,13 @@ from __future__ import annotations import asyncio -import functools import os -import time -import unittest -from collections import abc -from inspect import iscoroutinefunction -from test.asynchronous import AsyncIntegrationTest, async_client_context, client_knobs +from test.asynchronous import async_client_context from test.asynchronous.helpers import ConcurrentRunner -from test.utils_shared import ( - CMAPListener, - CompareType, - EventListener, - OvertCommandListener, - ScenarioDict, - ServerAndTopologyEventListener, - camel_to_snake, - camel_to_snake_args, - parse_spec_options, - prepare_spec_arguments, -) -from typing import List +from test.utils_shared import ScenarioDict -from bson import ObjectId, decode, encode, json_util -from bson.binary import Binary -from bson.int64 import Int64 -from bson.son import SON -from gridfs import GridFSBucket -from gridfs.asynchronous.grid_file import AsyncGridFSBucket -from pymongo.asynchronous import client_session -from pymongo.asynchronous.command_cursor import AsyncCommandCursor -from pymongo.asynchronous.cursor import AsyncCursor -from pymongo.errors import AutoReconnect, BulkWriteError, OperationFailure, PyMongoError +from bson import json_util from pymongo.lock import _async_cond_wait, _async_create_condition, _async_create_lock -from pymongo.read_concern import ReadConcern -from pymongo.read_preferences import ReadPreference -from pymongo.results import BulkWriteResult, _WriteResult -from pymongo.write_concern import WriteConcern _IS_SYNC = False @@ -219,597 +189,3 @@ class AsyncSpecTestCreator: self._create_tests() else: asyncio.run(self._create_tests()) - - -class AsyncSpecRunner(AsyncIntegrationTest): - mongos_clients: List - knobs: client_knobs - listener: EventListener - - async def asyncSetUp(self) -> None: - await super().asyncSetUp() - self.mongos_clients = [] - - # Speed up the tests by decreasing the heartbeat frequency. - self.knobs = client_knobs(heartbeat_frequency=0.1, min_heartbeat_interval=0.1) - self.knobs.enable() - self.targets = {} - self.listener = None # type: ignore - self.pool_listener = None - self.server_listener = None - self.maxDiff = None - - async def asyncTearDown(self) -> None: - self.knobs.disable() - - async def set_fail_point(self, command_args): - clients = self.mongos_clients if self.mongos_clients else [self.client] - for client in clients: - await self.configure_fail_point(client, command_args) - - async def targeted_fail_point(self, session, fail_point): - """Run the targetedFailPoint test operation. - - Enable the fail point on the session's pinned mongos. - """ - clients = {c.address: c for c in self.mongos_clients} - client = clients[session._pinned_address] - await self.configure_fail_point(client, fail_point) - self.addAsyncCleanup(self.set_fail_point, {"mode": "off"}) - - def assert_session_pinned(self, session): - """Run the assertSessionPinned test operation. - - Assert that the given session is pinned. - """ - self.assertIsNotNone(session._transaction.pinned_address) - - def assert_session_unpinned(self, session): - """Run the assertSessionUnpinned test operation. - - Assert that the given session is not pinned. - """ - self.assertIsNone(session._pinned_address) - self.assertIsNone(session._transaction.pinned_address) - - async def assert_collection_exists(self, database, collection): - """Run the assertCollectionExists test operation.""" - db = self.client[database] - self.assertIn(collection, await db.list_collection_names()) - - async def assert_collection_not_exists(self, database, collection): - """Run the assertCollectionNotExists test operation.""" - db = self.client[database] - self.assertNotIn(collection, await db.list_collection_names()) - - async def assert_index_exists(self, database, collection, index): - """Run the assertIndexExists test operation.""" - coll = self.client[database][collection] - self.assertIn(index, [doc["name"] async for doc in await coll.list_indexes()]) - - async def assert_index_not_exists(self, database, collection, index): - """Run the assertIndexNotExists test operation.""" - coll = self.client[database][collection] - self.assertNotIn(index, [doc["name"] async for doc in await coll.list_indexes()]) - - async def wait(self, ms): - """Run the "wait" test operation.""" - await asyncio.sleep(ms / 1000.0) - - def assertErrorLabelsContain(self, exc, expected_labels): - labels = [l for l in expected_labels if exc.has_error_label(l)] - self.assertEqual(labels, expected_labels) - - def assertErrorLabelsOmit(self, exc, omit_labels): - for label in omit_labels: - self.assertFalse( - exc.has_error_label(label), msg=f"error labels should not contain {label}" - ) - - async def kill_all_sessions(self): - clients = self.mongos_clients if self.mongos_clients else [self.client] - for client in clients: - try: - await client.admin.command("killAllSessions", []) - except (OperationFailure, AutoReconnect): - # "operation was interrupted" by killing the command's - # own session. - # On 8.0+ killAllSessions sometimes returns a network error. - pass - - def check_command_result(self, expected_result, result): - # Only compare the keys in the expected result. - filtered_result = {} - for key in expected_result: - try: - filtered_result[key] = result[key] - except KeyError: - pass - self.assertEqual(filtered_result, expected_result) - - # TODO: factor the following function with test_crud.py. - def check_result(self, expected_result, result): - if isinstance(result, _WriteResult): - for res in expected_result: - prop = camel_to_snake(res) - # SPEC-869: Only BulkWriteResult has upserted_count. - if prop == "upserted_count" and not isinstance(result, BulkWriteResult): - if result.upserted_id is not None: - upserted_count = 1 - else: - upserted_count = 0 - self.assertEqual(upserted_count, expected_result[res], prop) - elif prop == "inserted_ids": - # BulkWriteResult does not have inserted_ids. - if isinstance(result, BulkWriteResult): - self.assertEqual(len(expected_result[res]), result.inserted_count) - else: - # InsertManyResult may be compared to [id1] from the - # crud spec or {"0": id1} from the retryable write spec. - ids = expected_result[res] - if isinstance(ids, dict): - ids = [ids[str(i)] for i in range(len(ids))] - - self.assertEqual(ids, result.inserted_ids, prop) - elif prop == "upserted_ids": - # Convert indexes from strings to integers. - ids = expected_result[res] - expected_ids = {} - for str_index in ids: - expected_ids[int(str_index)] = ids[str_index] - self.assertEqual(expected_ids, result.upserted_ids, prop) - else: - self.assertEqual(getattr(result, prop), expected_result[res], prop) - - return True - else: - - def _helper(expected_result, result): - if isinstance(expected_result, abc.Mapping): - for i in expected_result.keys(): - self.assertEqual(expected_result[i], result[i]) - - elif isinstance(expected_result, list): - for i, k in zip(expected_result, result): - _helper(i, k) - else: - self.assertEqual(expected_result, result) - - _helper(expected_result, result) - return None - - def get_object_name(self, op): - """Allow subclasses to override handling of 'object' - - Transaction spec says 'object' is required. - """ - return op["object"] - - @staticmethod - def parse_options(opts): - return parse_spec_options(opts) - - async def run_operation(self, sessions, collection, operation): - original_collection = collection - name = camel_to_snake(operation["name"]) - if name == "run_command": - name = "command" - elif name == "download_by_name": - name = "open_download_stream_by_name" - elif name == "download": - name = "open_download_stream" - elif name == "map_reduce": - self.skipTest("PyMongo does not support mapReduce") - elif name == "count": - self.skipTest("PyMongo does not support count") - - database = collection.database - collection = database.get_collection(collection.name) - if "collectionOptions" in operation: - collection = collection.with_options( - **self.parse_options(operation["collectionOptions"]) - ) - - object_name = self.get_object_name(operation) - if object_name == "gridfsbucket": - # Only create the GridFSBucket when we need it (for the gridfs - # retryable reads tests). - obj = AsyncGridFSBucket(database, bucket_name=collection.name) - else: - objects = { - "client": database.client, - "database": database, - "collection": collection, - "testRunner": self, - } - objects.update(sessions) - obj = objects[object_name] - - # Combine arguments with options and handle special cases. - arguments = operation.get("arguments", {}) - arguments.update(arguments.pop("options", {})) - self.parse_options(arguments) - - cmd = getattr(obj, name) - - with_txn_callback = functools.partial( - self.run_operations, sessions, original_collection, in_with_transaction=True - ) - prepare_spec_arguments(operation, arguments, name, sessions, with_txn_callback) - - if name == "run_on_thread": - args = {"sessions": sessions, "collection": collection} - args.update(arguments) - arguments = args - - if not _IS_SYNC and iscoroutinefunction(cmd): - result = await cmd(**dict(arguments)) - else: - result = cmd(**dict(arguments)) - # Cleanup open change stream cursors. - if name == "watch": - self.addAsyncCleanup(result.close) - - if name == "aggregate": - if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: - # Read from the primary to ensure causal consistency. - out = collection.database.get_collection( - arguments["pipeline"][-1]["$out"], read_preference=ReadPreference.PRIMARY - ) - return out.find() - if "download" in name: - result = Binary(result.read()) - - if isinstance(result, AsyncCursor) or isinstance(result, AsyncCommandCursor): - return await result.to_list() - - return result - - def allowable_errors(self, op): - """Allow encryption spec to override expected error classes.""" - return (PyMongoError,) - - async def _run_op(self, sessions, collection, op, in_with_transaction): - expected_result = op.get("result") - if expect_error(op): - with self.assertRaises(self.allowable_errors(op), msg=op["name"]) as context: - await self.run_operation(sessions, collection, op.copy()) - exc = context.exception - if expect_error_message(expected_result): - if isinstance(exc, BulkWriteError): - errmsg = str(exc.details).lower() - else: - errmsg = str(exc).lower() - self.assertIn(expected_result["errorContains"].lower(), errmsg) - if expect_error_code(expected_result): - self.assertEqual(expected_result["errorCodeName"], exc.details.get("codeName")) - if expect_error_labels_contain(expected_result): - self.assertErrorLabelsContain(exc, expected_result["errorLabelsContain"]) - if expect_error_labels_omit(expected_result): - self.assertErrorLabelsOmit(exc, expected_result["errorLabelsOmit"]) - if expect_timeout_error(expected_result): - self.assertIsInstance(exc, PyMongoError) - if not exc.timeout: - # Re-raise the exception for better diagnostics. - raise exc - - # Reraise the exception if we're in the with_transaction - # callback. - if in_with_transaction: - raise context.exception - else: - result = await self.run_operation(sessions, collection, op.copy()) - if "result" in op: - if op["name"] == "runCommand": - self.check_command_result(expected_result, result) - else: - self.check_result(expected_result, result) - - async def run_operations(self, sessions, collection, ops, in_with_transaction=False): - for op in ops: - await self._run_op(sessions, collection, op, in_with_transaction) - - # TODO: factor with test_command_monitoring.py - def check_events(self, test, listener, session_ids): - events = listener.started_events - if not len(test["expectations"]): - return - - # Give a nicer message when there are missing or extra events - cmds = decode_raw([event.command for event in events]) - self.assertEqual(len(events), len(test["expectations"]), cmds) - for i, expectation in enumerate(test["expectations"]): - event_type = next(iter(expectation)) - event = events[i] - - # The tests substitute 42 for any number other than 0. - if event.command_name == "getMore" and event.command["getMore"]: - event.command["getMore"] = Int64(42) - elif event.command_name == "killCursors": - event.command["cursors"] = [Int64(42)] - elif event.command_name == "update": - # TODO: remove this once PYTHON-1744 is done. - # Add upsert and multi fields back into expectations. - updates = expectation[event_type]["command"]["updates"] - for update in updates: - update.setdefault("upsert", False) - update.setdefault("multi", False) - - # Replace afterClusterTime: 42 with actual afterClusterTime. - expected_cmd = expectation[event_type]["command"] - expected_read_concern = expected_cmd.get("readConcern") - if expected_read_concern is not None: - time = expected_read_concern.get("afterClusterTime") - if time == 42: - actual_time = event.command.get("readConcern", {}).get("afterClusterTime") - if actual_time is not None: - expected_read_concern["afterClusterTime"] = actual_time - - recovery_token = expected_cmd.get("recoveryToken") - if recovery_token == 42: - expected_cmd["recoveryToken"] = CompareType(dict) - - # Replace lsid with a name like "session0" to match test. - if "lsid" in event.command: - for name, lsid in session_ids.items(): - if event.command["lsid"] == lsid: - event.command["lsid"] = name - break - - for attr, expected in expectation[event_type].items(): - actual = getattr(event, attr) - expected = wrap_types(expected) - if isinstance(expected, dict): - for key, val in expected.items(): - if val is None: - if key in actual: - self.fail(f"Unexpected key [{key}] in {actual!r}") - elif key not in actual: - self.fail(f"Expected key [{key}] in {actual!r}") - else: - self.assertEqual( - val, decode_raw(actual[key]), f"Key [{key}] in {actual}" - ) - else: - self.assertEqual(actual, expected) - - def maybe_skip_scenario(self, test): - if test.get("skipReason"): - self.skipTest(test.get("skipReason")) - - def get_scenario_db_name(self, scenario_def): - """Allow subclasses to override a test's database name.""" - return scenario_def["database_name"] - - def get_scenario_coll_name(self, scenario_def): - """Allow subclasses to override a test's collection name.""" - return scenario_def["collection_name"] - - def get_outcome_coll_name(self, outcome, collection): - """Allow subclasses to override outcome collection.""" - return collection.name - - async def run_test_ops(self, sessions, collection, test): - """Added to allow retryable writes spec to override a test's - operation. - """ - await self.run_operations(sessions, collection, test["operations"]) - - def parse_client_options(self, opts): - """Allow encryption spec to override a clientOptions parsing.""" - return opts - - async def setup_scenario(self, scenario_def): - """Allow specs to override a test's setup.""" - db_name = self.get_scenario_db_name(scenario_def) - coll_name = self.get_scenario_coll_name(scenario_def) - documents = scenario_def["data"] - - # Setup the collection with as few majority writes as possible. - db = async_client_context.client.get_database(db_name) - coll_exists = bool(await db.list_collection_names(filter={"name": coll_name})) - if coll_exists: - await db[coll_name].delete_many({}) - # Only use majority wc only on the final write. - wc = WriteConcern(w="majority") - if documents: - db.get_collection(coll_name, write_concern=wc).insert_many(documents) - elif not coll_exists: - # Ensure collection exists. - await db.create_collection(coll_name, write_concern=wc) - - async def run_scenario(self, scenario_def, test): - self.maybe_skip_scenario(test) - - # Kill all sessions before and after each test to prevent an open - # transaction (from a test failure) from blocking collection/database - # operations during test set up and tear down. - await self.kill_all_sessions() - self.addAsyncCleanup(self.kill_all_sessions) - await self.setup_scenario(scenario_def) - database_name = self.get_scenario_db_name(scenario_def) - collection_name = self.get_scenario_coll_name(scenario_def) - # SPEC-1245 workaround StaleDbVersion on distinct - for c in self.mongos_clients: - await c[database_name][collection_name].distinct("x") - - # Configure the fail point before creating the client. - if "failPoint" in test: - fp = test["failPoint"] - await self.set_fail_point(fp) - self.addAsyncCleanup( - self.set_fail_point, {"configureFailPoint": fp["configureFailPoint"], "mode": "off"} - ) - - listener = OvertCommandListener() - pool_listener = CMAPListener() - server_listener = ServerAndTopologyEventListener() - # Create a new client, to avoid interference from pooled sessions. - client_options = self.parse_client_options(test["clientOptions"]) - use_multi_mongos = test["useMultipleMongoses"] - host = None - if use_multi_mongos: - if async_client_context.load_balancer: - host = async_client_context.MULTI_MONGOS_LB_URI - elif async_client_context.is_mongos: - host = async_client_context.mongos_seeds() - client = await self.async_rs_client( - h=host, event_listeners=[listener, pool_listener, server_listener], **client_options - ) - self.scenario_client = client - self.listener = listener - self.pool_listener = pool_listener - self.server_listener = server_listener - - # Create session0 and session1. - sessions = {} - session_ids = {} - for i in range(2): - # Don't attempt to create sessions if they are not supported by - # the running server version. - if not async_client_context.sessions_enabled: - break - session_name = "session%d" % i - opts = camel_to_snake_args(test["sessionOptions"][session_name]) - if "default_transaction_options" in opts: - txn_opts = self.parse_options(opts["default_transaction_options"]) - txn_opts = client_session.TransactionOptions(**txn_opts) - opts["default_transaction_options"] = txn_opts - - s = client.start_session(**dict(opts)) - - sessions[session_name] = s - # Store lsid so we can access it after end_session, in check_events. - session_ids[session_name] = s.session_id - - self.addAsyncCleanup(end_sessions, sessions) - - collection = client[database_name][collection_name] - await self.run_test_ops(sessions, collection, test) - - await end_sessions(sessions) - - self.check_events(test, listener, session_ids) - - # Disable fail points. - if "failPoint" in test: - fp = test["failPoint"] - await self.set_fail_point( - {"configureFailPoint": fp["configureFailPoint"], "mode": "off"} - ) - - # Assert final state is expected. - outcome = test["outcome"] - expected_c = outcome.get("collection") - if expected_c is not None: - outcome_coll_name = self.get_outcome_coll_name(outcome, collection) - - # Read from the primary with local read concern to ensure causal - # consistency. - outcome_coll = async_client_context.client[collection.database.name].get_collection( - outcome_coll_name, - read_preference=ReadPreference.PRIMARY, - read_concern=ReadConcern("local"), - ) - actual_data = await outcome_coll.find(sort=[("_id", 1)]).to_list() - - # The expected data needs to be the left hand side here otherwise - # CompareType(Binary) doesn't work. - self.assertEqual(wrap_types(expected_c["data"]), actual_data) - - -def expect_any_error(op): - if isinstance(op, dict): - return op.get("error") - - return False - - -def expect_error_message(expected_result): - if isinstance(expected_result, dict): - return isinstance(expected_result["errorContains"], str) - - return False - - -def expect_error_code(expected_result): - if isinstance(expected_result, dict): - return expected_result["errorCodeName"] - - return False - - -def expect_error_labels_contain(expected_result): - if isinstance(expected_result, dict): - return expected_result["errorLabelsContain"] - - return False - - -def expect_error_labels_omit(expected_result): - if isinstance(expected_result, dict): - return expected_result["errorLabelsOmit"] - - return False - - -def expect_timeout_error(expected_result): - if isinstance(expected_result, dict): - return expected_result["isTimeoutError"] - - return False - - -def expect_error(op): - expected_result = op.get("result") - return ( - expect_any_error(op) - or expect_error_message(expected_result) - or expect_error_code(expected_result) - or expect_error_labels_contain(expected_result) - or expect_error_labels_omit(expected_result) - or expect_timeout_error(expected_result) - ) - - -async def end_sessions(sessions): - for s in sessions.values(): - # Aborts the transaction if it's open. - await s.end_session() - - -def decode_raw(val): - """Decode RawBSONDocuments in the given container.""" - if isinstance(val, (list, abc.Mapping)): - return decode(encode({"v": val}))["v"] - return val - - -TYPES = { - "binData": Binary, - "long": Int64, - "int": int, - "string": str, - "objectId": ObjectId, - "object": dict, - "array": list, -} - - -def wrap_types(val): - """Support $$type assertion in command results.""" - if isinstance(val, list): - return [wrap_types(v) for v in val] - if isinstance(val, abc.Mapping): - typ = val.get("$$type") - if typ: - if isinstance(typ, str): - types = TYPES[typ] - else: - types = tuple(TYPES[t] for t in typ) - return CompareType(types) - d = {} - for key in val: - d[key] = wrap_types(val[key]) - return d - return val diff --git a/test/auth/unified/mongodb-oidc-no-retry.json b/test/auth/unified/mongodb-oidc-no-retry.json index 0a8658455..b32ada172 100644 --- a/test/auth/unified/mongodb-oidc-no-retry.json +++ b/test/auth/unified/mongodb-oidc-no-retry.json @@ -25,7 +25,8 @@ "$$placeholder": 1 }, "retryReads": false, - "retryWrites": false + "retryWrites": false, + "appName": "mongodb-oidc-no-retry" }, "observeEvents": [ "commandStartedEvent", @@ -147,7 +148,8 @@ "failCommands": [ "find" ], - "errorCode": 391 + "errorCode": 391, + "appName": "mongodb-oidc-no-retry" } } } @@ -212,7 +214,8 @@ "failCommands": [ "insert" ], - "errorCode": 391 + "errorCode": 391, + "appName": "mongodb-oidc-no-retry" } } } @@ -289,7 +292,8 @@ "failCommands": [ "insert" ], - "closeConnection": true + "closeConnection": true, + "appName": "mongodb-oidc-no-retry" } } } @@ -321,7 +325,8 @@ "failCommands": [ "saslStart" ], - "errorCode": 18 + "errorCode": 18, + "appName": "mongodb-oidc-no-retry" } } } @@ -398,7 +403,8 @@ "failCommands": [ "saslStart" ], - "errorCode": 18 + "errorCode": 18, + "appName": "mongodb-oidc-no-retry" } } } diff --git a/test/csot/convenient-transactions.json b/test/csot/convenient-transactions.json index f9d03429d..3400b82ba 100644 --- a/test/csot/convenient-transactions.json +++ b/test/csot/convenient-transactions.json @@ -27,7 +27,8 @@ "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ - "commandStartedEvent" + "commandStartedEvent", + "commandFailedEvent" ] } }, @@ -188,6 +189,11 @@ } } }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, { "commandStartedEvent": { "commandName": "abortTransaction", @@ -206,6 +212,105 @@ ] } ] + }, + { + "description": "withTransaction surfaces a timeout after exhausting transient transaction retries, retaining the last transient error as the timeout cause.", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "blockConnection": true, + "blockTimeMS": 25, + "errorCode": 24, + "errorLabels": [ + "TransientTransactionError" + ] + } + } + } + }, + { + "name": "withTransaction", + "object": "session", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 1 + }, + "session": "session" + }, + "expectError": { + "isError": true + } + } + ] + }, + "expectError": { + "isTimeoutError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "ignoreExtraEvents": true, + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandFailedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandFailedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ] } ] } diff --git a/test/discovery_and_monitoring/errors/error_handling_handshake.json b/test/discovery_and_monitoring/errors/error_handling_handshake.json index bf83f46f6..c60ee453d 100644 --- a/test/discovery_and_monitoring/errors/error_handling_handshake.json +++ b/test/discovery_and_monitoring/errors/error_handling_handshake.json @@ -85,7 +85,7 @@ } }, { - "description": "Mark server unknown on network timeout application error (beforeHandshakeCompletes)", + "description": "Ignore network timeout application error (beforeHandshakeCompletes)", "applicationErrors": [ { "address": "a:27017", diff --git a/test/discovery_and_monitoring/rs/disaggregated_storage_setversion.json b/test/discovery_and_monitoring/rs/disaggregated_storage_setversion.json new file mode 100644 index 000000000..c8b41d30c --- /dev/null +++ b/test/discovery_and_monitoring/rs/disaggregated_storage_setversion.json @@ -0,0 +1,167 @@ +{ + "description": "Static setVersion (DSC) is compatible with both pre and post DRIVERS-2412", + "uri": "mongodb://a/?replicaSet=rs", + "phases": [ + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000005" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ], + [ + "b:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": false, + "secondary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000005" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000005" + } + } + }, + { + "responses": [ + [ + "b:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000006" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "Unknown", + "setName": null, + "setVersion": null, + "electionId": null + }, + "b:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000006" + } + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000006" + } + } + }, + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000005" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "Unknown", + "setName": null, + "setVersion": null, + "electionId": null + }, + "b:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000006" + } + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000006" + } + } + } + ] +} diff --git a/test/discovery_and_monitoring/rs/member_list_update_with_unchanged_setversion_and_electionid.json b/test/discovery_and_monitoring/rs/member_list_update_with_unchanged_setversion_and_electionid.json new file mode 100644 index 000000000..0045591db --- /dev/null +++ b/test/discovery_and_monitoring/rs/member_list_update_with_unchanged_setversion_and_electionid.json @@ -0,0 +1,227 @@ +{ + "description": "Member list is updated when setVersion and electionId remain the same", + "uri": "mongodb://a/?replicaSet=rs", + "phases": [ + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ], + [ + "b:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": false, + "secondary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000001" + } + } + }, + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017", + "c:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + }, + "c:27017": { + "type": "Unknown", + "setName": null, + "setVersion": null, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000001" + } + } + }, + { + "responses": [ + [ + "c:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": false, + "secondary": true, + "hosts": [ + "a:27017", + "b:27017", + "c:27017" + ], + "setName": "rs", + "setVersion": 1, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + }, + "c:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000001" + } + } + }, + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000001" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000001" + } + } + } + ] +} diff --git a/test/discovery_and_monitoring/rs/migration_from_disaggregated_storage.json b/test/discovery_and_monitoring/rs/migration_from_disaggregated_storage.json new file mode 100644 index 000000000..c5109026b --- /dev/null +++ b/test/discovery_and_monitoring/rs/migration_from_disaggregated_storage.json @@ -0,0 +1,167 @@ +{ + "description": "DSC to ASC reverse migration - ASC primary with higher setVersion is accepted", + "uri": "mongodb://a/?replicaSet=rs", + "phases": [ + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000005" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ], + [ + "b:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": false, + "secondary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000005" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 1, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1, + "maxElectionId": { + "$oid": "000000000000000000000005" + } + } + }, + { + "responses": [ + [ + "b:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1000, + "electionId": { + "$oid": "000000000000000000000006" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "Unknown", + "setName": null, + "setVersion": null, + "electionId": null + }, + "b:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1000, + "electionId": { + "$oid": "000000000000000000000006" + } + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1000, + "maxElectionId": { + "$oid": "000000000000000000000006" + } + } + }, + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 1, + "electionId": { + "$oid": "000000000000000000000005" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "Unknown", + "setName": null, + "setVersion": null, + "electionId": null + }, + "b:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 1000, + "electionId": { + "$oid": "000000000000000000000006" + } + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 1000, + "maxElectionId": { + "$oid": "000000000000000000000006" + } + } + } + ] +} diff --git a/test/discovery_and_monitoring/rs/migration_to_disaggregated_storage.json b/test/discovery_and_monitoring/rs/migration_to_disaggregated_storage.json new file mode 100644 index 000000000..57f39c93b --- /dev/null +++ b/test/discovery_and_monitoring/rs/migration_to_disaggregated_storage.json @@ -0,0 +1,119 @@ +{ + "description": "ASC to DSC forward migration - DSC uses setVersionASC + 1 to prevent false stale detection", + "uri": "mongodb://a/?replicaSet=rs", + "phases": [ + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 10, + "electionId": { + "$oid": "000000000000000000000005" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ], + [ + "b:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": false, + "secondary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 10, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 10, + "electionId": { + "$oid": "000000000000000000000005" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 10, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 10, + "maxElectionId": { + "$oid": "000000000000000000000005" + } + } + }, + { + "responses": [ + [ + "a:27017", + { + "ok": 1, + "helloOk": true, + "isWritablePrimary": true, + "hosts": [ + "a:27017", + "b:27017" + ], + "setName": "rs", + "setVersion": 11, + "electionId": { + "$oid": "000000000000000000000006" + }, + "minWireVersion": 0, + "maxWireVersion": 17 + } + ] + ], + "outcome": { + "servers": { + "a:27017": { + "type": "RSPrimary", + "setName": "rs", + "setVersion": 11, + "electionId": { + "$oid": "000000000000000000000006" + } + }, + "b:27017": { + "type": "RSSecondary", + "setName": "rs", + "setVersion": 10, + "electionId": null + } + }, + "topologyType": "ReplicaSetWithPrimary", + "logicalSessionTimeoutMinutes": null, + "setName": "rs", + "maxSetVersion": 11, + "maxElectionId": { + "$oid": "000000000000000000000006" + } + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedNearestOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedNearestOnlyMatchingTags.json new file mode 100644 index 000000000..5a9e8797e --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedNearestOnlyMatchingTags.json @@ -0,0 +1,62 @@ +{ + "topology_description": { + "type": "ReplicaSetNoPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "Nearest", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedPrimaryPreferredOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedPrimaryPreferredOnlyMatchingTags.json new file mode 100644 index 000000000..086532e71 --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedPrimaryPreferredOnlyMatchingTags.json @@ -0,0 +1,62 @@ +{ + "topology_description": { + "type": "ReplicaSetNoPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "PrimaryPreferred", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedSecondaryOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedSecondaryOnlyMatchingTags.json new file mode 100644 index 000000000..18926581c --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedSecondaryOnlyMatchingTags.json @@ -0,0 +1,62 @@ +{ + "topology_description": { + "type": "ReplicaSetNoPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "Secondary", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedSecondaryPreferredOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedSecondaryPreferredOnlyMatchingTags.json new file mode 100644 index 000000000..ab5134535 --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetNoPrimary/read/DeprioritizedSecondaryPreferredOnlyMatchingTags.json @@ -0,0 +1,62 @@ +{ + "topology_description": { + "type": "ReplicaSetNoPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "SecondaryPreferred", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedNearestOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedNearestOnlyMatchingTags.json new file mode 100644 index 000000000..021f36148 --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedNearestOnlyMatchingTags.json @@ -0,0 +1,70 @@ +{ + "topology_description": { + "type": "ReplicaSetWithPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + }, + { + "address": "a:27017", + "avg_rtt_ms": 26, + "type": "RSPrimary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "Nearest", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedPrimaryPreferredOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedPrimaryPreferredOnlyMatchingTags.json new file mode 100644 index 000000000..4002907b3 --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedPrimaryPreferredOnlyMatchingTags.json @@ -0,0 +1,70 @@ +{ + "topology_description": { + "type": "ReplicaSetWithPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + }, + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "nyc" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "PrimaryPreferred", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedSecondaryOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedSecondaryOnlyMatchingTags.json new file mode 100644 index 000000000..2de5bdd4c --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedSecondaryOnlyMatchingTags.json @@ -0,0 +1,70 @@ +{ + "topology_description": { + "type": "ReplicaSetWithPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + }, + { + "address": "a:27017", + "avg_rtt_ms": 26, + "type": "RSPrimary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "Secondary", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedSecondaryPreferredOnlyMatchingTags.json b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedSecondaryPreferredOnlyMatchingTags.json new file mode 100644 index 000000000..7e1f39a60 --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/DeprioritizedSecondaryPreferredOnlyMatchingTags.json @@ -0,0 +1,70 @@ +{ + "topology_description": { + "type": "ReplicaSetWithPrimary", + "servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + }, + { + "address": "c:27017", + "avg_rtt_ms": 100, + "type": "RSSecondary", + "tags": { + "data_center": "tokyo" + } + }, + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "tokyo" + } + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "SecondaryPreferred", + "tag_sets": [ + { + "data_center": "nyc" + } + ] + }, + "deprioritized_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary", + "tags": { + "data_center": "nyc" + } + } + ], + "suitable_servers": [ + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "tokyo" + } + } + ], + "in_latency_window": [ + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary", + "tags": { + "data_center": "tokyo" + } + } + ] +} diff --git a/test/server_selection/server_selection/ReplicaSetWithPrimary/read/SecondaryPreferred_empty_tags.json b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/SecondaryPreferred_empty_tags.json new file mode 100644 index 000000000..8ec8049ef --- /dev/null +++ b/test/server_selection/server_selection/ReplicaSetWithPrimary/read/SecondaryPreferred_empty_tags.json @@ -0,0 +1,41 @@ +{ + "topology_description": { + "type": "ReplicaSetWithPrimary", + "servers": [ + { + "address": "a:27017", + "avg_rtt_ms": 5, + "type": "RSPrimary" + }, + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary" + } + ] + }, + "operation": "read", + "read_preference": { + "mode": "SecondaryPreferred", + "tag_sets": [ + { + "data_center": "nyc" + }, + {} + ] + }, + "suitable_servers": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary" + } + ], + "in_latency_window": [ + { + "address": "b:27017", + "avg_rtt_ms": 5, + "type": "RSSecondary" + } + ] +} diff --git a/test/test_auth_oidc.py b/test/test_auth_oidc.py index 1defe8200..e88e067b2 100644 --- a/test/test_auth_oidc.py +++ b/test/test_auth_oidc.py @@ -104,14 +104,16 @@ class OIDCTestBase(PyMongoTestCase): @contextmanager def fail_point(self, command_args): - cmd_on = SON([("configureFailPoint", "failCommand")]) + cmd_on = dict(configureFailPoint="failCommand", appName="auth_oidc") cmd_on.update(command_args) client = MongoClient(self.uri_admin) client.admin.command(cmd_on) try: yield finally: - client.admin.command("configureFailPoint", cmd_on["configureFailPoint"], mode="off") + client.admin.command( + "configureFailPoint", cmd_on["configureFailPoint"], mode="off", appName="auth_oidc" + ) client.close() diff --git a/test/test_encryption.py b/test/test_encryption.py index 88d37cfa0..af9f2e3df 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -872,6 +872,8 @@ class TestViews(EncryptionIntegrationTest): class TestCorpus(EncryptionIntegrationTest): + # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. + @client_context.require_version_max(6, 99) @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") def setUp(self): super().setUp() @@ -1048,6 +1050,8 @@ class TestBsonSizeBatches(EncryptionIntegrationTest): client_encrypted: MongoClient listener: OvertCommandListener + # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. + @client_context.require_version_max(6, 99) def setUp(self): super().setUp() db = client_context.client.db diff --git a/test/test_retryable_reads.py b/test/test_retryable_reads.py index c9f72ae54..18cd669f1 100644 --- a/test/test_retryable_reads.py +++ b/test/test_retryable_reads.py @@ -259,6 +259,84 @@ class TestRetryableReads(IntegrationTest): self.assertEqual(command_docs[0]["lsid"], command_docs[1]["lsid"]) self.assertIsNot(command_docs[0], command_docs[1]) + @client_context.require_replica_set + @client_context.require_secondaries_count(1) + @client_context.require_failCommand_fail_point + @client_context.require_version_min(4, 4, 0) + def test_03_01_retryable_reads_caused_by_overload_errors_are_retried_on_a_different_replicaset_server_when_one_is_available( + self + ): + listener = OvertCommandListener() + + # 1. Create a client `client` with `retryReads=true`, `readPreference=primaryPreferred`, and command event monitoring enabled. + client = self.rs_or_single_client( + event_listeners=[listener], retryReads=True, readPreference="primaryPreferred" + ) + + # 2. Configure a fail point with the RetryableError and SystemOverloadedError error labels. + command_args = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find"], + "errorLabels": ["RetryableError", "SystemOverloadedError"], + "errorCode": 6, + }, + } + set_fail_point(client, command_args) + + # 3. Reset the command event monitor to clear the fail point command from its stored events. + listener.reset() + + # 4. Execute a `find` command with `client`. + client.t.t.find_one({}) + + # 5. Assert that one failed command event and one successful command event occurred. + self.assertEqual(len(listener.failed_events), 1) + self.assertEqual(len(listener.succeeded_events), 1) + + # 6. Assert that both events occurred on different servers. + assert listener.failed_events[0].connection_id != listener.succeeded_events[0].connection_id + + @client_context.require_replica_set + @client_context.require_secondaries_count(1) + @client_context.require_failCommand_fail_point + @client_context.require_version_min(4, 4, 0) + def test_03_02_retryable_reads_caused_by_non_overload_errors_are_retried_on_the_same_replicaset_server( + self + ): + listener = OvertCommandListener() + + # 1. Create a client `client` with `retryReads=true`, `readPreference=primaryPreferred`, and command event monitoring enabled. + client = self.rs_or_single_client( + event_listeners=[listener], retryReads=True, readPreference="primaryPreferred" + ) + + # 2. Configure a fail point with the RetryableError error label. + command_args = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find"], + "errorLabels": ["RetryableError"], + "errorCode": 6, + }, + } + set_fail_point(client, command_args) + + # 3. Reset the command event monitor to clear the fail point command from its stored events. + listener.reset() + + # 4. Execute a `find` command with `client`. + client.t.t.find_one({}) + + # 5. Assert that one failed command event and one successful command event occurred. + self.assertEqual(len(listener.failed_events), 1) + self.assertEqual(len(listener.succeeded_events), 1) + + # 6. Assert that both events occurred the same server. + assert listener.failed_events[0].connection_id == listener.succeeded_events[0].connection_id + if __name__ == "__main__": unittest.main() diff --git a/test/test_session.py b/test/test_session.py index 40d0a53af..3963f88da 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -189,6 +189,52 @@ class TestSession(IntegrationTest): f"{f.__name__} did not return implicit session to pool", ) + # Explicit bound session + for f, args, kw in ops: + with client.start_session() as s: + with s.bind(): + listener.reset() + s._materialize() + last_use = s._server_session.last_use + start = time.monotonic() + self.assertLessEqual(last_use, start) + # In case "f" modifies its inputs. + args = copy.copy(args) + kw = copy.copy(kw) + f(*args, **kw) + self.assertGreaterEqual(len(listener.started_events), 1) + for event in listener.started_events: + self.assertIn( + "lsid", + event.command, + f"{f.__name__} sent no lsid with {event.command_name}", + ) + + self.assertEqual( + s.session_id, + event.command["lsid"], + f"{f.__name__} sent wrong lsid with {event.command_name}", + ) + + self.assertFalse(s.has_ended) + + self.assertTrue(s.has_ended) + with self.assertRaisesRegex(InvalidOperation, "ended session"): + with s.bind(): + f(*args, **kw) + + # Test a session cannot be used on another client. + with self.client2.start_session() as s: + with s.bind(): + # In case "f" modifies its inputs. + args = copy.copy(args) + kw = copy.copy(kw) + with self.assertRaisesRegex( + InvalidOperation, + "Only the client that created the bound session can perform operations within its context block", + ): + f(*args, **kw) + def test_implicit_sessions_checkout(self): # "To confirm that implicit sessions only allocate their server session after a # successful connection checkout" test from Driver Sessions Spec. @@ -825,6 +871,73 @@ class TestSession(IntegrationTest): with client.start_session() as s: self.assertRaises(TypeError, lambda: copy.copy(s)) + def test_nested_session_binding(self): + coll = self.client.pymongo_test.test + coll.insert_one({"x": 1}) + + session1 = self.client.start_session() + session2 = self.client.start_session() + session1._materialize() + session2._materialize() + try: + self.listener.reset() + # Uses implicit session + coll.find_one() + implicit_lsid = self.listener.started_events[0].command.get("lsid") + self.assertIsNotNone(implicit_lsid) + self.assertNotEqual(implicit_lsid, session1.session_id) + self.assertNotEqual(implicit_lsid, session2.session_id) + + with session1.bind(end_session=False): + self.listener.reset() + # Uses bound session1 + coll.find_one() + session1_lsid = self.listener.started_events[0].command.get("lsid") + self.assertEqual(session1_lsid, session1.session_id) + + with session2.bind(end_session=False): + self.listener.reset() + # Uses bound session2 + coll.find_one() + session2_lsid = self.listener.started_events[0].command.get("lsid") + self.assertEqual(session2_lsid, session2.session_id) + self.assertNotEqual(session2_lsid, session1.session_id) + + self.listener.reset() + # Use bound session1 again + coll.find_one() + session1_lsid = self.listener.started_events[0].command.get("lsid") + self.assertEqual(session1_lsid, session1.session_id) + self.assertNotEqual(session1_lsid, session2.session_id) + + self.listener.reset() + # Uses implicit session + coll.find_one() + implicit_lsid = self.listener.started_events[0].command.get("lsid") + self.assertIsNotNone(implicit_lsid) + self.assertNotEqual(implicit_lsid, session1.session_id) + self.assertNotEqual(implicit_lsid, session2.session_id) + + finally: + session1.end_session() + session2.end_session() + + def test_session_binding_end_session(self): + coll = self.client.pymongo_test.test + coll.insert_one({"x": 1}) + + with self.client.start_session().bind() as s1: + coll.find_one() + + self.assertTrue(s1.has_ended) + + with self.client.start_session().bind(end_session=False) as s2: + coll.find_one() + + self.assertFalse(s2.has_ended) + + s2.end_session() + class TestCausalConsistency(UnitTest): listener: SessionTestListener diff --git a/test/test_ssl.py b/test/test_ssl.py index b1e9a65eb..77bb086ec 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -48,19 +48,11 @@ from pymongo.write_concern import WriteConcern _HAVE_PYOPENSSL = False try: - # All of these must be available to use PyOpenSSL - import OpenSSL - import requests - import service_identity - - # Ensure service_identity>=18.1 is installed - from service_identity.pyopenssl import verify_ip_address - - from pymongo.ocsp_support import _load_trusted_ca_certs + from pymongo import pyopenssl_context _HAVE_PYOPENSSL = True except ImportError: - _load_trusted_ca_certs = None # type: ignore + pass if HAVE_SSL: @@ -136,11 +128,6 @@ class TestClientSSL(PyMongoTestCase): def test_use_pyopenssl_when_available(self): self.assertTrue(HAVE_PYSSL) - @unittest.skipUnless(_HAVE_PYOPENSSL, "Cannot test without PyOpenSSL") - def test_load_trusted_ca_certs(self): - trusted_ca_certs = _load_trusted_ca_certs(CA_BUNDLE_PEM) - self.assertEqual(2, len(trusted_ca_certs)) - class TestSSL(IntegrationTest): saved_port: int diff --git a/test/unified_format.py b/test/unified_format.py index 9aee28725..5516a7adf 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1451,11 +1451,6 @@ class UnifiedSpecTestMixinV1(IntegrationTest): self.assertListEqual(sorted_expected_documents, actual_documents) def run_scenario(self, spec, uri=None): - # Kill all sessions before and after each test to prevent an open - # transaction (from a test failure) from blocking collection/database - # operations during test set up and tear down. - self.kill_all_sessions() - # Handle flaky tests. flaky_tests = [ ("PYTHON-5170", ".*test_discovery_and_monitoring.*"), @@ -1491,6 +1486,15 @@ class UnifiedSpecTestMixinV1(IntegrationTest): if skip_reason is not None: raise unittest.SkipTest(f"{skip_reason}") + # Kill all sessions after each test with transactions to prevent an open + # transaction (from a test failure) from blocking collection/database + # operations during test set up and tear down. + for op in spec["operations"]: + name = op["name"] + if name == "startTransaction" or name == "withTransaction": + self.addCleanup(self.kill_all_sessions) + break + # process createEntities self._uri = uri self.entity_map = EntityMapUtil(self) diff --git a/test/utils_spec_runner.py b/test/utils_spec_runner.py index 9bf155e8f..f4c1c6bfc 100644 --- a/test/utils_spec_runner.py +++ b/test/utils_spec_runner.py @@ -16,43 +16,13 @@ from __future__ import annotations import asyncio -import functools import os -import time -import unittest -from collections import abc -from inspect import iscoroutinefunction -from test import IntegrationTest, client_context, client_knobs +from test import client_context from test.helpers import ConcurrentRunner -from test.utils_shared import ( - CMAPListener, - CompareType, - EventListener, - OvertCommandListener, - ScenarioDict, - ServerAndTopologyEventListener, - camel_to_snake, - camel_to_snake_args, - parse_spec_options, - prepare_spec_arguments, -) -from typing import List +from test.utils_shared import ScenarioDict -from bson import ObjectId, decode, encode, json_util -from bson.binary import Binary -from bson.int64 import Int64 -from bson.son import SON -from gridfs import GridFSBucket -from gridfs.synchronous.grid_file import GridFSBucket -from pymongo.errors import AutoReconnect, BulkWriteError, OperationFailure, PyMongoError +from bson import json_util from pymongo.lock import _cond_wait, _create_condition, _create_lock -from pymongo.read_concern import ReadConcern -from pymongo.read_preferences import ReadPreference -from pymongo.results import BulkWriteResult, _WriteResult -from pymongo.synchronous import client_session -from pymongo.synchronous.command_cursor import CommandCursor -from pymongo.synchronous.cursor import Cursor -from pymongo.write_concern import WriteConcern _IS_SYNC = True @@ -219,595 +189,3 @@ class SpecTestCreator: self._create_tests() else: asyncio.run(self._create_tests()) - - -class SpecRunner(IntegrationTest): - mongos_clients: List - knobs: client_knobs - listener: EventListener - - def setUp(self) -> None: - super().setUp() - self.mongos_clients = [] - - # Speed up the tests by decreasing the heartbeat frequency. - self.knobs = client_knobs(heartbeat_frequency=0.1, min_heartbeat_interval=0.1) - self.knobs.enable() - self.targets = {} - self.listener = None # type: ignore - self.pool_listener = None - self.server_listener = None - self.maxDiff = None - - def tearDown(self) -> None: - self.knobs.disable() - - def set_fail_point(self, command_args): - clients = self.mongos_clients if self.mongos_clients else [self.client] - for client in clients: - self.configure_fail_point(client, command_args) - - def targeted_fail_point(self, session, fail_point): - """Run the targetedFailPoint test operation. - - Enable the fail point on the session's pinned mongos. - """ - clients = {c.address: c for c in self.mongos_clients} - client = clients[session._pinned_address] - self.configure_fail_point(client, fail_point) - self.addCleanup(self.set_fail_point, {"mode": "off"}) - - def assert_session_pinned(self, session): - """Run the assertSessionPinned test operation. - - Assert that the given session is pinned. - """ - self.assertIsNotNone(session._transaction.pinned_address) - - def assert_session_unpinned(self, session): - """Run the assertSessionUnpinned test operation. - - Assert that the given session is not pinned. - """ - self.assertIsNone(session._pinned_address) - self.assertIsNone(session._transaction.pinned_address) - - def assert_collection_exists(self, database, collection): - """Run the assertCollectionExists test operation.""" - db = self.client[database] - self.assertIn(collection, db.list_collection_names()) - - def assert_collection_not_exists(self, database, collection): - """Run the assertCollectionNotExists test operation.""" - db = self.client[database] - self.assertNotIn(collection, db.list_collection_names()) - - def assert_index_exists(self, database, collection, index): - """Run the assertIndexExists test operation.""" - coll = self.client[database][collection] - self.assertIn(index, [doc["name"] for doc in coll.list_indexes()]) - - def assert_index_not_exists(self, database, collection, index): - """Run the assertIndexNotExists test operation.""" - coll = self.client[database][collection] - self.assertNotIn(index, [doc["name"] for doc in coll.list_indexes()]) - - def wait(self, ms): - """Run the "wait" test operation.""" - time.sleep(ms / 1000.0) - - def assertErrorLabelsContain(self, exc, expected_labels): - labels = [l for l in expected_labels if exc.has_error_label(l)] - self.assertEqual(labels, expected_labels) - - def assertErrorLabelsOmit(self, exc, omit_labels): - for label in omit_labels: - self.assertFalse( - exc.has_error_label(label), msg=f"error labels should not contain {label}" - ) - - def kill_all_sessions(self): - clients = self.mongos_clients if self.mongos_clients else [self.client] - for client in clients: - try: - client.admin.command("killAllSessions", []) - except (OperationFailure, AutoReconnect): - # "operation was interrupted" by killing the command's - # own session. - # On 8.0+ killAllSessions sometimes returns a network error. - pass - - def check_command_result(self, expected_result, result): - # Only compare the keys in the expected result. - filtered_result = {} - for key in expected_result: - try: - filtered_result[key] = result[key] - except KeyError: - pass - self.assertEqual(filtered_result, expected_result) - - # TODO: factor the following function with test_crud.py. - def check_result(self, expected_result, result): - if isinstance(result, _WriteResult): - for res in expected_result: - prop = camel_to_snake(res) - # SPEC-869: Only BulkWriteResult has upserted_count. - if prop == "upserted_count" and not isinstance(result, BulkWriteResult): - if result.upserted_id is not None: - upserted_count = 1 - else: - upserted_count = 0 - self.assertEqual(upserted_count, expected_result[res], prop) - elif prop == "inserted_ids": - # BulkWriteResult does not have inserted_ids. - if isinstance(result, BulkWriteResult): - self.assertEqual(len(expected_result[res]), result.inserted_count) - else: - # InsertManyResult may be compared to [id1] from the - # crud spec or {"0": id1} from the retryable write spec. - ids = expected_result[res] - if isinstance(ids, dict): - ids = [ids[str(i)] for i in range(len(ids))] - - self.assertEqual(ids, result.inserted_ids, prop) - elif prop == "upserted_ids": - # Convert indexes from strings to integers. - ids = expected_result[res] - expected_ids = {} - for str_index in ids: - expected_ids[int(str_index)] = ids[str_index] - self.assertEqual(expected_ids, result.upserted_ids, prop) - else: - self.assertEqual(getattr(result, prop), expected_result[res], prop) - - return True - else: - - def _helper(expected_result, result): - if isinstance(expected_result, abc.Mapping): - for i in expected_result.keys(): - self.assertEqual(expected_result[i], result[i]) - - elif isinstance(expected_result, list): - for i, k in zip(expected_result, result): - _helper(i, k) - else: - self.assertEqual(expected_result, result) - - _helper(expected_result, result) - return None - - def get_object_name(self, op): - """Allow subclasses to override handling of 'object' - - Transaction spec says 'object' is required. - """ - return op["object"] - - @staticmethod - def parse_options(opts): - return parse_spec_options(opts) - - def run_operation(self, sessions, collection, operation): - original_collection = collection - name = camel_to_snake(operation["name"]) - if name == "run_command": - name = "command" - elif name == "download_by_name": - name = "open_download_stream_by_name" - elif name == "download": - name = "open_download_stream" - elif name == "map_reduce": - self.skipTest("PyMongo does not support mapReduce") - elif name == "count": - self.skipTest("PyMongo does not support count") - - database = collection.database - collection = database.get_collection(collection.name) - if "collectionOptions" in operation: - collection = collection.with_options( - **self.parse_options(operation["collectionOptions"]) - ) - - object_name = self.get_object_name(operation) - if object_name == "gridfsbucket": - # Only create the GridFSBucket when we need it (for the gridfs - # retryable reads tests). - obj = GridFSBucket(database, bucket_name=collection.name) - else: - objects = { - "client": database.client, - "database": database, - "collection": collection, - "testRunner": self, - } - objects.update(sessions) - obj = objects[object_name] - - # Combine arguments with options and handle special cases. - arguments = operation.get("arguments", {}) - arguments.update(arguments.pop("options", {})) - self.parse_options(arguments) - - cmd = getattr(obj, name) - - with_txn_callback = functools.partial( - self.run_operations, sessions, original_collection, in_with_transaction=True - ) - prepare_spec_arguments(operation, arguments, name, sessions, with_txn_callback) - - if name == "run_on_thread": - args = {"sessions": sessions, "collection": collection} - args.update(arguments) - arguments = args - - if not _IS_SYNC and iscoroutinefunction(cmd): - result = cmd(**dict(arguments)) - else: - result = cmd(**dict(arguments)) - # Cleanup open change stream cursors. - if name == "watch": - self.addCleanup(result.close) - - if name == "aggregate": - if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: - # Read from the primary to ensure causal consistency. - out = collection.database.get_collection( - arguments["pipeline"][-1]["$out"], read_preference=ReadPreference.PRIMARY - ) - return out.find() - if "download" in name: - result = Binary(result.read()) - - if isinstance(result, Cursor) or isinstance(result, CommandCursor): - return result.to_list() - - return result - - def allowable_errors(self, op): - """Allow encryption spec to override expected error classes.""" - return (PyMongoError,) - - def _run_op(self, sessions, collection, op, in_with_transaction): - expected_result = op.get("result") - if expect_error(op): - with self.assertRaises(self.allowable_errors(op), msg=op["name"]) as context: - self.run_operation(sessions, collection, op.copy()) - exc = context.exception - if expect_error_message(expected_result): - if isinstance(exc, BulkWriteError): - errmsg = str(exc.details).lower() - else: - errmsg = str(exc).lower() - self.assertIn(expected_result["errorContains"].lower(), errmsg) - if expect_error_code(expected_result): - self.assertEqual(expected_result["errorCodeName"], exc.details.get("codeName")) - if expect_error_labels_contain(expected_result): - self.assertErrorLabelsContain(exc, expected_result["errorLabelsContain"]) - if expect_error_labels_omit(expected_result): - self.assertErrorLabelsOmit(exc, expected_result["errorLabelsOmit"]) - if expect_timeout_error(expected_result): - self.assertIsInstance(exc, PyMongoError) - if not exc.timeout: - # Re-raise the exception for better diagnostics. - raise exc - - # Reraise the exception if we're in the with_transaction - # callback. - if in_with_transaction: - raise context.exception - else: - result = self.run_operation(sessions, collection, op.copy()) - if "result" in op: - if op["name"] == "runCommand": - self.check_command_result(expected_result, result) - else: - self.check_result(expected_result, result) - - def run_operations(self, sessions, collection, ops, in_with_transaction=False): - for op in ops: - self._run_op(sessions, collection, op, in_with_transaction) - - # TODO: factor with test_command_monitoring.py - def check_events(self, test, listener, session_ids): - events = listener.started_events - if not len(test["expectations"]): - return - - # Give a nicer message when there are missing or extra events - cmds = decode_raw([event.command for event in events]) - self.assertEqual(len(events), len(test["expectations"]), cmds) - for i, expectation in enumerate(test["expectations"]): - event_type = next(iter(expectation)) - event = events[i] - - # The tests substitute 42 for any number other than 0. - if event.command_name == "getMore" and event.command["getMore"]: - event.command["getMore"] = Int64(42) - elif event.command_name == "killCursors": - event.command["cursors"] = [Int64(42)] - elif event.command_name == "update": - # TODO: remove this once PYTHON-1744 is done. - # Add upsert and multi fields back into expectations. - updates = expectation[event_type]["command"]["updates"] - for update in updates: - update.setdefault("upsert", False) - update.setdefault("multi", False) - - # Replace afterClusterTime: 42 with actual afterClusterTime. - expected_cmd = expectation[event_type]["command"] - expected_read_concern = expected_cmd.get("readConcern") - if expected_read_concern is not None: - time = expected_read_concern.get("afterClusterTime") - if time == 42: - actual_time = event.command.get("readConcern", {}).get("afterClusterTime") - if actual_time is not None: - expected_read_concern["afterClusterTime"] = actual_time - - recovery_token = expected_cmd.get("recoveryToken") - if recovery_token == 42: - expected_cmd["recoveryToken"] = CompareType(dict) - - # Replace lsid with a name like "session0" to match test. - if "lsid" in event.command: - for name, lsid in session_ids.items(): - if event.command["lsid"] == lsid: - event.command["lsid"] = name - break - - for attr, expected in expectation[event_type].items(): - actual = getattr(event, attr) - expected = wrap_types(expected) - if isinstance(expected, dict): - for key, val in expected.items(): - if val is None: - if key in actual: - self.fail(f"Unexpected key [{key}] in {actual!r}") - elif key not in actual: - self.fail(f"Expected key [{key}] in {actual!r}") - else: - self.assertEqual( - val, decode_raw(actual[key]), f"Key [{key}] in {actual}" - ) - else: - self.assertEqual(actual, expected) - - def maybe_skip_scenario(self, test): - if test.get("skipReason"): - self.skipTest(test.get("skipReason")) - - def get_scenario_db_name(self, scenario_def): - """Allow subclasses to override a test's database name.""" - return scenario_def["database_name"] - - def get_scenario_coll_name(self, scenario_def): - """Allow subclasses to override a test's collection name.""" - return scenario_def["collection_name"] - - def get_outcome_coll_name(self, outcome, collection): - """Allow subclasses to override outcome collection.""" - return collection.name - - def run_test_ops(self, sessions, collection, test): - """Added to allow retryable writes spec to override a test's - operation. - """ - self.run_operations(sessions, collection, test["operations"]) - - def parse_client_options(self, opts): - """Allow encryption spec to override a clientOptions parsing.""" - return opts - - def setup_scenario(self, scenario_def): - """Allow specs to override a test's setup.""" - db_name = self.get_scenario_db_name(scenario_def) - coll_name = self.get_scenario_coll_name(scenario_def) - documents = scenario_def["data"] - - # Setup the collection with as few majority writes as possible. - db = client_context.client.get_database(db_name) - coll_exists = bool(db.list_collection_names(filter={"name": coll_name})) - if coll_exists: - db[coll_name].delete_many({}) - # Only use majority wc only on the final write. - wc = WriteConcern(w="majority") - if documents: - db.get_collection(coll_name, write_concern=wc).insert_many(documents) - elif not coll_exists: - # Ensure collection exists. - db.create_collection(coll_name, write_concern=wc) - - def run_scenario(self, scenario_def, test): - self.maybe_skip_scenario(test) - - # Kill all sessions before and after each test to prevent an open - # transaction (from a test failure) from blocking collection/database - # operations during test set up and tear down. - self.kill_all_sessions() - self.addCleanup(self.kill_all_sessions) - self.setup_scenario(scenario_def) - database_name = self.get_scenario_db_name(scenario_def) - collection_name = self.get_scenario_coll_name(scenario_def) - # SPEC-1245 workaround StaleDbVersion on distinct - for c in self.mongos_clients: - c[database_name][collection_name].distinct("x") - - # Configure the fail point before creating the client. - if "failPoint" in test: - fp = test["failPoint"] - self.set_fail_point(fp) - self.addCleanup( - self.set_fail_point, {"configureFailPoint": fp["configureFailPoint"], "mode": "off"} - ) - - listener = OvertCommandListener() - pool_listener = CMAPListener() - server_listener = ServerAndTopologyEventListener() - # Create a new client, to avoid interference from pooled sessions. - client_options = self.parse_client_options(test["clientOptions"]) - use_multi_mongos = test["useMultipleMongoses"] - host = None - if use_multi_mongos: - if client_context.load_balancer: - host = client_context.MULTI_MONGOS_LB_URI - elif client_context.is_mongos: - host = client_context.mongos_seeds() - client = self.rs_client( - h=host, event_listeners=[listener, pool_listener, server_listener], **client_options - ) - self.scenario_client = client - self.listener = listener - self.pool_listener = pool_listener - self.server_listener = server_listener - - # Create session0 and session1. - sessions = {} - session_ids = {} - for i in range(2): - # Don't attempt to create sessions if they are not supported by - # the running server version. - if not client_context.sessions_enabled: - break - session_name = "session%d" % i - opts = camel_to_snake_args(test["sessionOptions"][session_name]) - if "default_transaction_options" in opts: - txn_opts = self.parse_options(opts["default_transaction_options"]) - txn_opts = client_session.TransactionOptions(**txn_opts) - opts["default_transaction_options"] = txn_opts - - s = client.start_session(**dict(opts)) - - sessions[session_name] = s - # Store lsid so we can access it after end_session, in check_events. - session_ids[session_name] = s.session_id - - self.addCleanup(end_sessions, sessions) - - collection = client[database_name][collection_name] - self.run_test_ops(sessions, collection, test) - - end_sessions(sessions) - - self.check_events(test, listener, session_ids) - - # Disable fail points. - if "failPoint" in test: - fp = test["failPoint"] - self.set_fail_point({"configureFailPoint": fp["configureFailPoint"], "mode": "off"}) - - # Assert final state is expected. - outcome = test["outcome"] - expected_c = outcome.get("collection") - if expected_c is not None: - outcome_coll_name = self.get_outcome_coll_name(outcome, collection) - - # Read from the primary with local read concern to ensure causal - # consistency. - outcome_coll = client_context.client[collection.database.name].get_collection( - outcome_coll_name, - read_preference=ReadPreference.PRIMARY, - read_concern=ReadConcern("local"), - ) - actual_data = outcome_coll.find(sort=[("_id", 1)]).to_list() - - # The expected data needs to be the left hand side here otherwise - # CompareType(Binary) doesn't work. - self.assertEqual(wrap_types(expected_c["data"]), actual_data) - - -def expect_any_error(op): - if isinstance(op, dict): - return op.get("error") - - return False - - -def expect_error_message(expected_result): - if isinstance(expected_result, dict): - return isinstance(expected_result["errorContains"], str) - - return False - - -def expect_error_code(expected_result): - if isinstance(expected_result, dict): - return expected_result["errorCodeName"] - - return False - - -def expect_error_labels_contain(expected_result): - if isinstance(expected_result, dict): - return expected_result["errorLabelsContain"] - - return False - - -def expect_error_labels_omit(expected_result): - if isinstance(expected_result, dict): - return expected_result["errorLabelsOmit"] - - return False - - -def expect_timeout_error(expected_result): - if isinstance(expected_result, dict): - return expected_result["isTimeoutError"] - - return False - - -def expect_error(op): - expected_result = op.get("result") - return ( - expect_any_error(op) - or expect_error_message(expected_result) - or expect_error_code(expected_result) - or expect_error_labels_contain(expected_result) - or expect_error_labels_omit(expected_result) - or expect_timeout_error(expected_result) - ) - - -def end_sessions(sessions): - for s in sessions.values(): - # Aborts the transaction if it's open. - s.end_session() - - -def decode_raw(val): - """Decode RawBSONDocuments in the given container.""" - if isinstance(val, (list, abc.Mapping)): - return decode(encode({"v": val}))["v"] - return val - - -TYPES = { - "binData": Binary, - "long": Int64, - "int": int, - "string": str, - "objectId": ObjectId, - "object": dict, - "array": list, -} - - -def wrap_types(val): - """Support $$type assertion in command results.""" - if isinstance(val, list): - return [wrap_types(v) for v in val] - if isinstance(val, abc.Mapping): - typ = val.get("$$type") - if typ: - if isinstance(typ, str): - types = TYPES[typ] - else: - types = tuple(TYPES[t] for t in typ) - return CompareType(types) - d = {} - for key in val: - d[key] = wrap_types(val[key]) - return d - return val diff --git a/tools/synchro.py b/tools/synchro.py index 3e326a108..ed794c596 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -37,6 +37,7 @@ replacements = { "AsyncRawBatchCursor": "RawBatchCursor", "AsyncRawBatchCommandCursor": "RawBatchCommandCursor", "AsyncClientSession": "ClientSession", + "_AsyncBoundSessionContext": "_BoundSessionContext", "AsyncChangeStream": "ChangeStream", "AsyncCollectionChangeStream": "CollectionChangeStream", "AsyncDatabaseChangeStream": "DatabaseChangeStream", diff --git a/uv.lock b/uv.lock index 7d0ff9fb0..78d0cc213 100644 --- a/uv.lock +++ b/uv.lock @@ -1562,7 +1562,6 @@ zstd = [ [package.dev-dependencies] coverage = [ { name = "coverage", extra = ["toml"] }, - { name = "pytest-cov" }, ] gevent = [ { name = "gevent" }, @@ -1612,10 +1611,7 @@ requires-dist = [ provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "test", "zstd"] [package.metadata.requires-dev] -coverage = [ - { name = "coverage", extras = ["toml"], specifier = ">=5,<=7.10.7" }, - { name = "pytest-cov", specifier = ">=4.0.0" }, -] +coverage = [{ name = "coverage", extras = ["toml"], specifier = ">=5,<=7.10.7" }] dev = [] gevent = [{ name = "gevent", specifier = ">=21.12" }] mockupdb = [{ name = "mockupdb", git = "https://github.com/mongodb-labs/mongo-mockup-db?rev=master" }] @@ -1763,21 +1759,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] -[[package]] -name = "pytest-cov" -version = "7.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage", extra = ["toml"] }, - { name = "pluggy" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "9.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0"