From 846b1fc25ced5979ade934bb27d6ffbb60f68267 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 08:50:51 -0500 Subject: [PATCH 1/5] PYTHON-5316 Update tests for other hosts (#2287) --- .evergreen/generated_configs/tasks.yml | 44 ++++++++++++++++++++ .evergreen/generated_configs/variants.yml | 45 ++++++++++++--------- .evergreen/scripts/generate_config.py | 42 +++++++++++++------ .evergreen/scripts/generate_config_utils.py | 6 +++ 4 files changed, 105 insertions(+), 32 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 5b09ce20f..fddd725af 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -3083,6 +3083,50 @@ tasks: SUB_TEST_NAME: gke tags: [auth_oidc, auth_oidc_remote] + # Other hosts tests + - name: test-sync-noauth-nossl-standalone + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_NAME: default_sync + tags: [other-hosts, standalone-noauth-nossl] + - name: test-async-noauth-ssl-replica-set + commands: + - func: run server + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + - func: run tests + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + TEST_NAME: default_async + tags: [other-hosts, replica_set-noauth-ssl] + - name: test-sync-auth-ssl-sharded-cluster + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + TEST_NAME: default_sync + tags: [other-hosts, sharded_cluster-auth-ssl] + # Perf tests - name: perf-8.0-standalone-ssl commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index dc46ae6bf..fe315ab9c 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -1,61 +1,66 @@ buildvariants: # Alternative hosts tests - - name: openssl-1.0.2-rhel7-python3.9 + - name: openssl-1.0.2-rhel7-v5.0-python3.9 tasks: - - name: .5.0 .standalone !.sync_async - display_name: OpenSSL 1.0.2 RHEL7 Python3.9 + - name: .other-hosts + display_name: OpenSSL 1.0.2 RHEL7 v5.0 Python3.9 run_on: - rhel79-small batchtime: 10080 expansions: + VERSION: "5.0" + PYTHON_VERSION: "3.9" PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: other-hosts-rhel9-fips + - name: other-hosts-rhel9-fips-latest tasks: - - name: .6.0 .standalone !.sync_async - display_name: Other hosts RHEL9-FIPS + - name: .other-hosts + display_name: Other hosts RHEL9-FIPS latest run_on: - rhel92-fips batchtime: 10080 expansions: + VERSION: latest NO_EXT: "1" REQUIRE_FIPS: "1" - - name: other-hosts-rhel8-zseries + - name: other-hosts-rhel8-zseries-latest tasks: - - name: .6.0 .standalone !.sync_async - display_name: Other hosts RHEL8-zseries + - name: .other-hosts + display_name: Other hosts RHEL8-zseries latest run_on: - rhel8-zseries-small batchtime: 10080 expansions: + VERSION: latest NO_EXT: "1" - - name: other-hosts-rhel8-power8 + - name: other-hosts-rhel8-power8-latest tasks: - - name: .6.0 .standalone !.sync_async - display_name: Other hosts RHEL8-POWER8 + - name: .other-hosts + display_name: Other hosts RHEL8-POWER8 latest run_on: - rhel8-power-small batchtime: 10080 expansions: + VERSION: latest NO_EXT: "1" - - name: other-hosts-rhel8-arm64 + - name: other-hosts-rhel8-arm64-latest tasks: - - name: .6.0 .standalone !.sync_async - display_name: Other hosts RHEL8-arm64 + - name: .other-hosts + display_name: Other hosts RHEL8-arm64 latest run_on: - rhel82-arm64-small batchtime: 10080 expansions: + VERSION: latest NO_EXT: "1" - - name: other-hosts-amazon2023 + - name: other-hosts-amazon2023-latest tasks: - - name: .latest !.sync_async .sharded_cluster .auth .ssl - - name: .latest !.sync_async .replica_set .noauth .ssl - - name: .latest !.sync_async .standalone .noauth .nossl - display_name: Other hosts Amazon2023 + - name: .other-hosts + display_name: Other hosts Amazon2023 latest run_on: - amazon2023-arm64-latest-large-m8g batchtime: 10080 expansions: + VERSION: latest NO_EXT: "1" # Atlas connect tests diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 75ee52de6..ebdbdd966 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -22,6 +22,7 @@ from generate_config_utils import ( create_variant, get_assume_role, get_s3_put, + get_standard_auth_ssl, get_subprocess_exec, get_task_name, get_variant_name, @@ -548,29 +549,29 @@ def create_alternative_hosts_variants(): variants = [] host = HOSTS["rhel7"] + version = "5.0" variants.append( create_variant( - [".5.0 .standalone !.sync_async"], - get_variant_name("OpenSSL 1.0.2", host, python=CPYTHONS[0]), + [".other-hosts"], + get_variant_name("OpenSSL 1.0.2", host, python=CPYTHONS[0], version=version), host=host, python=CPYTHONS[0], batchtime=batchtime, + expansions=dict(VERSION=version, PYTHON_VERSION=CPYTHONS[0]), ) ) + version = "latest" for host_name in OTHER_HOSTS: - expansions = dict() + expansions = dict(VERSION="latest") handle_c_ext(C_EXTS[0], expansions) host = HOSTS[host_name] if "fips" in host_name.lower(): expansions["REQUIRE_FIPS"] = "1" - tags = [".6.0 .standalone !.sync_async"] - if host_name == "Amazon2023": - tags = [f".latest !.sync_async {t}" for t in SUB_TASKS] variants.append( create_variant( - tags, - display_name=get_variant_name("Other hosts", host), + [".other-hosts"], + display_name=get_variant_name("Other hosts", host, version=version), batchtime=batchtime, host=host, expansions=expansions, @@ -615,14 +616,32 @@ def create_server_version_tasks(): return tasks +def create_other_hosts_tasks(): + tasks = [] + + for topology, sync in zip_cycle(TOPOLOGIES, SYNCS): + auth, ssl = get_standard_auth_ssl(topology) + tags = [ + "other-hosts", + f"{topology}-{auth}-{ssl}", + ] + expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology) + name = get_task_name("test", sync=sync, **expansions) + server_func = FunctionCall(func="run server", vars=expansions) + test_vars = expansions.copy() + test_vars["TEST_NAME"] = f"default_{sync}" + test_func = FunctionCall(func="run tests", vars=test_vars) + tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + return tasks + + def create_standard_linux_tasks(): tasks = [] for (version, topology), python in zip_cycle( list(product(ALL_VERSIONS, TOPOLOGIES)), ALL_PYTHONS ): - auth = "auth" if topology == "sharded_cluster" else "noauth" - ssl = "nossl" if topology == "standalone" else "ssl" + auth, ssl = get_standard_auth_ssl(topology) tags = [ "standard-linux", f"server-{version}", @@ -645,8 +664,7 @@ def create_standard_non_linux_tasks(): for (version, topology), python, sync in zip_cycle( list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS, SYNCS ): - auth = "auth" if topology == "sharded_cluster" else "noauth" - ssl = "nossl" if topology == "standalone" else "ssl" + auth, ssl = get_standard_auth_ssl(topology) tags = [ "standard-non-linux", f"server-{version}", diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index aa43af9a6..facb83263 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -239,6 +239,12 @@ def handle_c_ext(c_ext, expansions) -> None: expansions["NO_EXT"] = "1" +def get_standard_auth_ssl(topology): + auth = "auth" if topology == "sharded_cluster" else "noauth" + ssl = "nossl" if topology == "standalone" else "ssl" + return auth, ssl + + def get_assume_role(**kwargs): kwargs.setdefault("command_type", EvgCommandType.SETUP) kwargs.setdefault("role_arn", "${assume_role_arn}") From 149fe390d49de755522c35df4d974c9bae15043e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 10:40:25 -0500 Subject: [PATCH 2/5] PYTHON-5188 Make version setting a part of the release process (#2288) --- .github/workflows/release-python.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 21c7ca5f7..a684ff79b 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -3,12 +3,8 @@ name: Release on: workflow_dispatch: inputs: - version: - description: "The new version to set" - required: true following_version: description: "The post (dev) version to set" - required: true dry_run: description: "Dry Run?" default: false @@ -26,7 +22,6 @@ env: # to 'false' when the input is set to 'false'. DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }} FOLLOWING_VERSION: ${{ inputs.following_version || '' }} - VERSION: ${{ inputs.version || '10.10.10.10' }} defaults: run: @@ -56,7 +51,6 @@ jobs: - uses: mongodb-labs/drivers-github-tools/python/pre-publish@v2 id: pre-publish with: - version: ${{ env.VERSION }} dry_run: ${{ env.DRY_RUN }} build-dist: @@ -118,7 +112,6 @@ jobs: artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }} - uses: mongodb-labs/drivers-github-tools/python/post-publish@v2 with: - version: ${{ env.VERSION }} following_version: ${{ env.FOLLOWING_VERSION }} product_name: ${{ env.PRODUCT_NAME }} evergreen_project: ${{ env.EVERGREEN_PROJECT }} From f476d8bd975a40f6f5907bd2265a203424726bea Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Apr 2025 13:06:09 -0400 Subject: [PATCH 3/5] PYTHON-5324 - Fix Windows encryption test secrets path (#2289) --- .evergreen/scripts/setup_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 9b657507c..bc9f97f1e 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -387,9 +387,9 @@ def handle_test_env() -> None: if not DRIVERS_TOOLS: raise RuntimeError("Missing DRIVERS_TOOLS") csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") - run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir) + run_command(f"bash {csfle_dir.as_posix()}/setup-secrets.sh", cwd=csfle_dir) load_config_from_file(csfle_dir / "secrets-export.sh") - run_command(f"bash {csfle_dir}/start-servers.sh") + run_command(f"bash {csfle_dir.as_posix()}/start-servers.sh") if sub_test_name == "pyopenssl": UV_ARGS.append("--extra ocsp") From aa6fa7a6966be207c26bf70357504a79e4a73add Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Apr 2025 14:10:10 -0400 Subject: [PATCH 4/5] PYTHON-5284 - Remove eventlet tests for CPython > 3.9 (#2290) --- .evergreen/generated_configs/variants.yml | 11 ----------- .evergreen/scripts/generate_config.py | 4 ++++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index fe315ab9c..d36548f72 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -536,17 +536,6 @@ buildvariants: AUTH: auth SSL: ssl PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: green-eventlet-rhel8-python3.13 - tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Green Eventlet RHEL8 Python3.13 - run_on: - - rhel87-small - expansions: - GREEN_FRAMEWORK: eventlet - AUTH: auth - SSL: ssl - PYTHON_BINARY: /opt/python/3.13/bin/python3 - name: green-gevent-rhel8-python3.13 tasks: - name: .standalone .noauth .nossl .sync_async diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index ebdbdd966..7e88f8231 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -343,6 +343,10 @@ def create_green_framework_variants(): tasks = [".standalone .noauth .nossl .sync_async"] host = DEFAULT_HOST for python, framework in product([CPYTHONS[0], CPYTHONS[-1]], ["eventlet", "gevent"]): + if framework == "eventlet" and python == CPYTHONS[-1]: + # Eventlet has issues with dnspython > 2.0 and newer versions of CPython + # https://jira.mongodb.org/browse/PYTHON-5284 + continue expansions = dict(GREEN_FRAMEWORK=framework, AUTH="auth", SSL="ssl") display_name = get_variant_name(f"Green {framework.capitalize()}", host, python=python) variant = create_variant( From 4cac781530238e1830a1af67440ae68ceff663ec Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Apr 2025 14:31:05 -0400 Subject: [PATCH 5/5] PYTHON-5326 - Skip serverless tests with known issue (#2292) --- test/asynchronous/unified_format.py | 6 ++++++ test/unified_format.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 9099efbf0..10b247d1f 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -551,6 +551,12 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): self.skipTest("PYTHON-5170 tests are flakey") if "Driver extends timeout while streaming" in spec["description"] and not _IS_SYNC: self.skipTest("PYTHON-5174 tests are flakey") + if ( + "inserting _id with type null via clientBulkWrite" in spec["description"] + or "commitTransaction fails after Interrupted" in spec["description"] + or "commit is not retried after MaxTimeMSExpired error" in spec["description"] + ) and async_client_context.serverless: + self.skipTest("PYTHON-5326 known serverless failures") class_name = self.__class__.__name__.lower() description = spec["description"].lower() diff --git a/test/unified_format.py b/test/unified_format.py index 71d6cd50d..d3da2b3a8 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -550,6 +550,12 @@ class UnifiedSpecTestMixinV1(IntegrationTest): self.skipTest("PYTHON-5170 tests are flakey") if "Driver extends timeout while streaming" in spec["description"] and not _IS_SYNC: self.skipTest("PYTHON-5174 tests are flakey") + if ( + "inserting _id with type null via clientBulkWrite" in spec["description"] + or "commitTransaction fails after Interrupted" in spec["description"] + or "commit is not retried after MaxTimeMSExpired error" in spec["description"] + ) and client_context.serverless: + self.skipTest("PYTHON-5326 known serverless failures") class_name = self.__class__.__name__.lower() description = spec["description"].lower()