From e48365c5f14b3f6bd4ba7c76648f2d9322653960 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 12 Mar 2025 11:21:19 -0700 Subject: [PATCH 1/3] PYTHON-5202 WaitQueueTimeoutError should not clear the pool (#2192) --- pymongo/asynchronous/topology.py | 3 +++ pymongo/synchronous/topology.py | 3 +++ test/asynchronous/test_client.py | 13 +++++++++++-- test/test_client.py | 13 +++++++++++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index 76f0fb6cd..f00f62ffe 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -41,6 +41,7 @@ from pymongo.errors import ( OperationFailure, PyMongoError, ServerSelectionTimeoutError, + WaitQueueTimeoutError, WriteError, ) from pymongo.hello import Hello @@ -892,6 +893,8 @@ class Topology: # Clear the pool. await server.reset(service_id) elif isinstance(error, ConnectionFailure): + if isinstance(error, WaitQueueTimeoutError): + return # "Client MUST replace the server's description with type Unknown # ... MUST NOT request an immediate check of the server." if not self._settings.load_balanced: diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index ea0edae91..0af793a96 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -37,6 +37,7 @@ from pymongo.errors import ( OperationFailure, PyMongoError, ServerSelectionTimeoutError, + WaitQueueTimeoutError, WriteError, ) from pymongo.hello import Hello @@ -890,6 +891,8 @@ class Topology: # Clear the pool. server.reset(service_id) elif isinstance(error, ConnectionFailure): + if isinstance(error, WaitQueueTimeoutError): + return # "Client MUST replace the server's description with type Unknown # ... MUST NOT request an immediate check of the server." if not self._settings.load_balanced: diff --git a/test/asynchronous/test_client.py b/test/asynchronous/test_client.py index f9678b11e..f529dcce1 100644 --- a/test/asynchronous/test_client.py +++ b/test/asynchronous/test_client.py @@ -113,6 +113,7 @@ from pymongo.errors import ( NetworkTimeout, OperationFailure, ServerSelectionTimeoutError, + WaitQueueTimeoutError, WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent @@ -1313,8 +1314,16 @@ class TestClient(AsyncIntegrationTest): self.assertAlmostEqual(30, client.options.server_selection_timeout) async def test_waitQueueTimeoutMS(self): - client = await self.async_rs_or_single_client(waitQueueTimeoutMS=2000) - self.assertEqual((await async_get_pool(client)).opts.wait_queue_timeout, 2) + listener = CMAPListener() + client = await self.async_rs_or_single_client( + waitQueueTimeoutMS=10, maxPoolSize=1, event_listeners=[listener] + ) + pool = await async_get_pool(client) + self.assertEqual(pool.opts.wait_queue_timeout, 0.01) + async with pool.checkout(): + with self.assertRaises(WaitQueueTimeoutError): + await client.test.command("ping") + self.assertFalse(listener.events_by_type(monitoring.PoolClearedEvent)) async def test_socketKeepAlive(self): pool = await async_get_pool(self.client) diff --git a/test/test_client.py b/test/test_client.py index a34026393..e445fa632 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -102,6 +102,7 @@ from pymongo.errors import ( NetworkTimeout, OperationFailure, ServerSelectionTimeoutError, + WaitQueueTimeoutError, WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent @@ -1272,8 +1273,16 @@ class TestClient(IntegrationTest): self.assertAlmostEqual(30, client.options.server_selection_timeout) def test_waitQueueTimeoutMS(self): - client = self.rs_or_single_client(waitQueueTimeoutMS=2000) - self.assertEqual((get_pool(client)).opts.wait_queue_timeout, 2) + listener = CMAPListener() + client = self.rs_or_single_client( + waitQueueTimeoutMS=10, maxPoolSize=1, event_listeners=[listener] + ) + pool = get_pool(client) + self.assertEqual(pool.opts.wait_queue_timeout, 0.01) + with pool.checkout(): + with self.assertRaises(WaitQueueTimeoutError): + client.test.command("ping") + self.assertFalse(listener.events_by_type(monitoring.PoolClearedEvent)) def test_socketKeepAlive(self): pool = get_pool(self.client) From 6e5126d6bbb1a64f40b0b517c7a43901b474d8a9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 15:47:56 -0500 Subject: [PATCH 2/3] PYTHON-5196 Convert OIDC tests to use new test scripts (#2194) --- .evergreen/config.yml | 153 --------------------- .evergreen/generated_configs/tasks.yml | 44 ++++++ .evergreen/generated_configs/variants.yml | 9 +- .evergreen/run-mongodb-oidc-remote-test.sh | 60 -------- .evergreen/run-mongodb-oidc-test.sh | 31 +---- .evergreen/run-tests.sh | 1 + .evergreen/scripts/generate_config.py | 19 ++- .evergreen/scripts/kms_tester.py | 21 +-- .evergreen/scripts/oidc_tester.py | 99 +++++++++++++ .evergreen/scripts/run_server.py | 5 + .evergreen/scripts/run_tests.py | 19 ++- .evergreen/scripts/setup_tests.py | 12 +- .evergreen/scripts/teardown_tests.py | 6 + .evergreen/scripts/utils.py | 9 +- CONTRIBUTING.md | 9 +- test/auth_oidc/test_auth_oidc.py | 5 + 16 files changed, 237 insertions(+), 265 deletions(-) delete mode 100755 .evergreen/run-mongodb-oidc-remote-test.sh create mode 100644 .evergreen/scripts/oidc_tester.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 54931dcb4..25ea2c4b9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -289,28 +289,6 @@ functions: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-atlas-tests.sh - "run oidc auth test with test credentials": - - command: subprocess.exec - type: test - params: - working_dir: "src" - binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: - - .evergreen/run-mongodb-oidc-test.sh - - "run oidc k8s auth test": - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - env: - OIDC_ENV: k8s - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "K8S_VARIANT"] - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - "cleanup": - command: subprocess.exec params: @@ -417,96 +395,6 @@ task_groups: tasks: - ".serverless" - - name: testazureoidc_task_group - setup_group: - - func: fetch source - - func: setup system - - command: subprocess.exec - params: - binary: bash - env: - AZUREOIDC_VMNAME_PREFIX: "PYTHON_DRIVER" - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/create-and-setup-vm.sh - teardown_task: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - oidc-auth-test-azure - - - name: testgcpoidc_task_group - setup_group: - - func: fetch source - - func: setup system - - command: subprocess.exec - params: - binary: bash - env: - GCPOIDC_VMNAME_PREFIX: "PYTHON_DRIVER" - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh - teardown_task: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - oidc-auth-test-gcp - - - name: testk8soidc_task_group - setup_group: - - func: fetch source - - func: setup system - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - duration_seconds: 1800 - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh - teardown_task: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - oidc-auth-test-k8s - - - name: testoidc_task_group - setup_group: - - func: fetch source - - func: setup system - - func: "assume ec2 role" - - command: subprocess.exec - params: - binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup.sh - teardown_task: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - oidc-auth-test - - name: test_aws_lambda_task_group setup_group: - func: fetch source @@ -659,47 +547,6 @@ tasks: env: TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/test/lambda - - name: "oidc-auth-test" - commands: - - func: "run oidc auth test with test credentials" - - - name: "oidc-auth-test-azure" - commands: - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - env: - OIDC_ENV: azure - include_expansions_in_env: ["DRIVERS_TOOLS"] - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - - - name: "oidc-auth-test-gcp" - commands: - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - env: - OIDC_ENV: gcp - include_expansions_in_env: ["DRIVERS_TOOLS"] - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - - - name: "oidc-auth-test-k8s" - commands: - - func: "run oidc k8s auth test" - vars: - K8S_VARIANT: eks - - func: "run oidc k8s auth test" - vars: - K8S_VARIANT: gke - - func: "run oidc k8s auth test" - vars: - K8S_VARIANT: aks # }}} - name: "coverage-report" tags: ["coverage"] diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 02ee29e6e..9d52cf957 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1042,6 +1042,50 @@ tasks: TEST_NAME: ocsp tags: [ocsp, ocsp-rsa] + # Oidc tests + - name: test-auth-oidc-default + commands: + - func: run tests + vars: + TEST_NAME: auth_oidc + SUB_TEST_NAME: default + tags: [auth_oidc] + - name: test-auth-oidc-azure + commands: + - func: run tests + vars: + TEST_NAME: auth_oidc + SUB_TEST_NAME: azure + tags: [auth_oidc, auth_oidc_remote] + - name: test-auth-oidc-gcp + commands: + - func: run tests + vars: + TEST_NAME: auth_oidc + SUB_TEST_NAME: gcp + tags: [auth_oidc, auth_oidc_remote] + - name: test-auth-oidc-eks + commands: + - func: run tests + vars: + TEST_NAME: auth_oidc + SUB_TEST_NAME: eks + tags: [auth_oidc, auth_oidc_remote] + - name: test-auth-oidc-aks + commands: + - func: run tests + vars: + TEST_NAME: auth_oidc + SUB_TEST_NAME: aks + tags: [auth_oidc, auth_oidc_remote] + - name: test-auth-oidc-gke + commands: + - func: run tests + vars: + TEST_NAME: auth_oidc + SUB_TEST_NAME: gke + tags: [auth_oidc, auth_oidc_remote] + # Server tests - name: test-4.0-standalone-auth-ssl-sync commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 80f08bc7a..cf3e0cc90 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -920,24 +920,21 @@ buildvariants: # Oidc auth tests - name: auth-oidc-ubuntu-22 tasks: - - name: testoidc_task_group - - name: testazureoidc_task_group - - name: testgcpoidc_task_group - - name: testk8soidc_task_group + - name: .auth_oidc display_name: Auth OIDC Ubuntu-22 run_on: - ubuntu2204-small batchtime: 10080 - name: auth-oidc-macos tasks: - - name: testoidc_task_group + - name: .auth_oidc !.auth_oidc_remote display_name: Auth OIDC macOS run_on: - macos-14 batchtime: 10080 - name: auth-oidc-win64 tasks: - - name: testoidc_task_group + - name: .auth_oidc !.auth_oidc_remote display_name: Auth OIDC Win64 run_on: - windows-64-vsMulti-small diff --git a/.evergreen/run-mongodb-oidc-remote-test.sh b/.evergreen/run-mongodb-oidc-remote-test.sh deleted file mode 100755 index bb90bddf0..000000000 --- a/.evergreen/run-mongodb-oidc-remote-test.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -set +x # Disable debug trace -set -eu - -echo "Running MONGODB-OIDC remote tests" - -OIDC_ENV=${OIDC_ENV:-"test"} - -# Make sure DRIVERS_TOOLS is set. -if [ -z "$DRIVERS_TOOLS" ]; then - echo "Must specify DRIVERS_TOOLS" - exit 1 -fi - -# Set up the remote files to test. -git add . -git commit -m "add files" || true -export TEST_TAR_FILE=/tmp/mongo-python-driver.tgz -git archive -o $TEST_TAR_FILE HEAD - -pushd $DRIVERS_TOOLS - -if [ $OIDC_ENV == "test" ]; then - echo "Test OIDC environment does not support remote test!" - exit 1 - -elif [ $OIDC_ENV == "azure" ]; then - export AZUREOIDC_DRIVERS_TAR_FILE=$TEST_TAR_FILE - export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" - bash ./.evergreen/auth_oidc/azure/run-driver-test.sh - -elif [ $OIDC_ENV == "gcp" ]; then - export GCPOIDC_DRIVERS_TAR_FILE=$TEST_TAR_FILE - export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" - bash ./.evergreen/auth_oidc/gcp/run-driver-test.sh - -elif [ $OIDC_ENV == "k8s" ]; then - # Make sure K8S_VARIANT is set. - if [ -z "$K8S_VARIANT" ]; then - echo "Must specify K8S_VARIANT" - popd - exit 1 - fi - - bash ./.evergreen/auth_oidc/k8s/setup-pod.sh - bash ./.evergreen/auth_oidc/k8s/run-self-test.sh - export K8S_DRIVERS_TAR_FILE=$TEST_TAR_FILE - export K8S_TEST_CMD="OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh" - source ./.evergreen/auth_oidc/k8s/secrets-export.sh # for MONGODB_URI - bash ./.evergreen/auth_oidc/k8s/run-driver-test.sh - bash ./.evergreen/auth_oidc/k8s/teardown-pod.sh - -else - echo "Unrecognized OIDC_ENV $OIDC_ENV" - pod - exit 1 -fi - -popd diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 759ac5d2b..bd67106a3 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -3,31 +3,14 @@ set +x # Disable debug trace set -eu -echo "Running MONGODB-OIDC authentication tests" - -OIDC_ENV=${OIDC_ENV:-"test"} - -if [ $OIDC_ENV == "test" ]; then - # Make sure DRIVERS_TOOLS is set. - if [ -z "$DRIVERS_TOOLS" ]; then - echo "Must specify DRIVERS_TOOLS" - exit 1 - fi - source ${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh - -elif [ $OIDC_ENV == "azure" ]; then - source ./env.sh - -elif [ $OIDC_ENV == "gcp" ]; then - source ./secrets-export.sh - -elif [ $OIDC_ENV == "k8s" ]; then - echo "Running oidc on k8s" +echo "Running MONGODB-OIDC authentication tests on ${OIDC_ENV}..." +if [ ${OIDC_ENV} == "k8s" ]; then + SUB_TEST_NAME=$K8S_VARIANT-remote else - echo "Unrecognized OIDC_ENV $OIDC_ENV" - exit 1 + SUB_TEST_NAME=$OIDC_ENV-remote fi - -COVERAGE=1 bash ./.evergreen/just.sh setup-tests auth_oidc +bash ./.evergreen/just.sh setup-tests auth_oidc $SUB_TEST_NAME bash ./.evergreen/just.sh run-tests "${@:1}" + +echo "Running MONGODB-OIDC authentication tests on ${OIDC_ENV}... done." diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 61d505d45..f9a853f27 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,6 +26,7 @@ fi # Source the local secrets export file if available. if [ -f "./secrets-export.sh" ]; then + echo "Sourcing local secrets file" . "./secrets-export.sh" fi diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 505c6de06..14f30fed9 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -663,11 +663,11 @@ def create_serverless_variants(): def create_oidc_auth_variants(): variants = [] - other_tasks = ["testazureoidc_task_group", "testgcpoidc_task_group", "testk8soidc_task_group"] for host_name in ["ubuntu22", "macos", "win64"]: - tasks = ["testoidc_task_group"] if host_name == "ubuntu22": - tasks += other_tasks + tasks = [".auth_oidc"] + else: + tasks = [".auth_oidc !.auth_oidc_remote"] host = HOSTS[host_name] variants.append( create_variant( @@ -884,6 +884,19 @@ def create_aws_tasks(): return tasks +def create_oidc_tasks(): + tasks = [] + for sub_test in ["default", "azure", "gcp", "eks", "aks", "gke"]: + vars = dict(TEST_NAME="auth_oidc", SUB_TEST_NAME=sub_test) + test_func = FunctionCall(func="run tests", vars=vars) + task_name = f"test-auth-oidc-{sub_test}" + tags = ["auth_oidc"] + if sub_test != "default": + tags.append("auth_oidc_remote") + tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) + return tasks + + def _create_ocsp_task(algo, variant, server_type, base_task_name): file_name = f"{algo}-basic-tls-ocsp-{variant}.json" diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index d38ec3a69..40fd65919 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -2,9 +2,16 @@ from __future__ import annotations import os -from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env +from utils import ( + DRIVERS_TOOLS, + LOGGER, + TMP_DRIVER_FILE, + create_archive, + read_env, + run_command, + write_env, +) -TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 DIRS = dict( gcp=f"{DRIVERS_TOOLS}/.evergreen/csfle/gcpkms", azure=f"{DRIVERS_TOOLS}/.evergreen/csfle/azurekms", @@ -45,12 +52,6 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up GCP VM...") -def _create_archive() -> None: - run_command("git add .", cwd=ROOT) - run_command('git commit -m "add files"', check=False, cwd=ROOT) - run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) - - def _load_kms_config(sub_test_target: str) -> dict[str, str]: target_dir = DIRS[sub_test_target] config = read_env(f"{target_dir}/secrets-export.sh") @@ -87,7 +88,7 @@ def setup_kms(sub_test_name: str) -> None: run_command("./setup-secrets.sh", cwd=kms_dir) if success: - _create_archive() + create_archive() if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" @@ -108,7 +109,7 @@ def setup_kms(sub_test_name: str) -> None: write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) -def test_kms_remote(sub_test_name: str) -> None: +def test_kms_send_to_remote(sub_test_name: str) -> None: env = _load_kms_config(sub_test_name) if sub_test_name == "azure": key_name = os.environ["KEY_NAME"] diff --git a/.evergreen/scripts/oidc_tester.py b/.evergreen/scripts/oidc_tester.py new file mode 100644 index 000000000..fd702cf1d --- /dev/null +++ b/.evergreen/scripts/oidc_tester.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import os + +from utils import DRIVERS_TOOLS, TMP_DRIVER_FILE, create_archive, read_env, run_command, write_env + +K8S_NAMES = ["aks", "gke", "eks"] +K8S_REMOTE_NAMES = [f"{n}-remote" for n in K8S_NAMES] + + +def _get_target_dir(sub_test_name: str) -> str: + if sub_test_name == "default": + target_dir = "auth_oidc" + elif sub_test_name.startswith("azure"): + target_dir = "auth_oidc/azure" + elif sub_test_name.startswith("gcp"): + target_dir = "auth_oidc/gcp" + elif sub_test_name in K8S_NAMES + K8S_REMOTE_NAMES: + target_dir = "auth_oidc/k8s" + else: + raise ValueError(f"Invalid sub test name '{sub_test_name}'") + return f"{DRIVERS_TOOLS}/.evergreen/{target_dir}" + + +def setup_oidc(sub_test_name: str) -> dict[str, str] | None: + target_dir = _get_target_dir(sub_test_name) + env = os.environ.copy() + + if sub_test_name == "eks" and "AWS_ACCESS_KEY_ID" in os.environ: + # Store AWS creds for kubectl access. + for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]: + if key in os.environ: + write_env(key, os.environ[key]) + + if sub_test_name == "azure": + env["AZUREOIDC_VMNAME_PREFIX"] = "PYTHON_DRIVER" + if "-remote" not in sub_test_name: + run_command(f"bash {target_dir}/setup.sh", env=env) + if sub_test_name in K8S_NAMES: + run_command(f"bash {target_dir}/setup-pod.sh {sub_test_name}") + run_command(f"bash {target_dir}/run-self-test.sh") + return None + + source_file = None + if sub_test_name == "default": + source_file = f"{target_dir}/secrets-export.sh" + elif sub_test_name in ["azure-remote", "gcp-remote"]: + source_file = "./secrets-export.sh" + if sub_test_name in K8S_REMOTE_NAMES: + return os.environ.copy() + if source_file is None: + return None + + config = read_env(source_file) + write_env("MONGODB_URI_SINGLE", config["MONGODB_URI_SINGLE"]) + write_env("MONGODB_URI", config["MONGODB_URI"]) + write_env("DB_IP", config["MONGODB_URI"]) + + if sub_test_name == "default": + write_env("OIDC_TOKEN_FILE", config["OIDC_TOKEN_FILE"]) + write_env("OIDC_TOKEN_DIR", config["OIDC_TOKEN_DIR"]) + if "OIDC_DOMAIN" in config: + write_env("OIDC_DOMAIN", config["OIDC_DOMAIN"]) + elif sub_test_name == "azure-remote": + write_env("AZUREOIDC_RESOURCE", config["AZUREOIDC_RESOURCE"]) + elif sub_test_name == "gcp-remote": + write_env("GCPOIDC_AUDIENCE", config["GCPOIDC_AUDIENCE"]) + return config + + +def test_oidc_send_to_remote(sub_test_name: str) -> None: + env = os.environ.copy() + target_dir = _get_target_dir(sub_test_name) + create_archive() + if sub_test_name in ["azure", "gcp"]: + upper_name = sub_test_name.upper() + env[f"{upper_name}OIDC_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE + env[ + f"{upper_name}OIDC_TEST_CMD" + ] = f"OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh" + elif sub_test_name in K8S_NAMES: + env["K8S_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE + env["K8S_TEST_CMD"] = "OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh" + run_command(f"bash {target_dir}/run-driver-test.sh", env=env) + + +def teardown_oidc(sub_test_name: str) -> None: + target_dir = _get_target_dir(sub_test_name) + # For k8s, make sure an error while tearing down the pod doesn't prevent + # the Altas server teardown. + error = None + if sub_test_name in K8S_NAMES: + try: + run_command(f"bash {target_dir}/teardown-pod.sh") + except Exception as e: + error = e + run_command(f"bash {target_dir}/teardown.sh") + if error: + raise error diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index 4ef1568ad..f43ada4bb 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -36,6 +36,11 @@ def start_server(): elif test_name == "load_balancer": set_env("LOAD_BALANCER") + elif test_name == "auth_oidc": + raise ValueError( + "OIDC auth does not use run-orchestration directly, do not use run-server!" + ) + elif test_name == "ocsp": opts.ssl = True if "ORCHESTRATION_FILE" not in os.environ: diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index cd781ccd7..38fd3c67c 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -100,18 +100,29 @@ def run() -> None: if TEST_PERF: start_time = datetime.now() - # Run remote kms tests. + # Send kms tests to run remotely. if TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]: - from kms_tester import test_kms_remote + from kms_tester import test_kms_send_to_remote - test_kms_remote(SUB_TEST_NAME) + test_kms_send_to_remote(SUB_TEST_NAME) return - # Run remote ecs tests. + # Send ecs tests to run remotely. if TEST_NAME == "auth_aws" and SUB_TEST_NAME == "ecs": run_command(f"{DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh ecs") return + # Send OIDC tests to run remotely. + if ( + TEST_NAME == "auth_oidc" + and SUB_TEST_NAME != "default" + and not SUB_TEST_NAME.endswith("-remote") + ): + from oidc_tester import test_oidc_send_to_remote + + test_oidc_send_to_remote(SUB_TEST_NAME) + return + if os.environ.get("DEBUG_LOG"): TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG} -o log_cli=1".split()) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index b75a821c3..8432eacd5 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -161,6 +161,13 @@ def handle_test_env() -> None: if group := GROUP_MAP.get(test_name, ""): UV_ARGS.append(f"--group {group}") + if test_name == "auth_oidc": + from oidc_tester import setup_oidc + + config = setup_oidc(sub_test_name) + if not config: + AUTH = "noauth" + if AUTH != "noauth": if test_name == "data_lake": config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh") @@ -174,9 +181,8 @@ def handle_test_env() -> None: write_env("SINGLE_MONGOS_LB_URI", config["SERVERLESS_URI"]) write_env("MULTI_MONGOS_LB_URI", config["SERVERLESS_URI"]) elif test_name == "auth_oidc": - DB_USER = os.environ["OIDC_ADMIN_USER"] - DB_PASSWORD = os.environ["OIDC_ADMIN_PWD"] - write_env("DB_IP", os.environ["MONGODB_URI"]) + DB_USER = config["OIDC_ADMIN_USER"] + DB_PASSWORD = config["OIDC_ADMIN_PWD"] elif test_name == "index_management": config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh") DB_USER = config["DRIVERS_ATLAS_LAMBDA_USER"] diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index fedbdc2fe..988d7ec48 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -24,6 +24,12 @@ elif TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]: teardown_kms(SUB_TEST_NAME) +# Tear down OIDC if applicable. +elif TEST_NAME == "auth_oidc": + from oidc_tester import teardown_oidc + + teardown_oidc(SUB_TEST_NAME) + # Tear down ocsp if applicable. elif TEST_NAME == "ocsp": run_command(f"bash {DRIVERS_TOOLS}/.evergreen/teardown.sh") diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index dcb50cc4d..70a527028 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -13,6 +13,7 @@ from typing import Any HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 LOGGER = logging.getLogger("test") logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") @@ -49,7 +50,7 @@ TEST_SUITE_MAP = { } # Tests that require a sub test suite. -SUB_TEST_REQUIRED = ["auth_aws", "kms"] +SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms"] def get_test_options( @@ -138,3 +139,9 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: kwargs.setdefault("check", True) subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 LOGGER.info("Running command '%s'... done.", cmd) + + +def create_archive() -> None: + run_command("git add .", cwd=ROOT) + run_command('git commit -m "add files"', check=False, cwd=ROOT) + run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d8783d9d..7e70c025e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -236,12 +236,19 @@ the pages will re-render and the browser will automatically refresh. - Set up the test with `just setup-tests load_balancer`. - Run the tests with `just run-tests`. -### AWS tests +### AWS auth tests - Run `just run-server auth_aws` to start the server. - Run `just setup-tests auth_aws ` to set up the AWS test. - Run the tests with `just run-tests`. +### OIDC auth tests + +- Run `just setup-tests auth_oidc ` to set up the OIDC test. +- Run the tests with `just run-tests`. + +The supported types are [`default`, `azure`, `gcp`, `eks`, `aks`, and `gke`]. + ### KMS tests For KMS tests that are run locally, and expected to fail, in this case using `azure`: diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index a5334d79b..6dc36dc8a 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -70,6 +70,11 @@ class OIDCTestBase(PyMongoTestCase): cls.uri_single = os.environ["MONGODB_URI_SINGLE"] cls.uri_multiple = os.environ.get("MONGODB_URI_MULTI") cls.uri_admin = os.environ["MONGODB_URI"] + if ENVIRON == "test": + if not TOKEN_DIR: + raise ValueError("Please set OIDC_TOKEN_DIR") + if not TOKEN_FILE: + raise ValueError("Please set OIDC_TOKEN_FILE") def setUp(self): self.request_called = 0 From 189923f7c3821965ad92b65b1b137f171f9e4842 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 12 Mar 2025 15:19:40 -0700 Subject: [PATCH 3/3] PYTHON-5198 Fix test_03_invalid_keyid (#2195) --- test/asynchronous/test_encryption.py | 3 ++- test/test_encryption.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 000d98a11..728a6f913 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2982,9 +2982,10 @@ class TestAutomaticDecryptionKeys(AsyncEncryptionIntegrationTest): ) async def test_03_invalid_keyid(self): + # checkAuthForCreateCollection can be removed when SERVER-102101 is fixed. with self.assertRaisesRegex( EncryptedCollectionError, - "create.encryptedFields.fields.keyId' is the wrong type 'bool', expected type 'binData", + "(create|checkAuthForCreateCollection).encryptedFields.fields.keyId' is the wrong type 'bool', expected type 'binData", ): await self.client_encryption.create_encrypted_collection( database=self.db, diff --git a/test/test_encryption.py b/test/test_encryption.py index 6efb16744..36c0ab0e2 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2964,9 +2964,10 @@ class TestAutomaticDecryptionKeys(EncryptionIntegrationTest): ) def test_03_invalid_keyid(self): + # checkAuthForCreateCollection can be removed when SERVER-102101 is fixed. with self.assertRaisesRegex( EncryptedCollectionError, - "create.encryptedFields.fields.keyId' is the wrong type 'bool', expected type 'binData", + "(create|checkAuthForCreateCollection).encryptedFields.fields.keyId' is the wrong type 'bool', expected type 'binData", ): self.client_encryption.create_encrypted_collection( database=self.db,