From 894d5e1c7fbc8e884cff81789be4e0dec9d42f31 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Mar 2025 12:53:23 -0500 Subject: [PATCH 1/3] PYTHON-5231 Finish up test scripts and add documentation for creating a new test suite (#2224) --- .evergreen/run-tests.sh | 6 ----- .evergreen/scripts/run_server.py | 5 ---- .evergreen/scripts/setup_tests.py | 36 ++++++++++++++++--------- .evergreen/scripts/utils.py | 45 ++++++++++++++++++++++++++----- CONTRIBUTING.md | 23 ++++++++++++++-- 5 files changed, 82 insertions(+), 33 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6f53ced61..04dd16d34 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -24,12 +24,6 @@ else exit 1 fi -# Source the local secrets export file if available. -if [ -f "./secrets-export.sh" ]; then - echo "Sourcing local secrets file" - . "./secrets-export.sh" -fi - # List the packages. uv sync ${UV_ARGS} --reinstall uv pip list diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f85207daa..5d9aa54e1 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -28,11 +28,6 @@ def start_server(): elif test_name == "load_balancer": set_env("LOAD_BALANCER") - elif test_name == "auth_oidc": - raise ValueError( - "OIDC auth does not use run-orchestration directly, do not use run-server!" - ) - elif test_name == "ocsp": opts.ssl = True if "ORCHESTRATION_FILE" not in os.environ: diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 17f9de1a7..74971bca7 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -115,8 +115,17 @@ def setup_libmongocrypt(): run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") -def get_secrets(name: str) -> None: - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh {name}") +def load_config_from_file(path: str | Path) -> dict[str, str]: + config = read_env(path) + for key, value in config.items(): + write_env(key, value) + return config + + +def get_secrets(name: str) -> dict[str, str]: + secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling") + run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) + return load_config_from_file(secrets_dir / "secrets-export.sh") def handle_test_env() -> None: @@ -158,7 +167,7 @@ def handle_test_env() -> None: # Handle pass through env vars. for var in PASS_THROUGH_ENV: - if is_set(var): + if is_set(var) or getattr(opts, var.lower()): write_env(var, os.environ[var]) if extra := EXTRAS_MAP.get(test_name, ""): @@ -233,12 +242,11 @@ def handle_test_env() -> None: if is_set("MONGODB_URI"): write_env("PYMONGO_MUST_CONNECT", "true") - if is_set("DISABLE_TEST_COMMANDS"): + if is_set("DISABLE_TEST_COMMANDS") or opts.disable_test_commands: write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") if test_name == "enterprise_auth": - get_secrets("drivers/enterprise_auth") - config = read_env(f"{ROOT}/secrets-export.sh") + config = get_secrets("drivers/enterprise_auth") if PLATFORM == "windows": LOGGER.info("Setting GSSAPI_PASS") write_env("GSSAPI_PASS", config["SASL_PASS"]) @@ -316,7 +324,7 @@ def handle_test_env() -> None: write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") - compressors = os.environ.get("COMPRESSORS") + compressors = os.environ.get("COMPRESSORS") or opts.compressor if compressors == "snappy": UV_ARGS.append("--extra snappy") elif compressors == "zstd": @@ -349,13 +357,15 @@ def handle_test_env() -> None: if test_name == "encryption": if not DRIVERS_TOOLS: raise RuntimeError("Missing DRIVERS_TOOLS") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh") + csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") + run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir) + load_config_from_file(csfle_dir / "secrets-export.sh") + run_command(f"bash {csfle_dir}/start-servers.sh") if sub_test_name == "pyopenssl": UV_ARGS.append("--extra ocsp") - if is_set("TEST_CRYPT_SHARED"): + if is_set("TEST_CRYPT_SHARED") or opts.crypt_shared: config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh") CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix() LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR) @@ -414,15 +424,15 @@ def handle_test_env() -> None: # Add coverage if requested. # Only cover CPython. PyPy reports suspiciously low coverage. - if is_set("COVERAGE") and platform.python_implementation() == "CPython": + if (is_set("COVERAGE") or opts.cov) and platform.python_implementation() == "CPython": # Keep in sync with combine-coverage.sh. # coverage >=5 is needed for relative_files=true. UV_ARGS.append("--group coverage") TEST_ARGS = f"{TEST_ARGS} --cov" write_env("COVERAGE") - if is_set("GREEN_FRAMEWORK"): - framework = os.environ["GREEN_FRAMEWORK"] + if is_set("GREEN_FRAMEWORK") or opts.green_framework: + framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"] UV_ARGS.append(f"--group {framework}") else: diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 0ff3b76a5..3eb44f2ab 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -52,7 +52,10 @@ TEST_SUITE_MAP = { # Tests that require a sub test suite. SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"] -EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"] +EXTRA_TESTS = ["mod_wsgi", "aws_lambda"] + +# Tests that do not use run-orchestration. +NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"] def get_test_options( @@ -75,19 +78,47 @@ def get_test_options( else: parser.add_argument( "test_name", - choices=sorted(TEST_SUITE_MAP), + choices=set(TEST_SUITE_MAP) - set(NO_RUN_ORCHESTRATION), nargs="?", default="default", help="The optional name of the test suite to be run, which informs the server configuration.", ) parser.add_argument( - "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level" + "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level." ) parser.add_argument( - "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level" + "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level." ) - parser.add_argument("--auth", action="store_true", help="Whether to add authentication") - parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration") + parser.add_argument("--auth", action="store_true", help="Whether to add authentication.") + parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.") + + # Add the test modifiers. + if require_sub_test_name: + parser.add_argument( + "--debug-log", action="store_true", help="Enable pymongo standard logging." + ) + parser.add_argument("--cov", action="store_true", help="Add test coverage.") + parser.add_argument( + "--green-framework", + nargs=1, + choices=["eventlet", "gevent"], + help="Optional green framework to test against.", + ) + parser.add_argument( + "--compressor", + nargs=1, + choices=["zlib", "zstd", "snappy"], + help="Optional compression algorithm.", + ) + parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.") + parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.") + parser.add_argument( + "--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use." + ) + parser.add_argument( + "--disable-test-commands", action="store_true", help="Disable test commands." + ) + # Get the options. if not allow_extra_opts: opts, extra_opts = parser.parse_args(), [] @@ -113,7 +144,7 @@ def get_test_options( return opts, extra_opts -def read_env(path: Path | str) -> dict[str, Any]: +def read_env(path: Path | str) -> dict[str, str]: config = dict() with Path(path).open() as fid: for line in fid.readlines(): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e75510171..d8cc8c8bc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -217,9 +217,11 @@ the pages will re-render and the browser will automatically refresh. ### Usage -- Run `just run-server` with optional args to set up the server. - All given flags will be passed to `run-orchestration.sh` in `$DRIVERS_TOOLS`. +- Run `just run-server` with optional args to set up the server. All given options will be passed to + `run-orchestration.sh` in `$DRIVERS_TOOLS`. See `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h` + for a full list of options. - Run `just setup-tests` with optional args to set up the test environment, secrets, etc. + See `just setup-tests -h` for a full list of available options. - Run `just run-tests` to run the tests in an appropriate Python environment. - When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server. @@ -346,11 +348,28 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Run the tests: `just run-tests`. ## Enable Debug Logs + - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest`. - Add `log_cli_level = "DEBUG` and `log_cli = 1` to the `tool.pytest.ini_options` section in `pyproject.toml` for Evergreen patches or to enable debug logs by default on your machine. - You can also set `DEBUG_LOG=1` and run either `just setup-tests` or `just-test`. +- Finally, you can use `just setup-tests --debug-log`. - For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for the patch. +## Adding a new test suite + +- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add + to the list of pytest markers in `pyproject.toml`. Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`. If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead. +- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in + `.evergreen/scripts/utils.py`. +- If there is something special required to run the local server or there is an extra flag that should always be set + like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`. +- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`. This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`. +- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`. +- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`. +- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`. +- Regenerate the test variants and tasks using the instructions in `.evergreen/scripts/generate_config.py`. +- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. + ## Re-sync Spec Tests If you would like to re-sync the copy of the specification tests in the From 95cedeefb85fc010e50a2797b1dd461ee36247e8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Mar 2025 13:40:03 -0500 Subject: [PATCH 2/3] PYTHON-5232 Fix aws lambda test setup (#2226) --- .evergreen/scripts/setup_tests.py | 2 +- pyproject.toml | 1 + uv.lock | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 74971bca7..d02cd8759 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -198,7 +198,7 @@ def handle_test_env() -> None: AUTH = "auth" if test_name == "aws_lambda": - UV_ARGS.append("--with pip") + UV_ARGS.append("--group pip") # Store AWS creds if they were given. if "AWS_ACCESS_KEY_ID" in os.environ: for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]: diff --git a/pyproject.toml b/pyproject.toml index f8c25ed60..353f52787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ Tracker = "https://jira.mongodb.org/projects/PYTHON/issues" dev = [ "pre-commit>=4.0" ] +pip = ["pip"] gevent = ["gevent"] eventlet = ["eventlet"] coverage = [ diff --git a/uv.lock b/uv.lock index 758036fe5..39aae339e 100644 --- a/uv.lock +++ b/uv.lock @@ -1036,7 +1036,6 @@ snappy = [ { name = "python-snappy" }, ] test = [ - { name = "pip" }, { name = "pytest" }, { name = "pytest-asyncio" }, ] @@ -1064,6 +1063,9 @@ mockupdb = [ perf = [ { name = "simplejson" }, ] +pip = [ + { name = "pip" }, +] pymongocrypt-source = [ { name = "pymongocrypt" }, ] @@ -1081,7 +1083,6 @@ requires-dist = [ { name = "cryptography", marker = "extra == 'ocsp'", specifier = ">=2.5" }, { name = "dnspython", specifier = ">=1.16.0,<3.0.0" }, { name = "furo", marker = "extra == 'docs'", specifier = "==2024.8.6" }, - { name = "pip", marker = "extra == 'test'" }, { name = "pykerberos", marker = "os_name != 'nt' and extra == 'gssapi'" }, { name = "pymongo-auth-aws", marker = "extra == 'aws'", specifier = ">=1.1.0,<2.0.0" }, { name = "pymongo-auth-aws", marker = "extra == 'encryption'", specifier = ">=1.1.0,<2.0.0" }, @@ -1111,6 +1112,7 @@ eventlet = [{ name = "eventlet" }] gevent = [{ name = "gevent" }] mockupdb = [{ name = "mockupdb", git = "https://github.com/mongodb-labs/mongo-mockup-db?rev=master" }] perf = [{ name = "simplejson" }] +pip = [{ name = "pip" }] pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] typing = [ { name = "mypy", specifier = "==1.14.1" }, From 38ceda4c09d0945b50d296abf9c59e0389c455db Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Mar 2025 13:42:29 -0500 Subject: [PATCH 3/3] PYTHON-5189 Explicitly test drivers on Graviton processors (#2222) --- .evergreen/generated_configs/variants.yml | 11 +++++++++++ .evergreen/scripts/generate_config.py | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index aa20fef89..53e178bd1 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -45,6 +45,17 @@ buildvariants: batchtime: 10080 expansions: NO_EXT: "1" + - name: other-hosts-amazon2023 + tasks: + - name: .latest !.sync_async .sharded_cluster .auth .ssl + - name: .latest !.sync_async .replica_set .noauth .ssl + - name: .latest !.sync_async .standalone .noauth .nossl + display_name: Other hosts Amazon2023 + run_on: + - amazon2023-arm64-latest-large-m8g + batchtime: 10080 + expansions: + NO_EXT: "1" # Atlas connect tests - name: atlas-connect-rhel8-python3.9 diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 0a2496c66..50c81a584 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -73,9 +73,16 @@ HOSTS["perf"] = Host("perf", "rhel90-dbx-perf-large", "", dict()) DEFAULT_HOST = HOSTS["rhel8"] # Other hosts -OTHER_HOSTS = ["RHEL9-FIPS", "RHEL8-zseries", "RHEL8-POWER8", "RHEL8-arm64"] +OTHER_HOSTS = ["RHEL9-FIPS", "RHEL8-zseries", "RHEL8-POWER8", "RHEL8-arm64", "Amazon2023"] for name, run_on in zip( - OTHER_HOSTS, ["rhel92-fips", "rhel8-zseries-small", "rhel8-power-small", "rhel82-arm64-small"] + OTHER_HOSTS, + [ + "rhel92-fips", + "rhel8-zseries-small", + "rhel8-power-small", + "rhel82-arm64-small", + "amazon2023-arm64-latest-large-m8g", + ], ): HOSTS[name] = Host(name, run_on, name, dict()) @@ -772,9 +779,12 @@ def create_alternative_hosts_variants(): handle_c_ext(C_EXTS[0], expansions) for host_name in OTHER_HOSTS: host = HOSTS[host_name] + tags = [".6.0 .standalone !.sync_async"] + if host_name == "Amazon2023": + tags = [f".latest !.sync_async {t}" for t in SUB_TASKS] variants.append( create_variant( - [".6.0 .standalone !.sync_async"], + tags, display_name=get_display_name("Other hosts", host), batchtime=batchtime, host=host,