From 7243b43e635d30da8816e4bbe5b6b30cb96fa807 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Apr 2025 10:42:43 -0500 Subject: [PATCH 1/2] PYTHON-5245 Convert remaining tasks to generated config (#2255) --- .evergreen/config.yml | 60 -------------------------- .evergreen/generated_configs/tasks.yml | 58 +++++++++++++++++++++++++ .evergreen/scripts/generate_config.py | 54 ++++++++++++++++++++++- .pre-commit-config.yaml | 2 +- 4 files changed, 111 insertions(+), 63 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1d9de12c3..a297c4916 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -290,63 +290,3 @@ post: - func: "upload mo artifacts" - func: "upload test results" - func: "cleanup" - -tasks: - # Wildcard task. Do you need to find out what tools are available and where? - # Throw it here, and execute this task on all buildvariants - - name: getdata - commands: - - command: subprocess.exec - binary: bash - type: test - params: - args: - - src/.evergreen/scripts/run-getdata.sh - - - name: "coverage-report" - tags: ["coverage"] - depends_on: - # BUILD-3165: We can't use "*" (all tasks) and specify "variant". - # Instead list out all coverage tasks using tags. - - name: ".standalone" - variant: ".coverage_tag" - # Run the coverage task even if some tasks fail. - status: "*" - # Run the coverage task even if some tasks are not scheduled in a patch build. - patch_optional: true - - name: ".replica_set" - variant: ".coverage_tag" - status: "*" - patch_optional: true - - name: ".sharded_cluster" - variant: ".coverage_tag" - status: "*" - patch_optional: true - commands: - - func: "download and merge coverage" - - - name: "check-import-time" - tags: ["pr"] - commands: - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - include_expansions_in_env: ["PYTHON_BINARY"] - args: - - .evergreen/scripts/check-import-time.sh - - ${revision} - - ${github_commit} - - name: "backport-pr" - allowed_requesters: ["commit"] - commands: - - command: subprocess.exec - type: test - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh - - mongodb - - mongo-python-driver - - ${github_commit} diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index e70f0cbdf..c42cf444b 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -713,6 +713,39 @@ tasks: AWS_ROLE_SESSION_NAME: test tags: [auth-aws, auth-aws-web-identity] + # Backport pr tests + - name: backport-pr + commands: + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh + - mongodb + - mongo-python-driver + - ${github_commit} + working_dir: src + type: test + + # Coverage report tests + - name: coverage-report + commands: + - func: download and merge coverage + depends_on: + - name: .standalone + variant: .coverage_tag + status: "*" + patch_optional: true + - name: .replica_set + variant: .coverage_tag + status: "*" + patch_optional: true + - name: .sharded_cluster + variant: .coverage_tag + status: "*" + patch_optional: true + tags: [coverage] + # Doctest tests - name: test-doctests commands: @@ -776,6 +809,31 @@ tasks: - func: run tests tags: [free-threading] + # Getdata tests + - name: getdata + commands: + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/run-getdata.sh + working_dir: src + type: test + + # Import time tests + - name: check-import-time + commands: + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/check-import-time.sh + - ${revision} + - ${github_commit} + working_dir: src + type: test + tags: [pr] + # Kms tests - name: test-gcpkms commands: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 419bead92..824786ed4 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -9,9 +9,9 @@ from pathlib import Path from typing import Any from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_command import FunctionCall +from shrub.v3.evg_command import EvgCommandType, FunctionCall, subprocess_exec from shrub.v3.evg_project import EvgProject -from shrub.v3.evg_task import EvgTask, EvgTaskRef +from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef from shrub.v3.shrub_service import ShrubService ############## @@ -233,6 +233,13 @@ def handle_c_ext(c_ext, expansions) -> None: expansions["NO_EXT"] = "1" +def get_subprocess_exec(**kwargs): + kwargs.setdefault("binary", "bash") + kwargs.setdefault("working_dir", "src") + kwargs.setdefault("command_type", EvgCommandType.TEST) + return subprocess_exec(**kwargs) + + def generate_yaml(tasks=None, variants=None): """Generate the yaml for a given set of tasks and variants.""" project = EvgProject(tasks=tasks, buildvariants=variants) @@ -1055,6 +1062,49 @@ def create_atlas_data_lake_tasks(): return tasks +def create_getdata_tasks(): + # Wildcard task. Do you need to find out what tools are available and where? + # Throw it here, and execute this task on all buildvariants + cmd = get_subprocess_exec(args=[".evergreen/scripts/run-getdata.sh"]) + return [EvgTask(name="getdata", commands=[cmd])] + + +def create_coverage_report_tasks(): + tags = ["coverage"] + task_name = "coverage-report" + # BUILD-3165: We can't use "*" (all tasks) and specify "variant". + # Instead list out all coverage tasks using tags. + # Run the coverage task even if some tasks fail. + # Run the coverage task even if some tasks are not scheduled in a patch build. + task_deps = [] + for name in [".standalone", ".replica_set", ".sharded_cluster"]: + task_deps.append( + EvgTaskDependency(name=name, variant=".coverage_tag", status="*", patch_optional=True) + ) + cmd = FunctionCall(func="download and merge coverage") + return [EvgTask(name=task_name, tags=tags, depends_on=task_deps, commands=[cmd])] + + +def create_import_time_tasks(): + name = "check-import-time" + tags = ["pr"] + args = [".evergreen/scripts/check-import-time.sh", "${revision}", "${github_commit}"] + cmd = get_subprocess_exec(args=args) + return [EvgTask(name=name, tags=tags, commands=[cmd])] + + +def create_backport_pr_tasks(): + name = "backport-pr" + args = [ + "${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh", + "mongodb", + "mongo-python-driver", + "${github_commit}", + ] + cmd = get_subprocess_exec(args=args) + return [EvgTask(name=name, commands=[cmd], allowed_requesters=["commit"])] + + def create_ocsp_tasks(): tasks = [] tests = [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bab2ea47d..deab0724a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -121,4 +121,4 @@ repos: entry: .evergreen/scripts/generate-config.sh language: python require_serial: true - additional_dependencies: ["shrub.py>=3.2.0", "pyyaml>=6.0.2"] + additional_dependencies: ["shrub.py>=3.8.0", "pyyaml>=6.0.2"] From 5177e4ec5305ffff983449ad41e8da84b782a283 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Apr 2025 14:20:44 -0500 Subject: [PATCH 2/2] PYTHON-5261 Clean up compression variants (#2257) --- .evergreen/generated_configs/tasks.yml | 93 ++++++++++++++++++++++- .evergreen/generated_configs/variants.yml | 84 +++----------------- .evergreen/scripts/generate_config.py | 83 +++++++++++--------- 3 files changed, 151 insertions(+), 109 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index c42cf444b..b2b8dc119 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -9,14 +9,14 @@ tasks: tags: [atlas_connect] # Atlas data lake tests - - name: test-atlas-data-lake-with_ext + - name: test-atlas-data-lake-without_ext commands: - func: run tests vars: TEST_NAME: data_lake NO_EXT: "1" tags: [atlas_data_lake] - - name: test-atlas-data-lake-without_ext + - name: test-atlas-data-lake-with_ext commands: - func: run tests vars: @@ -727,6 +727,95 @@ tasks: working_dir: src type: test + # Compression tests + - name: test-compression-v4.0-python3.9 + commands: + - func: run server + vars: + VERSION: "4.0" + - func: run tests + tags: [compression, "4.0"] + - name: test-compression-v4.2-python3.9 + commands: + - func: run server + vars: + VERSION: "4.2" + - func: run tests + tags: [compression, "4.2"] + - name: test-compression-v4.4-python3.9 + commands: + - func: run server + vars: + VERSION: "4.4" + - func: run tests + tags: [compression, "4.4"] + - name: test-compression-v5.0-python3.9 + commands: + - func: run server + vars: + VERSION: "5.0" + - func: run tests + tags: [compression, "5.0"] + - name: test-compression-v6.0-python3.9 + commands: + - func: run server + vars: + VERSION: "6.0" + - func: run tests + tags: [compression, "6.0"] + - name: test-compression-v7.0-python3.9 + commands: + - func: run server + vars: + VERSION: "7.0" + - func: run tests + tags: [compression, "7.0"] + - name: test-compression-v8.0-python3.9 + commands: + - func: run server + vars: + VERSION: "8.0" + - func: run tests + tags: [compression, "8.0"] + - name: test-compression-rapid-python3.9 + commands: + - func: run server + vars: + VERSION: rapid + - func: run tests + tags: [compression, rapid] + - name: test-compression-latest-python3.9 + commands: + - func: run server + vars: + VERSION: latest + - func: run tests + tags: [compression, latest] + - name: test-compression-latest-python3.13-no-c + commands: + - func: run server + vars: + VERSION: latest + - func: run tests + vars: + NO_EXT: "1" + tags: [compression, latest] + - name: test-compression-latest-python3.13 + commands: + - func: run server + vars: + VERSION: latest + - func: run tests + vars: {} + tags: [compression, latest] + - name: test-compression-latest-pypy3.10 + commands: + - func: run server + vars: + VERSION: latest + - func: run tests + tags: [compression, latest] + # Coverage report tests - name: coverage-report commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 892e50f9d..7082dda44 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -160,90 +160,30 @@ buildvariants: - rhel87-small # Compression tests - - name: compression-snappy-rhel8-python3.9-no-c + - name: compression-snappy-rhel8 tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Compression snappy RHEL8 Python3.9 No C + - name: .compression + display_name: Compression snappy RHEL8 run_on: - rhel87-small expansions: - COMPRESSORS: snappy - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: compression-snappy-rhel8-python3.10 + COMPRESSOR: snappy + - name: compression-zlib-rhel8 tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Compression snappy RHEL8 Python3.10 + - name: .compression + display_name: Compression zlib RHEL8 run_on: - rhel87-small expansions: - COMPRESSORS: snappy - PYTHON_BINARY: /opt/python/3.10/bin/python3 - - name: compression-zlib-rhel8-python3.11-no-c + COMPRESSOR: zlib + - name: compression-zstd-rhel8 tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Compression zlib RHEL8 Python3.11 No C + - name: .compression !.4.0 + display_name: Compression zstd RHEL8 run_on: - rhel87-small expansions: - COMPRESSORS: zlib - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.11/bin/python3 - - name: compression-zlib-rhel8-python3.12 - tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Compression zlib RHEL8 Python3.12 - run_on: - - rhel87-small - expansions: - COMPRESSORS: zlib - PYTHON_BINARY: /opt/python/3.12/bin/python3 - - name: compression-zstd-rhel8-python3.13-no-c - tasks: - - name: .standalone .noauth .nossl .sync_async !.4.0 - display_name: Compression zstd RHEL8 Python3.13 No C - run_on: - - rhel87-small - expansions: - COMPRESSORS: zstd - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.13/bin/python3 - - name: compression-zstd-rhel8-python3.9 - tasks: - - name: .standalone .noauth .nossl .sync_async !.4.0 - display_name: Compression zstd RHEL8 Python3.9 - run_on: - - rhel87-small - expansions: - COMPRESSORS: zstd - PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: compression-snappy-rhel8-pypy3.10 - tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Compression snappy RHEL8 PyPy3.10 - run_on: - - rhel87-small - expansions: - COMPRESSORS: snappy - PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 - - name: compression-zlib-rhel8-pypy3.10 - tasks: - - name: .standalone .noauth .nossl .sync_async - display_name: Compression zlib RHEL8 PyPy3.10 - run_on: - - rhel87-small - expansions: - COMPRESSORS: zlib - PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 - - name: compression-zstd-rhel8-pypy3.10 - tasks: - - name: .standalone .noauth .nossl .sync_async !.4.0 - display_name: Compression zstd RHEL8 PyPy3.10 - run_on: - - rhel87-small - expansions: - COMPRESSORS: zstd - PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + COMPRESSOR: zstd # Coverage report tests - name: coverage-report diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 824786ed4..723ef6ba3 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -26,7 +26,7 @@ MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] BATCHTIME_WEEK = 10080 AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")] TOPOLOGIES = ["standalone", "replica_set", "sharded_cluster"] -C_EXTS = ["with_ext", "without_ext"] +C_EXTS = ["without_ext", "with_ext"] # By default test each of the topologies with a subset of auth/ssl. SUB_TASKS = [ ".sharded_cluster .auth .ssl", @@ -217,7 +217,7 @@ def get_variant_name(base: str, host: Host | None = None, **kwargs) -> str: def get_task_name(base: str, **kwargs): - return get_common_name(base, "-", **kwargs).lower() + return get_common_name(base, "-", **kwargs).replace(" ", "-").lower() def zip_cycle(*iterables, empty_default=None): @@ -430,42 +430,22 @@ def create_load_balancer_variants(): def create_compression_variants(): - # Compression tests - standalone versions of each server, across python versions, with and without c extensions. - # PyPy interpreters are always tested without extensions. + # Compression tests - standalone versions of each server, across python versions. host = DEFAULT_HOST - base_task = ".standalone .noauth .nossl .sync_async" - task_names = dict(snappy=[base_task], zlib=[base_task], zstd=[f"{base_task} !.4.0"]) + base_task = ".compression" variants = [] - for ind, (compressor, c_ext) in enumerate(product(["snappy", "zlib", "zstd"], C_EXTS)): - expansions = dict(COMPRESSORS=compressor) - handle_c_ext(c_ext, expansions) - base_name = f"Compression {compressor}" - python = CPYTHONS[ind % len(CPYTHONS)] - display_name = get_variant_name(base_name, host, python=python, **expansions) - variant = create_variant( - task_names[compressor], - display_name, - python=python, - host=host, - expansions=expansions, + for compressor in "snappy", "zlib", "zstd": + expansions = dict(COMPRESSOR=compressor) + tasks = [base_task] if compressor != "zstd" else [f"{base_task} !.4.0"] + display_name = get_variant_name(f"Compression {compressor}", host) + variants.append( + create_variant( + tasks, + display_name, + host=host, + expansions=expansions, + ) ) - variants.append(variant) - - other_pythons = PYPYS + CPYTHONS[ind:] - for compressor, python in zip_cycle(["snappy", "zlib", "zstd"], other_pythons): - expansions = dict(COMPRESSORS=compressor) - handle_c_ext(c_ext, expansions) - base_name = f"Compression {compressor}" - display_name = get_variant_name(base_name, host, python=python, **expansions) - variant = create_variant( - task_names[compressor], - display_name, - python=python, - host=host, - expansions=expansions, - ) - variants.append(variant) - return variants @@ -866,6 +846,39 @@ def create_load_balancer_tasks(): return tasks +def create_compression_tasks(): + tasks = [] + versions = get_versions_from("4.0") + # Test all server versions with min python. + for version in versions: + python = CPYTHONS[0] + tags = ["compression", version] + name = get_task_name("test-compression", python=python, version=version) + server_func = FunctionCall(func="run server", vars=dict(VERSION=version)) + test_func = FunctionCall(func="run tests") + tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + + # Test latest with max python, with and without c exts. + version = "latest" + tags = ["compression", "latest"] + for c_ext in C_EXTS: + python = CPYTHONS[-1] + expansions = dict() + handle_c_ext(c_ext, expansions) + name = get_task_name("test-compression", python=python, version=version, **expansions) + server_func = FunctionCall(func="run server", vars=dict(VERSION=version)) + test_func = FunctionCall(func="run tests", vars=expansions) + tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + + # Test on latest with pypy. + python = PYPYS[-1] + name = get_task_name("test-compression", python=python, version=version) + server_func = FunctionCall(func="run server", vars=dict(VERSION=version)) + test_func = FunctionCall(func="run tests") + tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + return tasks + + def create_kms_tasks(): tasks = [] for kms_type in ["gcp", "azure"]: