PYTHON-5277 Convert remaining Evergreen functions to generated config (#2281)

2025-04-14 09:35:37 -05:00 · 2025-04-14 09:35:37 -05:00 · 3723edc199
commit 3723edc199
parent 5d14b3458e
4 changed files with 284 additions and 109 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -29,114 +29,6 @@ include:
  - filename: .evergreen/generated_configs/tasks.yml
  - filename: .evergreen/generated_configs/variants.yml

-functions:
-  "fetch source":
-    # Executes clone and applies the submitted patch, if any
-    - command: git.get_project
-      params:
-        directory: "src"
-    # Applies the subitted patch, if any
-    # Deprecated. Should be removed. But still needed for certain agents (ZAP)
-    - command: git.apply_patch
-
-  "setup system":
-    # Make an evergreen expansion file with dynamic values
-    - command: subprocess.exec
-      params:
-        include_expansions_in_env: ["is_patch", "project", "version_id"]
-        binary: bash
-        working_dir: "src"
-        args:
-          - .evergreen/scripts/setup-system.sh
-    # Load the expansion file to make an evergreen variable with the current unique version
-    - command: expansions.update
-      params:
-        file: src/expansion.yml
-
-  "upload test results":
-    - command: attach.results
-      params:
-        file_location: "${DRIVERS_TOOLS}/results.json"
-    - command: attach.xunit_results
-      params:
-        file: "src/xunit-results/TEST-*.xml"
-
-  "run server":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        working_dir: "src"
-        include_expansions_in_env: [VERSION, TOPOLOGY, AUTH, SSL, ORCHESTRATION_FILE, PYTHON_BINARY, PYTHON_VERSION, IS_WIN32,
-          STORAGE_ENGINE, REQUIRE_API_VERSION, DRIVERS_TOOLS, TEST_CRYPT_SHARED, AUTH_AWS, LOAD_BALANCER, LOCAL_ATLAS, NO_EXT]
-        args: [.evergreen/just.sh, run-server, "${TEST_NAME}"]
-    - command: expansions.update
-      params:
-        file: ${DRIVERS_TOOLS}/mo-expansion.yml
-
-  "run just script":
-    - command: subprocess.exec
-      type: test
-      params:
-       include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
-       binary: bash
-       working_dir: "src"
-       args: [.evergreen/just.sh, "${JUSTFILE_TARGET}"]
-
-  "run tests":
-    - command: subprocess.exec
-      type: test
-      params:
-        include_expansions_in_env: [AUTH, SSL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
-          AWS_SESSION_TOKEN, COVERAGE, PYTHON_BINARY, LIBMONGOCRYPT_URL, MONGODB_URI, PYTHON_VERSION, IS_WIN32,
-          DISABLE_TEST_COMMANDS, GREEN_FRAMEWORK, NO_EXT, COMPRESSORS, MONGODB_API_VERSION, DEBUG_LOG,
-          ORCHESTRATION_FILE, OCSP_SERVER_TYPE, VERSION, REQUIRE_FIPS]
-        binary: bash
-        working_dir: "src"
-        args: [.evergreen/just.sh, setup-tests, "${TEST_NAME}", "${SUB_TEST_NAME}"]
-    - command: subprocess.exec
-      type: test
-      params:
-        working_dir: "src"
-        binary: bash
-        args: [.evergreen/just.sh, run-tests]
-
-  "cleanup":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        working_dir: "src"
-        args:
-          - .evergreen/scripts/cleanup.sh
-
-  "teardown system":
-    - command: subprocess.exec
-      params:
-          binary: bash
-          working_dir: "src"
-          args: [.evergreen/just.sh, teardown-tests]
-    - command: subprocess.exec
-      params:
-        binary: bash
-        working_dir: "src"
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
-
-  "assume ec2 role":
-    - command: ec2.assume_role
-      params:
-        role_arn: ${aws_test_secrets_role}
-        duration_seconds: 3600
-
-  "attach benchmark test results":
-    - command: attach.results
-      params:
-        file_location: src/report.json
-
-  "send dashboard data":
-    - command: perf.send
-      params:
-        file: src/results.json
-
 pre:
  - func: "fetch source"
  - func: "setup system"
--- a/.evergreen/generated_configs/functions.yml
+++ b/.evergreen/generated_configs/functions.yml
@ -1,4 +1,27 @@
 functions:
+  # Assume ec2 role
+  assume ec2 role:
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+        duration_seconds: 3600
+
+  # Attach benchmark test results
+  attach benchmark test results:
+    - command: attach.results
+      params:
+        file_location: src/report.json
+
+  # Cleanup
+  cleanup:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/scripts/cleanup.sh
+        working_dir: src
+      type: test
+
  # Download and merge coverage
  download and merge coverage:
    - command: ec2.assume_role
@ -56,6 +79,142 @@ functions:
        optional: "true"
      type: setup

+  # Fetch source
+  fetch source:
+    - command: git.get_project
+      params:
+        directory: src
+
+  # Run just script
+  run just script:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - ${JUSTFILE_TARGET}
+        working_dir: src
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+      type: test
+
+  # Run server
+  run server:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - run-server
+          - ${TEST_NAME}
+        working_dir: src
+        include_expansions_in_env:
+          - VERSION
+          - TOPOLOGY
+          - AUTH
+          - SSL
+          - ORCHESTRATION_FILE
+          - PYTHON_BINARY
+          - PYTHON_VERSION
+          - STORAGE_ENGINE
+          - REQUIRE_API_VERSION
+          - DRIVERS_TOOLS
+          - TEST_CRYPT_SHARED
+          - AUTH_AWS
+          - LOAD_BALANCER
+          - LOCAL_ATLAS
+          - NO_EXT
+      type: test
+    - command: expansions.update
+      params:
+        file: ${DRIVERS_TOOLS}/mo-expansion.yml
+
+  # Run tests
+  run tests:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - setup-tests
+          - ${TEST_NAME}
+          - ${SUB_TEST_NAME}
+        working_dir: src
+        include_expansions_in_env:
+          - AUTH
+          - SSL
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+          - COVERAGE
+          - PYTHON_BINARY
+          - LIBMONGOCRYPT_URL
+          - MONGODB_URI
+          - PYTHON_VERSION
+          - DISABLE_TEST_COMMANDS
+          - GREEN_FRAMEWORK
+          - NO_EXT
+          - COMPRESSORS
+          - MONGODB_API_VERSION
+          - DEBUG_LOG
+          - ORCHESTRATION_FILE
+          - OCSP_SERVER_TYPE
+          - VERSION
+          - IS_WIN32
+          - REQUIRE_FIPS
+      type: test
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - run-tests
+        working_dir: src
+      type: test
+
+  # Send dashboard data
+  send dashboard data:
+    - command: perf.send
+      params:
+        file: src/results.json
+
+  # Setup system
+  setup system:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/scripts/setup-system.sh
+        working_dir: src
+        include_expansions_in_env:
+          - is_patch
+          - project
+          - version_id
+      type: test
+    - command: expansions.update
+      params:
+        file: src/expansion.yml
+
+  # Teardown system
+  teardown system:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - teardown-tests
+        working_dir: src
+      type: test
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
+        working_dir: src
+      type: test
+
  # Upload coverage
  upload coverage:
    - command: ec2.assume_role
@ -115,3 +274,12 @@ functions:
        display_name: drivers-tools-logs.tar.gz
        optional: "true"
      type: setup
+
+  # Upload test results
+  upload test results:
+    - command: attach.results
+      params:
+        file_location: ${DRIVERS_TOOLS}/results.json
+    - command: attach.xunit_results
+      params:
+        file: src/xunit-results/TEST-*.xml
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@ -37,6 +37,12 @@ from shrub.v3.evg_build_variant import BuildVariant
 from shrub.v3.evg_command import (
    FunctionCall,
    archive_targz_pack,
+    attach_results,
+    attach_xunit_results,
+    ec2_assume_role,
+    expansions_update,
+    git_get_project,
+    perf_send,
 )
 from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef

@ -1097,6 +1103,115 @@ def create_upload_mo_artifacts_func():
    return "upload mo artifacts", cmds


+def create_fetch_source_func():
+    # Executes clone and applies the submitted patch, if any.
+    cmd = git_get_project(directory="src")
+    return "fetch source", [cmd]
+
+
+def create_setup_system_func():
+    # Make an evergreen expansion file with dynamic values.
+    includes = ["is_patch", "project", "version_id"]
+    args = [".evergreen/scripts/setup-system.sh"]
+    setup_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
+    # Load the expansion file to make an evergreen variable with the current unique version.
+    expansion_cmd = expansions_update(file="src/expansion.yml")
+    return "setup system", [setup_cmd, expansion_cmd]
+
+
+def create_upload_test_results_func():
+    results_cmd = attach_results(file_location="${DRIVERS_TOOLS}/results.json")
+    xresults_cmd = attach_xunit_results(file="src/xunit-results/TEST-*.xml")
+    return "upload test results", [results_cmd, xresults_cmd]
+
+
+def create_run_server_func():
+    includes = [
+        "VERSION",
+        "TOPOLOGY",
+        "AUTH",
+        "SSL",
+        "ORCHESTRATION_FILE",
+        "PYTHON_BINARY",
+        "PYTHON_VERSION",
+        "STORAGE_ENGINE",
+        "REQUIRE_API_VERSION",
+        "DRIVERS_TOOLS",
+        "TEST_CRYPT_SHARED",
+        "AUTH_AWS",
+        "LOAD_BALANCER",
+        "LOCAL_ATLAS",
+        "NO_EXT",
+    ]
+    args = [".evergreen/just.sh", "run-server", "${TEST_NAME}"]
+    sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
+    expansion_cmd = expansions_update(file="${DRIVERS_TOOLS}/mo-expansion.yml")
+    return "run server", [sub_cmd, expansion_cmd]
+
+
+def create_run_just_script_func():
+    includes = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+    args = [".evergreen/just.sh", "${JUSTFILE_TARGET}"]
+    cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
+    return "run just script", [cmd]
+
+
+def create_run_tests_func():
+    includes = [
+        "AUTH",
+        "SSL",
+        "AWS_ACCESS_KEY_ID",
+        "AWS_SECRET_ACCESS_KEY",
+        "AWS_SESSION_TOKEN",
+        "COVERAGE",
+        "PYTHON_BINARY",
+        "LIBMONGOCRYPT_URL",
+        "MONGODB_URI",
+        "PYTHON_VERSION",
+        "DISABLE_TEST_COMMANDS",
+        "GREEN_FRAMEWORK",
+        "NO_EXT",
+        "COMPRESSORS",
+        "MONGODB_API_VERSION",
+        "DEBUG_LOG",
+        "ORCHESTRATION_FILE",
+        "OCSP_SERVER_TYPE",
+        "VERSION",
+        "IS_WIN32",
+        "REQUIRE_FIPS",
+    ]
+    args = [".evergreen/just.sh", "setup-tests", "${TEST_NAME}", "${SUB_TEST_NAME}"]
+    setup_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
+    test_cmd = get_subprocess_exec(args=[".evergreen/just.sh", "run-tests"])
+    return "run tests", [setup_cmd, test_cmd]
+
+
+def create_cleanup_func():
+    cmd = get_subprocess_exec(args=[".evergreen/scripts/cleanup.sh"])
+    return "cleanup", [cmd]
+
+
+def create_teardown_system_func():
+    tests_cmd = get_subprocess_exec(args=[".evergreen/just.sh", "teardown-tests"])
+    drivers_cmd = get_subprocess_exec(args=["${DRIVERS_TOOLS}/.evergreen/teardown.sh"])
+    return "teardown system", [tests_cmd, drivers_cmd]
+
+
+def create_assume_ec2_role_func():
+    cmd = ec2_assume_role(role_arn="${aws_test_secrets_role}", duration_seconds=3600)
+    return "assume ec2 role", [cmd]
+
+
+def create_attach_benchmark_test_results_func():
+    cmd = attach_results(file_location="src/report.json")
+    return "attach benchmark test results", [cmd]
+
+
+def create_send_dashboard_data_func():
+    cmd = perf_send(file="src/results.json")
+    return "send dashboard data", [cmd]
+
+
 mod = sys.modules[__name__]
 write_variants_to_file(mod)
 write_tasks_to_file(mod)
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -121,4 +121,4 @@ repos:
    entry: .evergreen/scripts/generate-config.sh
    language: python
    require_serial: true
-    additional_dependencies: ["shrub.py>=3.9.0", "pyyaml>=6.0.2"]
+    additional_dependencies: ["shrub.py>=3.10.0", "pyyaml>=6.0.2"]