SERVER-116106 Re-enable yamllinters Evergreen Linting (#45819)

GitOrigin-RevId: 60c752b57c02f46e480b5938009732598c85f2f7
2026-01-05 23:26:06 -05:00 · 2026-01-05 23:26:06 -05:00 · f43af38b37
commit f43af38b37
parent f6b65d34c7
20 changed files with 262 additions and 129 deletions
--- a/bazel/wrapper_hook/lint.py
+++ b/bazel/wrapper_hook/lint.py
@ -26,6 +26,8 @@ SUPPORTED_EXTENSIONS = (
    ".defs",
    ".inl",
    ".idl",
+    ".yml",
+    ".yaml",
 )


@ -148,8 +150,8 @@ class LintRunner:
            self.fail = True
            if not self.keep_going:
                raise LinterFail("File missing bazel target.")
-
-        print(f"All {type_name} files have BUILD.bazel targets!")
+        else:
+            print(f"All {type_name} files have BUILD.bazel targets!")

    def run_bazel(self, target: str, args: List = []):
        p = subprocess.run([self.bazel_bin, "run", target] + (["--"] + args if args else []))
@ -302,7 +304,7 @@ def run_rules_lint(bazel_bin: str, args: List[str]):
    )
    lint_all = parsed_args.all or "..." in args or "//..." in args
    files_to_lint = [arg for arg in args if not arg.startswith("-")]
-    if not lint_all and files_to_lint:
+    if not lint_all and not files_to_lint:
        origin_branch = parsed_args.origin_branch
        max_distance = 100
        distance = _git_distance([f"{origin_branch}..HEAD"])
@ -335,7 +337,10 @@ def run_rules_lint(bazel_bin: str, args: List[str]):
    if lint_all:
        lr.run_bazel("//buildscripts:pyrightlint", ["lint-all"])
    elif any(file.endswith(".py") for file in files_to_lint):
-        lr.run_bazel("//buildscripts:pyrightlint", ["lints"] + files_to_lint)
+        lr.run_bazel(
+            "//buildscripts:pyrightlint",
+            ["lints"] + [str(file) for file in files_to_lint if file.endswith(".py")],
+        )

    if lint_all or "poetry.lock" in files_to_lint or "pyproject.toml" in files_to_lint:
        lr.run_bazel("//buildscripts:poetry_lock_check")
@ -347,6 +352,7 @@ def run_rules_lint(bazel_bin: str, args: List[str]):
                f"--evg-project-name={parsed_args.lint_yaml_project}",
            ],
        )
+        lr.run_bazel("//buildscripts:yamllinters")

    if lint_all or parsed_args.large_files:
        lr.run_bazel("buildscripts:large_file_check", ["--exclude", "src/third_party/*"])
--- a/buildscripts/BUILD.bazel
+++ b/buildscripts/BUILD.bazel
@ -408,3 +408,24 @@ py_binary(
    visibility = ["//visibility:public"],
    deps = [],  # 'requests' optional (external link checks skipped if absent)
 )
+
+py_binary(
+    name = "yamllinters",
+    srcs = ["yamllinters.py"],
+    main = "yamllinters.py",
+    visibility = ["//visibility:public"],
+    deps = [
+        dependency(
+            "yamllint",
+            group = "lint",
+        ),
+        dependency(
+            "evergreen-lint",
+            group = "lint",
+        ),
+        dependency(
+            "evergreen-py",
+            group = "testing",
+        ),
+    ],
+)
--- a/buildscripts/yamllinters.py
+++ b/buildscripts/yamllinters.py
@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""YAML linters wrapper script for Bazel."""
+
+import os
+import subprocess
+import sys
+
+
+def run_command(cmd, **kwargs):
+    """Run a command and exit on failure."""
+    result = subprocess.run(cmd, **kwargs)
+    if result.returncode != 0:
+        sys.exit(result.returncode)
+
+
+def main():
+    # Change to workspace root if running under Bazel
+    if "BUILD_WORKING_DIRECTORY" in os.environ:
+        os.chdir(os.environ["BUILD_WORKING_DIRECTORY"])
+    else:
+        # Change to repo root (one level up from buildscripts)
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        os.chdir(os.path.join(script_dir, ".."))
+
+    # Run yamllint as a Python module
+    run_command(
+        [
+            sys.executable,
+            "-m",
+            "yamllint",
+            "-c",
+            "etc/yamllint_config.yml",
+            "buildscripts",
+            "etc",
+            "jstests",
+        ],
+        shell=False,
+    )
+
+    # Evaluate evergreen configs
+    # Set up environment with extended PATH for evergreen command
+    env = os.environ.copy()
+    home_dir = env.get("HOME", "")
+
+    evergreen_cmd_base = [f"{home_dir}/evergreen", "evaluate"]
+
+    run_command(
+        evergreen_cmd_base + ["etc/evergreen.yml"],
+        stdout=open("etc/evaluated_evergreen.yml", "w"),
+        env=env,
+    )
+
+    run_command(
+        evergreen_cmd_base + ["etc/evergreen_nightly.yml"],
+        stdout=open("etc/evaluated_evergreen_nightly.yml", "w"),
+        env=env,
+    )
+
+    # Process system_perf.yml
+    # Remove references to the DSI repo before evergreen evaluate.
+    # The DSI module references break 'evaluate', the system_perf config should
+    # parse without them, and we don't want changes to the DSI repository to
+    # break checking that the rest of the imports etc. work.
+    with (
+        open("etc/system_perf.yml", "r") as infile,
+        open("etc/trimmed_system_perf.yml", "w") as outfile,
+    ):
+        drop = False
+        for line in infile:
+            if "lint_yaml trim start" in line:
+                drop = True
+            if "lint_yaml trim end" in line:
+                drop = False
+            if not drop:
+                outfile.write(line)
+
+    run_command(
+        evergreen_cmd_base + ["etc/trimmed_system_perf.yml"],
+        stdout=open("etc/evaluated_system_perf.yml", "w"),
+        env=env,
+    )
+
+    # Run evergreen-lint using module invocation
+    run_command(
+        [sys.executable, "-m", "evergreen_lint", "-c", "./etc/evergreen_lint.yml", "lint"],
+        shell=False,
+    )
+
+    print("YAML linting completed successfully!")
+
+
+if __name__ == "__main__":
+    main()
--- a/buildscripts/yamllinters.sh
+++ b/buildscripts/yamllinters.sh
@ -1,7 +1,11 @@
 set -o errexit

-BASEDIR=$(dirname "$0")
-cd "$BASEDIR/../"
+if [ -n $BUILD_WORKING_DIRECTORY ]; then
+    cd $BUILD_WORKING_DIRECTORY
+else
+    BASEDIR=$(dirname "$0")
+    cd "$BASEDIR/../"
+fi

 yamllint -c etc/yamllint_config.yml buildscripts etc jstests

--- a/etc/evergreen_lint.yml
+++ b/etc/evergreen_lint.yml
@ -25,7 +25,8 @@ rules:
  # # We completed a project to completely eject all shell scripts from the
  # # YAML, and now require this of all shell scripts. Add your scripts to the
  # # evergreen/ directory, and make use of subprocess.exec.
-  - rule: "no-shell-exec"
+  # TODO(SERVER-116120): Re-enable once artifact extract is moved to shell script
+  # - rule: "no-shell-exec"

  # Historically, expansions.update has been used to inject shell scripts
  # into expansions, which were then used to inject commonly used snippets
@ -54,7 +55,7 @@ rules:
  - rule: "enforce-tasks-distro-with-special-tag"
    tags:
      - task_tag_name: "requires_large_host"
-        allowed_distro_regex: "(.*-x*large|.*-medium|macos-.*|ubi.*)"
+        allowed_distro_regex: "(.*large|.*-medium|macos-.*|ubi.*)"

  - rule: "enforce-tags-for-tasks"
    tag_groups:
@ -95,6 +96,10 @@ rules:

          # https://github.com/10gen/mothra/blob/main/mothra/teams/et.yaml
          - assigned_to_jira_team_streams
+
+          # TODO (SERVER-116119): This team currently does not exist, delete this comment and link to the
+          # mothra config file the team is defined in once it's created.
+          - assigned_to_jira_team_platsec_server
        min_num_of_tags: 1
        max_num_of_tags: 1
      # Every task should have required selection tag
@ -139,6 +144,7 @@ rules:
          - requires_large_host_commit_queue
          - requires_all_feature_flags
          - requires_execution_on_windows_patch_build
+          - requires_extra_system_deps

  - rule: "enforce-tags-for-variants"
    tags:
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@ -343,7 +343,7 @@ functions:
      bucket: mciuploads
      local_file: src/mongo-binaries.tgz.sha256

-  "fetch binary zstd shas": &fetch_zstd_binary_shas
+  "fetch binary zstd shas":
    command: s3.get
    display_name: "fetch binary zstd shas"
    params:
@ -1463,6 +1463,7 @@ functions:
  "generate resmoke tasks":
    - *fetch_artifacts
    - *fetch_artifacts_zstd
+    - *f_expansions_write
    - *extract_artifacts
    - *f_expansions_write
    - *kill_processes
@ -3492,7 +3493,7 @@ functions:
        args:
          - "./src/evergreen/container_registry_login.sh"

-  "build and push module images": &build_and_push_module_images
+  "build and push module images":
    command: subprocess.exec
    params:
      binary: bash
--- a/etc/evergreen_yml_components/tasks/compile_tasks.yml
+++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml
@ -484,8 +484,7 @@ tasks:
          display_name: "LibFuzzer Tests"

  ## compile_dbtest ##
-  - &compile_dbtest
-    name: compile_dbtest
+  - name: compile_dbtest
    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
    depends_on:
      - name: archive_dist_test
@ -497,8 +496,7 @@ tasks:
            --config=evg

  ## run_dbtest ##
-  - &run_dbtest
-    name: run_dbtest
+  - name: run_dbtest
    tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
    depends_on:
      - name: compile_dbtest
@ -513,8 +511,7 @@ tasks:
          suite: dbtest
          install_dir: bazel-bin/install/bin

-  - &archive_dbtest
-    name: archive_dbtest
+  - name: archive_dbtest
    tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
    depends_on:
      - name: compile_dbtest
@ -1200,6 +1197,7 @@ tasks:
          content_type: application/tar
          display_name: gcc_pgo
          optional: true
+      - func: "f_expansions_write"
      - command: subprocess.exec
        params:
          binary: bash
@ -1218,6 +1216,7 @@ tasks:
    run_on: rhel8.7-small
    tags: [
        "assigned_to_jira_team_devprod_build",
+        "auxiliary",
        "incompatible_aubsan",
        "incompatible_community",
        "incompatible_development_variant",
--- a/etc/evergreen_yml_components/tasks/misc_tasks.yml
+++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml
@ -56,16 +56,6 @@ variables:
    private_key_file: src/powercycle.pem
    private_key_remote: ${__project_aws_ssh_key_value}

-  - &sbom_upload_config_vars
-    CONTAINER_COMMAND: podman # podman or docker
-    CONTAINER_IMAGE: 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0
-    CONTAINER_OPTIONS: --pull=always --platform="linux/amd64" -it --rm
-    CONTAINER_ENV_FILES: ${workdir}/kondukto_credentials.env
-    CONTAINER_VOLUMES: ${workdir}:/workdir
-    KODUKTO_ROLE_ARN: arn:aws:iam::119629040606:role/kondukto
-    SBOM_FILE_PATH: ${workdir}/sbom.json
-    LOCAL_REPO_PATH: ${workdir}/src
-
 ################################################
 #                    Tasks                     #
 ################################################
@ -1730,7 +1720,7 @@ tasks:
        timeout_secs: 1800 # 30 minute timeout for no output

  - name: push_mongod_to_ecr
-    tags: ["assigned_to_jira_team_disag_mongod"]
+    tags: ["assigned_to_jira_team_server_disagg", "auxiliary"]
    depends_on:
      - name: package
    commands:
@ -1962,7 +1952,7 @@ tasks:
            GITHUB_TOKEN: ${github_token}

  - name: update_sbom
-    tags: []
+    tags: ["auxiliary", "assigned_to_jira_team_platsec_server"]
    exec_timeout_secs: 3600 # 60 minute timeout
    commands:
      - command: manifest.load
@ -1970,6 +1960,7 @@ tasks:
        params:
          directory: src
          clone_depth: 1
+      - func: "f_expansions_write"
      - func: "restore git history and tags"
      - func: "add git tag"
      - func: "f_expansions_write"
@ -1980,28 +1971,18 @@ tasks:
        display_name: Assume Silkbomb IAM role (access to Endor Labs API key)
        params:
          role_arn: arn:aws:iam::119629040606:role/silkbomb
-      - command: shell.exec
+      - command: subprocess.exec
        display_name: Write Endor Labs API credentials to config.yml
        params:
+          binary: bash
          silent: true
-          shell: bash
          include_expansions_in_env:
            [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
          env:
            ENDOR_CONFIG_PATH: "${workdir}/.endorctl"
-          script: |
-            set -e
-            # use AWS CLI to get the Endor Labs API credentials from AWS Secrets Manager
-            ENDOR_API_CREDENTIALS_KEY=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_KEY')
-            ENDOR_API_CREDENTIALS_SECRET=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_SECRET')
-            # save credentials to config file
-            mkdir -p ${workdir}/.endorctl
-            cat << EOF > $ENDOR_CONFIG_PATH/config.yaml
-            ENDOR_API: https://api.endorlabs.com
-            ENDOR_API_CREDENTIALS_KEY: $ENDOR_API_CREDENTIALS_KEY
-            ENDOR_API_CREDENTIALS_SECRET: $ENDOR_API_CREDENTIALS_SECRET
            ENDOR_NAMESPACE: ${ENDOR_NAMESPACE}
-            EOF
+          args:
+            - "src/evergreen/write_endor_credentials.sh"
      - command: subprocess.exec
        display_name: Install endorctl
        params:
@ -2051,7 +2032,7 @@ tasks:

  - name: upload_sbom_via_silkbomb_if_changed
    allowed_requesters: ["commit"]
-    tags: []
+    tags: ["auxiliary", "assigned_to_jira_team_platsec_server"]
    exec_timeout_secs: 600 # 10 minute timeout
    commands:
      - command: manifest.load
@ -2065,19 +2046,17 @@ tasks:
        display_name: Assume Silkbomb IAM role
        params:
          role_arn: arn:aws:iam::119629040606:role/silkbomb
-      - command: shell.exec
+      - command: subprocess.exec
        display_name: Write temporary AWS credentials to Silkbomb environment file
        params:
+          binary: bash
          silent: true
-          shell: bash
          include_expansions_in_env:
            [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
-          script: |
-            cat << EOF > ${workdir}/silkbomb.env
-            AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-            AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-            AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
-            EOF
+          env:
+            SILKBOMB_ENV_FILE: "${workdir}/silkbomb.env"
+          args:
+            - "src/evergreen/write_silkbomb_env.sh"
      - command: ec2.assume_role
        display_name: Assume DevProd Platforms ECR readonly IAM role
        params:
@ -2303,7 +2282,7 @@ tasks:

  - name: create_todo_tickets
    patchable: false
-    tags: []
+    tags: ["auxiliary", "assigned_to_jira_team_devprod_build"]
    exec_timeout_secs: 600 # 10 minute timeout
    commands:
      - command: manifest.load
--- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
+++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
@ -79,51 +79,6 @@ variables:
          resmoke_jobs_max: 1
          suite: benchmark_suite

-  # THIS HAS COPIES IN
-  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
-  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
-  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
-  # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
-  # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES
-  - &jstestfuzz_config_vars
-    is_jstestfuzz: true
-    num_files: 15
-    num_tasks: 5 # Upperbound by `max_sub_suites` specified through the variant with each task still running `num_files` files.
-    # It is error prone to require each fuzzer-related Evergreen task to need to override the
-    # resmoke_args variable. However, the resmoke_args variable must be defined as a string in the
-    # task generation configuration to satisfy mongodb/mongo-task-generator. We therefore specify an
-    # empty string for the variable to reflect there are no additional arguments provided to resmoke
-    # by default for the fuzzer-related tasks.
-    resmoke_args: ""
-    resmoke_jobs_max: 1
-    should_shuffle: false
-    continue_on_failure: false
-    # Terminate the function when there has been no output to stdout for 30 minutes. E.g. when something is stuck in an infinite loop.
-    # resmoke.py writes the test output to logkeeper and only writes to stdout when starting the next test.
-    # resmoke.py not producing output on stdout means that the test is still running and presumably not going to finish.
-    # Note that timeout_secs is different from exec_timeout_secs, which applies to a task and times out regardless of whether output has been written to stdout.
-    timeout_secs: 1800
-
-  # Used for tests that invoke 'resmoke.py --suites=jstestfuzz*'.
-  # THIS HAS COPIES IN
-  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
-  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
-  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
-  # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
-  # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES
-  - &jstestfuzz_template
-    name: jstestfuzz_template
-    exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours.
-    depends_on:
-      - name: version_gen
-        variant: generate-tasks-for-version
-        # This is added because of EVG-18211.
-        # Without this we are adding extra dependencies on evergreen and it is causing strain
-        omit_generated_tasks: true
-      - archive_dist_test
-    commands:
-      - func: "generate resmoke tasks"
-
 ################################################
 #                    Tasks                     #
 ################################################
@ -523,6 +478,7 @@ task_groups:
      - func: "upload mongodatafiles"
      - func: "attach multiversion download links"
      - func: "save failed tests"
+      - func: "f_expansions_write"
      - func: "kill processes"
      - func: "save mongo coredumps"
      - func: "generate hang analyzer tasks"
--- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
+++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
@ -74,7 +74,6 @@ variables:
          suite: benchmark_suite

  # THIS HAS COPIES IN
-  # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
@ -2421,6 +2420,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
        "sharding",
        "jscore",
@ -2433,6 +2433,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
        "jscore",
      ]
@ -2444,6 +2445,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
        "sharding",
      ]
@ -2455,6 +2457,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
      ]
    commands:
@ -2465,6 +2468,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "sharding",
        "jscore",
        "execution_control_with_prioritization",
@ -2477,6 +2481,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
        "jscore",
      ]
@ -2488,6 +2493,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
      ]
    commands:
@ -2498,6 +2504,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
        "sharding",
      ]
@ -2509,6 +2516,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
      ]
    commands:
@ -2519,6 +2527,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_workload_scheduling",
+        "default",
        "execution_control_with_prioritization",
        "auth",
      ]
@ -3023,6 +3032,7 @@ tasks:
    tags:
      [
        "assigned_to_jira_team_server_catalog_and_routing",
+        "default",
        "large",
        "incompatible_mac",
        "common",
--- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
+++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
@ -61,7 +61,6 @@ variables:
          suite: benchmark_suite

  # THIS HAS COPIES IN
-  # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
  # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
--- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
+++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
@ -2350,7 +2350,7 @@ tasks:
        vars:
          suite: query_golden_join_optimization

-  - <<: *task_template
+  - <<: *gen_task_template
    name: query_join_optimization_passthrough_gen
    tags: [
        "assigned_to_jira_team_server_query_optimization",
@ -2362,7 +2362,7 @@ tasks:
        vars:
          resmoke_jobs_max: 5

-  - <<: *task_template
+  - <<: *gen_task_template
    name: query_join_optimization_no_passthrough_with_mongod_gen
    tags: [
        "assigned_to_jira_team_server_query_optimization",
@ -2441,7 +2441,7 @@ tasks:
    name: config_fuzzer_aggregation_mongos_passthrough_gen
    tags:
      [
-        "assigned_to_jira_team_query_integration",
+        "assigned_to_jira_team_server_query_integration",
        "default",
        "config_fuzzer",
        "large",
--- a/etc/evergreen_yml_components/variants/rhel/test_dev.yml
+++ b/etc/evergreen_yml_components/variants/rhel/test_dev.yml
@ -199,10 +199,16 @@ buildvariants:
      - name: sharding_multiversion_gen
      - name: sharding_jscore_multiversion_gen
      - name: sharding_jscore_kill_primary_multiversion_gen
-      - name: .sharding .txns
+      - name: .sharding .txns !.requires_large_host
+      - name: .sharding .txns .requires_large_host
+        distros:
+          - rhel8.8-large
      # Skip csrs stepdown suite because most tests can't handle the first shard stepping down.
      - name: .sharding .common !.csrs !.feature_flag_guarded
-      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host
+        distros:
+          - rhel8.8-large
      - name: .concurrency .sharded !.large
      - name: .concurrency .sharded .large
      - name: .unsplittable_collections
@ -275,8 +281,14 @@ buildvariants:
          - rhel8.8-xlarge
      - name: .rollbackfuzzer
      - name: .sharding .common !.feature_flag_guarded
-      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded
-      - name: .sharding .txns
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host
+        distros:
+          - rhel8.8-medium
+      - name: .sharding .txns !.requires_large_host
+      - name: .sharding .txns .requires_large_host
+        distros:
+          - rhel8.8-medium
      - name: .updatefuzzer
      - name: aggregation_repeat_queries_multiplan_single_solutions
      - name: audit
@ -346,19 +358,27 @@ buildvariants:
      - name: aggregation_mongos_pqs_hints_gen
      - name: aggregation_pqs_fallback_gen
      - name: aggregation_pqs_hints
+        distros:
+          - rhel8.8-medium
      - name: aggregation_sharded_collections_pqs_fallback_gen
      - name: aggregation_sharded_collections_pqs_hints_gen
      - name: aggregation_sharded_collections_query_shape_hash_stability_gen
      - name: replica_sets_jscore_pqs_fallback_gen
      - name: replica_sets_jscore_pqs_hints_gen
      - name: replica_sets_jscore_pqs_index_filters
+        distros:
+          - rhel8.8-medium
      - name: sharded_collections_pqs_fallback_gen
      - name: sharded_collections_pqs_hints_gen
      - name: sharded_collections_pqs_index_filters
+        distros:
+          - rhel8.8-medium
      - name: sharded_collections_query_shape_hash_stability_gen
      - name: sharding_pqs_fallback_gen
      - name: sharding_pqs_hints_gen
      - name: sharding_pqs_index_filters
+        distros:
+          - rhel8.8-medium

  - name: upload-sbom-if-changed
    display_name: "Upload SBOM if changed"
--- a/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml
+++ b/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml
@ -106,7 +106,10 @@ buildvariants:
          - rhel8.8-large
      - name: replica_sets_gen
      - name: .replica_sets .common
-      - name: .replica_sets .multi_oplog !.encrypt
+      - name: .replica_sets .multi_oplog !.encrypt !.requires_large_host
+      - name: .replica_sets .multi_oplog !.encrypt .requires_large_host
+        distros:
+          - rhel8.8-large
      - name: replica_sets_multi_stmt_txn_jscore_passthrough_gen
      - name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen
        distros:
@ -120,7 +123,10 @@ buildvariants:
      - name: sharded_multi_stmt_txn_jscore_passthrough_gen
        distros:
          - rhel8.8-medium
-      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host
+        distros:
+          - rhel8.8-large
      - name: .sharding .common !.multiversion !.csrs !.encrypt !.feature_flag_guarded
      - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen
      - name: .ssl
@ -171,6 +177,8 @@ buildvariants:
      - name: multiversion_auth_gen
      - name: multiversion_gen
      - name: noPassthrough_gen
+        distros:
+          - rhel8.8-large
      - name: .replica_sets !.multi_oplog !.large
      - name: .replica_sets !.multi_oplog .large
        distros:
--- a/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml
+++ b/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml
@ -461,6 +461,8 @@ buildvariants:
        distros:
          - rhel8.8-large
      - name: hotBackups_gen
+        distros:
+          - rhel8.8-large
      - name: generate_buildid_to_debug_symbols_mapping

  - name: &enterprise-rhel8-sdam-replica-set-monitor-64-bit enterprise-rhel8-sdam-replica-set-monitor-64-bit
@ -487,7 +489,10 @@ buildvariants:
          - rhel8.8-large
      - name: .concurrency .common !.kill_terminate !.feature_flag_guarded
      - name: .jscore .common !.requires_large_host
-      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host
+        distros:
+          - rhel8.8-large
      - name: .sharding .common !.csrs !.encrypt !.feature_flag_guarded
      - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen
      - name: generate_buildid_to_debug_symbols_mapping
@ -529,6 +534,8 @@ buildvariants:
      - name: .multi_shard
      - name: multiversion_auth_gen
      - name: noPassthrough_gen
+        distros:
+          - rhel8.8-large
      - name: .query_fuzzer
      - name: .replica_sets !.large
      - name: .resharding_fuzzer
@ -588,8 +595,12 @@ buildvariants:
        patch_only: true
        distros:
          - rhel8.8-large
-      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host
        patch_only: true
+      - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host
+        patch_only: true
+        distros:
+          - rhel8.8-large
      - name: .sharding .common !.csrs !.encrypt !.feature_flag_guarded
        patch_only: true
      - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen
--- a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
+++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
@ -134,7 +134,6 @@ variables:
    #   variant: rhel8-debug-aubsan-compile

  # THIS HAS COPIES IN:
-  # - etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml
  # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
  # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES
  - &linux_debug_aubsan_compile_variant_dependency
@ -846,6 +845,8 @@ buildvariants:
        distros:
          - rhel8.8-large
      - name: noPassthrough_gen
+        distros:
+          - rhel8.8-large
      - name: sharding_gen

  - name: &rhel8-debug-ubsan-roll-back-incremental-feature-flags rhel8-debug-ubsan-roll-back-incremental-feature-flags
--- a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml
+++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml
@ -32,24 +32,6 @@ variables:
    # - name: generate_buildid_to_debug_symbols_mapping
    #   variant: rhel8-debug-aubsan-compile

-  # THIS HAS COPIES IN:
-  # - etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml
-  # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
-  # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES
-  - &linux_debug_aubsan_compile_variant_dependency
-    depends_on:
-      - name: archive_dist_test
-        variant: &linux_debug_aubsan_compile_variant_name linux-debug-aubsan-compile-required
-      - name: archive_jstestshell
-        variant: *linux_debug_aubsan_compile_variant_name
-      - name: version_gen
-        variant: generate-tasks-for-version
-        # This is added because of EVG-18211.
-        # Without this we are adding extra dependencies on evergreen and it is causing strain
-        omit_generated_tasks: true
-    # - name: generate_buildid_to_debug_symbols_mapping
-    #   variant: rhel8-debug-aubsan-compile
-
  # THIS HAS COPIES IN:
  # - etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml
  # - etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml
--- a/evergreen/extract_artifacts.sh
+++ b/evergreen/extract_artifacts.sh
@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -o errexit
+
+# Extract artifacts from either zstd or gzip compressed tarballs
+tar --zstd -xf fetched_artifacts.zst || tar -xf fetched_artifacts.tgz
--- a/evergreen/write_endor_credentials.sh
+++ b/evergreen/write_endor_credentials.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# Write Endor Labs API credentials to config.yml
+# Requires: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, ENDOR_CONFIG_PATH, ENDOR_NAMESPACE
+
+# use AWS CLI to get the Endor Labs API credentials from AWS Secrets Manager
+ENDOR_API_CREDENTIALS_KEY=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_KEY')
+ENDOR_API_CREDENTIALS_SECRET=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_SECRET')
+
+# save credentials to config file
+mkdir -p ${ENDOR_CONFIG_PATH}
+cat <<EOF >${ENDOR_CONFIG_PATH}/config.yaml
+ENDOR_API: https://api.endorlabs.com
+ENDOR_API_CREDENTIALS_KEY: $ENDOR_API_CREDENTIALS_KEY
+ENDOR_API_CREDENTIALS_SECRET: $ENDOR_API_CREDENTIALS_SECRET
+ENDOR_NAMESPACE: ${ENDOR_NAMESPACE}
+EOF
--- a/evergreen/write_silkbomb_env.sh
+++ b/evergreen/write_silkbomb_env.sh
@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# Write temporary AWS credentials to Silkbomb environment file
+# Requires: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, SILKBOMB_ENV_FILE
+
+cat <<EOF >${SILKBOMB_ENV_FILE}
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+EOF