SERVER-120810 SERVER-122566 Improve bazel-resmoke task generation (#50184)

GitOrigin-RevId: e4f1090773ece0cd0f7cf500a5790462331e91c6

bazel/resmoke/BUILD.bazel

@@ -94,6 +94,18 @@ bool_flag(
     build_setting_default = False,
 )
 
+bool_flag(
+    name = "skip_deps_for_cquery",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "skip_deps_for_cquery_enabled",
+    flag_values = {
+        "//bazel/resmoke:skip_deps_for_cquery": "True",
+    },
+)
+
 config_setting(
     name = "installed_dist_test_enabled",
     flag_values = {

bazel/resmoke/resmoke.bzl

@@ -155,6 +155,7 @@ def resmoke_suite_test(
             "//buildscripts:bazel_local_resources",
         ] + select({
             "//bazel/resmoke:installed_dist_test_enabled": [],
+            "//bazel/resmoke:skip_deps_for_cquery_enabled": [],
             "//conditions:default": deps,
         }),
         main = resmoke_shim,
@@ -175,6 +176,7 @@ def resmoke_suite_test(
             "GIT_PYTHON_REFRESH": "quiet",  # Ignore "Bad git executable" error when importing git python. Git commands will still error if run.
         } | select({
             "//bazel/resmoke:installed_dist_test_enabled": {},
+            "//bazel/resmoke:skip_deps_for_cquery_enabled": {},
             "//conditions:default": {"DEPS_PATH": deps_path},
         }),
         exec_properties = exec_properties | test_exec_properties(tags),

buildscripts/BUILD.bazel

@@ -309,6 +309,8 @@ py_binary(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        "//buildscripts/ciconfig",
+        "//buildscripts/util",
         dependency(
             "typer",
             group = "core",
@@ -324,24 +326,6 @@ py_binary(
     ],
 )
 
-py_binary(
-    name = "append_result_tasks",
-    srcs = ["append_result_tasks.py"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//buildscripts:bazel_burn_in",
-        "//buildscripts:gather_failed_tests",
-        dependency(
-            "typer",
-            group = "core",
-        ),
-        dependency(
-            "shrub-py",
-            group = "testing",
-        ),
-    ],
-)
-
 sh_binary(
     name = "setup_node_env",
     srcs = ["setup_node_env.sh"],

buildscripts/OWNERS.yml

@@ -50,7 +50,7 @@ filters:
   - "ciconfig":
     approvers:
       - 10gen/devprod-correctness
-  - "append_result_tasks.py":
+  - "evergreen_activate_result_tasks.py":
     approvers:
       - 10gen/devprod-correctness
   - "generate_result_tasks.py":

buildscripts/append_result_tasks.py

@@ -1,192 +0,0 @@
-"""
-Add generated tasks for displaying bazel test results to the current variant.
-
-This script creates a json config used in Evergreen's generate.tasks to add
-the appropriate subset of tasks to the variant, based on what tests were
-reported to run in the build events JSON.
-
-A task group is used to speed up successive tasks, and reduce the penalty of
-setup costs.
-
-See also: buildscripts/generate_result_tasks.py
-
-Usage:
-   bazel run //buildscripts:append_result_tasks -- --outfile=generated_tasks.json
-
-Options:
-    --outfile           File path for the generated task config.
-    --build_events      Location of the build events JSON, default "./build_events.json".
-"""
-
-import json
-import os
-import re
-
-import typer
-from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, TaskGroup
-from shrub.v2.command import BuiltInCommand
-from shrub.v2.task import ExistingTask
-from typing_extensions import Annotated
-
-from buildscripts.bazel_burn_in import BAZEL_BURN_IN_TESTS
-from buildscripts.gather_failed_tests import process_bep
-from buildscripts.util.read_config import read_config_file
-
-app = typer.Typer(pretty_exceptions_show_locals=False)
-
-
-@app.command()
-def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_events.json"):
-    os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
-
-    expansions = read_config_file("../expansions.yml")
-    build_variant = expansions.get("build_variant")
-    task_name = expansions.get("task_name")
-
-    failed_tests, successful_tests = process_bep(build_events)
-    tasks = failed_tests + successful_tests
-
-    if not tasks:
-        print("No test executions reported in the build events, exiting...")
-        return
-
-    # Shrub's TaskGroup doesn't supporting adding existing tasks, so leave `tasks` empty and patch
-    # the real list in later.
-    task_group = TaskGroup(
-        name=f"{task_name}_results_{build_variant}",
-        tasks=[],
-        max_hosts=len(tasks),
-        setup_group_can_fail_task=True,
-        setup_group=[
-            FunctionCall("git get project and add git tag"),
-            FunctionCall("get engflow cert"),
-            FunctionCall("get engflow key"),
-            BuiltInCommand(
-                "s3.get",
-                {
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_file": "build_events.json",
-                    "remote_file": "${project}/${version_id}/${build_variant}/"
-                    + f"{task_name}/build_events.json",
-                    "bucket": "mciuploads",
-                },
-            ),
-            BuiltInCommand(
-                "s3.get",
-                {
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_file": "resmoke-tests-bazel-invocation.txt",
-                    "remote_file": "${project}/${build_variant}/${revision}/"
-                    + f"bazel-invocation-{task_name}-0.txt",
-                    "bucket": "mciuploads",
-                },
-            ),
-        ],
-        # Between tasks, remove the test logs and outputs. The tasks share hosts and leaving them
-        # can cause the task to include test logs from other bazel targets.
-        setup_task=[BuiltInCommand("shell.exec", {"script": "rm -rf build/ results/ report.json"})],
-        teardown_task=[
-            BuiltInCommand("attach.results", {"file_location": "report.json"}),
-            BuiltInCommand(
-                "s3.put",
-                {
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_file": "bazel-invocation.txt",
-                    "remote_file": "${project}/${build_variant}/${revision}/bazel-invocation-${task_id}.txt",
-                    "bucket": "mciuploads",
-                    "permissions": "public-read",
-                    "content_type": "text/plain",
-                    "display_name": "Bazel invocation for local usage",
-                },
-            ),
-            BuiltInCommand(
-                "s3.put",
-                {
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_files_include_filter_prefix": "results",
-                    "local_files_include_filter": "**/*outputs.zip",
-                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
-                    "bucket": "mciuploads",
-                    "permissions": "private",
-                    "visibility": "signed",
-                    "preserve_path": "true",
-                    "content_type": "application/zip",
-                },
-            ),
-            BuiltInCommand(
-                "s3.put",
-                {
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_files_include_filter_prefix": "results",
-                    "local_files_include_filter": "**/*_MANIFEST",
-                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
-                    "bucket": "mciuploads",
-                    "permissions": "private",
-                    "visibility": "signed",
-                    "preserve_path": "true",
-                    "content_type": "text/plain",
-                },
-            ),
-            BuiltInCommand(
-                "s3.put",
-                {
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_files_include_filter_prefix": "results",
-                    "local_files_include_filter": "**/*test.log",
-                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
-                    "bucket": "mciuploads",
-                    "permissions": "private",
-                    "visibility": "signed",
-                    "preserve_path": "true",
-                    "content_type": "text/plain",
-                },
-            ),
-            FunctionCall("generate result task hang analyzer"),
-        ],
-        teardown_group=[
-            FunctionCall("kill processes"),
-            BuiltInCommand("shell.exec", {"script": "rm -rf build/ results/ report.json"}),
-        ],
-    )
-
-    build_variant = BuildVariant(name=build_variant)
-    if re.match(BAZEL_BURN_IN_TESTS, task_name):
-        build_variant.display_task(
-            display_name="burn_in_tests",
-            execution_existing_tasks=[ExistingTask(task_name)]
-            + [ExistingTask(task) for task in tasks],
-        )
-
-    build_variant.add_task_group(task_group)
-    shrub_project = ShrubProject.empty()
-    shrub_project.add_build_variant(build_variant)
-
-    # Patch in the real list of tasks in the task group.
-    project = shrub_project.as_dict()
-    project["task_groups"][0]["tasks"] = tasks
-
-    # Typical variants running resmoke tests set a variant-wide dependency. During conversion,
-    # these are not a dependency for the `resmoke_tests` task or the results tasks added here.
-    # Set an explicitly depends_on in the task group's reference to override it.
-    # The task that generated the task is used as a no-op dependency, as a workaround for not
-    # being able to set an empty depends_on. Remove with SERVER-119809.
-    if re.match(BAZEL_BURN_IN_TESTS, task_name):
-        depends_on = {"name": "version_burn_in_gen", "variant": "generate-tasks-for-version"}
-    else:
-        depends_on = {"name": "bazel_result_tasks_gen", "variant": "generate-tasks-for-version"}
-    for variant in project.get("buildvariants", []):
-        for task in variant.get("tasks", []):
-            task["depends_on"] = depends_on
-
-    with open(outfile, "w") as f:
-        f.write(json.dumps(project, indent=4))
-
-
-if __name__ == "__main__":
-    app()

buildscripts/bazel_burn_in.py

@@ -34,6 +34,7 @@ import yaml
 from git import Repo
 from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskGroup
 from shrub.v2.command import BuiltInCommand
+from shrub.v2.task import ExistingTask
 from typing_extensions import Annotated
 
 if __name__ == "__main__" and __package__ is None:
@@ -45,7 +46,7 @@ from buildscripts.burn_in_tests import (
     MockFileChangeDetector,
 )
 from buildscripts.ciconfig.evergreen import parse_evergreen_file
-from buildscripts.generate_result_tasks import make_results_task
+from buildscripts.generate_result_tasks import make_results_task, make_task_group
 from buildscripts.util import buildozer_utils as buildozer
 
 BAZEL_BURN_IN_TESTS = r"resmoke_tests_burn_in_*"
@@ -220,26 +221,27 @@ def make_task(targets_to_run, variant_name):
                         "--test_tag_filters=${resmoke_tests_tag_filter},-incompatible_with_bazel_remote_test "
                         "--test_arg=--testTimeout=960 "
                         "--test_timeout=1500 "
-                        "--test_sharding_strategy=disabled "
-                        "--test_arg=--sanityCheck"
+                        "--config=evg "
                     ),
                     "task_compile_flags": (
                         "--keep_going "
                         "--verbose_failures "
                         "--simple_build_id=True "
                         "--define=MONGO_VERSION=${version} "
-                        "--config=evg "
+                        "--linkstatic=True "
                         "--features=strip_debug "
                         "--separate_debug=False "
                         "--remote_download_outputs=minimal "
                         "--zip_undeclared_test_outputs"
                     ),
                     "generate_burn_in_targets": True,
+                    "compiling_for_test": True,
+                    "build_timeout_seconds": 1800,
                 },
             ),
         ],
     )
-    return TaskGroup(
+    return task, TaskGroup(
         name=f"resmoke_tests_burn_in_{variant_name}-TG",
         tasks=[task],
         max_hosts=-1,
@@ -256,35 +258,7 @@ def make_task(targets_to_run, variant_name):
             FunctionCall("get engflow creds"),
         ],
         teardown_task=[
-            BuiltInCommand("generate.tasks", {"optional": True, "files": ["generated_tasks.json"]}),
-            BuiltInCommand(
-                "s3.put",
-                {
-                    "optional": True,
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_file": "src/generated_tasks.json",
-                    "remote_file": "${project}/${version_id}/${build_variant}/${task_name}/generated_tasks.json",
-                    "bucket": "mciuploads",
-                    "permissions": "private",
-                    "visibility": "signed",
-                    "content_type": "application/json",
-                },
-            ),
-            BuiltInCommand(
-                "s3.put",
-                {
-                    "optional": True,
-                    "aws_key": "${aws_key}",
-                    "aws_secret": "${aws_secret}",
-                    "local_file": "src/build_events.json",
-                    "remote_file": "${project}/${version_id}/${build_variant}/${task_name}/build_events.json",
-                    "bucket": "mciuploads",
-                    "permissions": "private",
-                    "visibility": "signed",
-                    "content_type": "application/json",
-                },
-            ),
+            FunctionCall("s3.put bazel build events"),
             FunctionCall("debug full disk"),
             FunctionCall("attach bazel invocation"),
             FunctionCall("save failed tests"),
@@ -333,10 +307,14 @@ def generate_tasks(
 
     evg_conf = parse_evergreen_file("etc/evergreen.yml")
 
+    project = {"tasks": [], "task_groups": [], "buildvariants": []}
+
     shrub_project = ShrubProject.empty()
 
-    results_tasks = []
-    seen_targets = set()
+    targets_all = set()
+
+    resmoke_tests_tasks = []
+    result_tasks = {}
     for variant_name in evg_conf.variant_names:
         variant = evg_conf.get_variant(variant_name)
         if not (variant.is_required_variant() or variant.is_suggested_variant()):
@@ -354,26 +332,60 @@ def generate_tasks(
                 if target.original_target in targets_with_tag
             ]
             if burn_in_targets_to_run:
-                burn_in_task = make_task(burn_in_targets_to_run, variant_name)
+                targets_all.update(burn_in_targets_to_run)
 
-                for target in burn_in_targets_to_run:
-                    if target not in seen_targets:
-                        seen_targets.add(target)
-                        results_tasks.append(make_results_task(target))
+                build_variant = BuildVariant(name=variant.name)
+
+                burn_in_task, burn_in_task_group = make_task(burn_in_targets_to_run, variant_name)
+                resmoke_tests_tasks.append(burn_in_task)
+                build_variant.add_task_group(burn_in_task_group)
+
+                results_task_group = make_task_group(
+                    "resmoke_tests_burn_in",
+                    variant.name,
+                    targets,
+                    f"resmoke_tests_burn_in_{variant.name}",
+                )
+                result_tasks[results_task_group.name] = burn_in_targets_to_run
+                build_variant.add_task_group(results_task_group)
+
+                build_variant.display_task(
+                    display_name="burn_in_tests",
+                    execution_existing_tasks=[ExistingTask(burn_in_task.name)]
+                    + [ExistingTask(task) for task in burn_in_targets_to_run],
+                    activate=False,
+                )
 
-                build_variant = BuildVariant(name=variant_name)
-                build_variant.add_task_group(burn_in_task)
                 shrub_project.add_build_variant(build_variant)
 
-    # Patch in fields that not supported by shrub
     project = shrub_project.as_dict()
-    project["tasks"] = project.get("tasks", [])
+    tasks = [make_results_task(target) for target in targets_all] + [
+        task.as_dict() for task in resmoke_tests_tasks
+    ]
+    project["tasks"] = tasks
+
     for variant in project.get("buildvariants", []):
         for task in variant.get("tasks", []):
             task["activate"] = False
+
+            # Typical variants running resmoke tests set a variant-wide dependency. During conversion,
+            # these are not a dependency for the `resmoke_tests` task or the results tasks added here.
+            # Set an explicitly depends_on in the task group's reference to override it. Remove with SERVER-119809.
+            if task["name"] in result_tasks:
+                task["depends_on"] = {
+                    "name": f"resmoke_tests_burn_in_{variant["name"]}",
+                }
+            else:
+                task["depends_on"] = {
+                    "name": "version_burn_in_gen",
+                    "variant": "generate-tasks-for-version",
+                    "omit_generated_tasks": True,
+                }
     for task in project["tasks"]:
-        task["exec_timeout_secs"] = 1800
-    project["tasks"].extend([task for task in results_tasks])
+        task["exec_timeout_secs"] = 3720
+    for task_group in project.get("task_groups", []):
+        if task_group["name"] in result_tasks:
+            task_group["tasks"] = result_tasks[task_group["name"]]
 
     with open(outfile, "w") as f:
         f.write(json.dumps(project, indent=4))

buildscripts/evergreen_activate_gen_tasks.py

@@ -33,7 +33,6 @@ LOGGER = structlog.getLogger(__name__)
 EVG_CONFIG_FILE = "./.evergreen.yml"
 BURN_IN_TAGS = "burn_in_tags"
 BURN_IN_TESTS = "burn_in_tests"
-BAZEL_BURN_IN_TESTS = r"resmoke_tests_burn_in*"
 BURN_IN_VARIANT_SUFFIX = "generated-by-burn-in-tags"
 
 
@@ -92,7 +91,7 @@ def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None:
                             tasks_not_activated.append(task.task_id)
             else:
                 for task in task_list:
-                    if re.match(BAZEL_BURN_IN_TESTS, task.display_name):
+                    if re.match(BURN_IN_TESTS, task.display_name):
                         LOGGER.info(
                             "Activating task", task_id=task.task_id, task_name=task.display_name
                         )

buildscripts/evergreen_activate_result_tasks.py

@@ -0,0 +1,210 @@
+"""Activate result task groups in the existing build."""
+
+import json
+import os
+import sys
+from typing import Annotated, Optional
+
+import structlog
+import typer
+from urllib3.util import Retry
+
+from evergreen.api import (
+    DEFAULT_HTTP_RETRY_ATTEMPTS,
+    DEFAULT_HTTP_RETRY_BACKOFF_FACTOR,
+    DEFAULT_HTTP_RETRY_CODES,
+    EvergreenApi,
+    RetryingEvergreenApi,
+)
+
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from buildscripts.util.cmdutils import enable_logging
+from buildscripts.util.fileops import read_yaml_file
+
+LOGGER = structlog.getLogger(__name__)
+
+EVG_CONFIG_FILE = "./.evergreen.yml"
+
+app = typer.Typer(pretty_exceptions_show_locals=False)
+
+
+def get_executed_test_labels(build_events_file: str) -> set[str]:
+    """
+    Parse a Bazel build events NDJSON file and return all executed test target labels.
+
+    :param build_events_file: Path to the build_events.json NDJSON file.
+    :return: Set of Bazel target labels that had test results.
+    """
+    labels = set()
+    with open(build_events_file) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            event = json.loads(line)
+            if "testResult" in event:
+                label = event["id"]["testResult"]["label"]
+                labels.add(label)
+    return labels
+
+
+def activate_result_task_group(
+    build_variant: str,
+    task_name: str,
+    version_id: str,
+    evg_api: EvergreenApi,
+    build_events_file: Optional[str] = None,
+) -> None:
+    """
+    Activate the result task group for the given variant and task.
+
+    :param build_variant: The build variant name.
+    :param task_name: The task name (e.g., "resmoke_tests").
+    :param version_id: The Evergreen version ID.
+    :param evg_api: Evergreen API client.
+    :param build_events_file: Optional path to build_events.json. When provided, asserts that
+        every executed test has a corresponding Evergreen task; raises RuntimeError otherwise.
+    """
+    result_task_group_name = f"{task_name}_results_{build_variant}"
+
+    try:
+        version = evg_api.version_by_id(version_id)
+        build_id = version.build_variants_map.get(build_variant)
+
+        if not build_id:
+            LOGGER.warning(
+                "Build variant not found in version",
+                build_variant=build_variant,
+                version_id=version_id,
+            )
+            return
+
+        task_list = evg_api.tasks_by_build(build_id)
+
+        # Collect all task names that need activation
+        tasks_to_activate = []
+        already_activated_count = 0
+        evg_task_names = set()
+        for task in task_list:
+            # Result tasks are bazel targets that start with "//"
+            if task.display_name.startswith("//") and "_burn_in_" not in task.display_name:
+                evg_task_names.add(task.display_name)
+                if task.activated:
+                    already_activated_count += 1
+                    LOGGER.debug(
+                        "Task already activated, skipping",
+                        task_id=task.task_id,
+                        task_name=task.display_name,
+                    )
+                else:
+                    tasks_to_activate.append(task.display_name)
+                    LOGGER.debug(
+                        "Found result task to activate",
+                        task_id=task.task_id,
+                        task_name=task.display_name,
+                    )
+
+        if build_events_file:
+            executed_labels = get_executed_test_labels(build_events_file)
+            missing = executed_labels - evg_task_names
+            if missing:
+                missing_sorted = sorted(missing)
+                LOGGER.error(
+                    "Executed tests have no corresponding Evergreen task — "
+                    "this indicates a bug in task generation",
+                    missing_count=len(missing_sorted),
+                    missing_tasks=missing_sorted,
+                )
+                raise RuntimeError(
+                    f"{len(missing_sorted)} executed test(s) have no corresponding Evergreen task: "
+                    + ", ".join(missing_sorted)
+                )
+
+        if not tasks_to_activate and not already_activated_count:
+            LOGGER.warning(
+                "No result tasks found to activate",
+                task_group=result_task_group_name,
+                build_variant=build_variant,
+            )
+            return
+
+        LOGGER.info(
+            "Activating result tasks",
+            count=len(tasks_to_activate),
+            already_activated=already_activated_count,
+            task_group=result_task_group_name,
+        )
+
+        variants = [{"name": build_variant, "tasks": tasks_to_activate}]
+        evg_api.activate_version_tasks(version_id, variants)
+
+        LOGGER.info(
+            "Successfully activated result tasks",
+            count=len(tasks_to_activate),
+            task_group=result_task_group_name,
+        )
+
+    except Exception:
+        LOGGER.error(
+            "Failed to activate result task group",
+            task_group=result_task_group_name,
+            build_variant=build_variant,
+            version_id=version_id,
+            exc_info=True,
+        )
+        raise
+
+
+@app.command()
+def main(
+    expansion_file: Annotated[
+        str, typer.Option(help="Location of expansions file generated by evergreen.")
+    ],
+    evergreen_config: Annotated[
+        str, typer.Option(help="Location of evergreen configuration file.")
+    ] = EVG_CONFIG_FILE,
+    build_events_file: Annotated[
+        Optional[str],
+        typer.Option(
+            help="Path to the Bazel build events NDJSON file (build_events.json). "
+            "When provided, asserts that every executed test has a corresponding "
+            "Evergreen task, raising an error if any are missing."
+        ),
+    ] = None,
+    verbose: Annotated[bool, typer.Option(help="Enable verbose logging.")] = False,
+) -> None:
+    """
+    Activate the result task group for the current build variant and task.
+    """
+    enable_logging(verbose)
+
+    expansions = read_yaml_file(expansion_file)
+    build_variant = expansions.get("build_variant")
+    task_name = expansions.get("task_name")
+    version_id = expansions.get("version_id")
+
+    if not all([build_variant, task_name, version_id]):
+        LOGGER.error(
+            "Missing required expansions",
+            build_variant=build_variant,
+            task_name=task_name,
+            version_id=version_id,
+        )
+        return
+
+    evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config, log_on_error=True)
+    evg_api._http_retry = Retry(
+        total=DEFAULT_HTTP_RETRY_ATTEMPTS + 10,
+        backoff_factor=DEFAULT_HTTP_RETRY_BACKOFF_FACTOR,
+        status_forcelist=DEFAULT_HTTP_RETRY_CODES,
+        raise_on_status=False,
+        raise_on_redirect=False,
+    )
+
+    activate_result_task_group(build_variant, task_name, version_id, evg_api, build_events_file)
+
+
+if __name__ == "__main__":
+    app()

buildscripts/generate_result_tasks.py

@@ -2,10 +2,12 @@
 Generate tasks for displaying bazel test results for all resmoke bazel tests.
 
 This script creates a json config used in Evergreen's generate.tasks to add
-tasks for displaying test results. It does not add the tasks to any variant.
-They are added to variants by the resmoke_tests task based on which tests ran.
+tasks for displaying test results AND assigns them to appropriate variants upfront.
+
+Queries bazel to determine which test targets will run on each variant based on
+tag filters and other expansions, then generates both task definitions and
+variant assignments in a single step.
 
-See also: buildscripts/append_result_tasks.py
 
 Usage:
    bazel run //buildscripts:generate_result_tasks -- --outfile=generated_tasks.json
@@ -17,6 +19,8 @@ Options:
 import glob
 import json
 import os
+import re
+import shlex
 import subprocess
 import sys
 from functools import cache
@@ -25,10 +29,20 @@ from typing import Optional
 import runfiles
 import typer
 import yaml
-from shrub.v2 import FunctionCall, Task
+from shrub.v2 import BuildVariant, FunctionCall, Task, TaskGroup
+from shrub.v2.command import BuiltInCommand
 from typing_extensions import Annotated
 
+from buildscripts.ciconfig.evergreen import Task as EvergreenTask
+from buildscripts.ciconfig.evergreen import Variant as EvergreenVariant
+from buildscripts.ciconfig.evergreen import parse_evergreen_file
+from buildscripts.util.read_config import read_config_file
+
 RESMOKE_TEST_QUERY = 'attr(tags, "resmoke_suite_test", //...)'
+RESMOKE_TESTS_TAG_FILTER = "resmoke_tests_tag_filter"
+MASTER_PROJECT_NAME = "mongodb-mongo-master"
+MASTER_PROJECT_CONFIG = "etc/evergreen.yml"
+NIGHTLY_PROJECT_CONFIG = "etc/evergreen_nightly.yml"
 
 app = typer.Typer(pretty_exceptions_show_locals=False)
 
@@ -47,6 +61,118 @@ def make_results_task(target: str) -> Task:
     return task
 
 
+def make_task_group(
+    name: str,
+    variant: str,
+    targets,
+    resmoke_task: Optional[str] = "resmoke_tests",
+) -> TaskGroup:
+    task_group = TaskGroup(
+        name=f"{name}_results_{variant}",
+        tasks=[],
+        max_hosts=len(targets),
+        setup_group_can_fail_task=True,
+        setup_group=[
+            FunctionCall("git get project and add git tag"),
+            FunctionCall("get engflow cert"),
+            FunctionCall("get engflow key"),
+            BuiltInCommand(
+                "s3.get",
+                {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_file": "build_events.json",
+                    "remote_file": "${project}/${version_id}/${build_variant}/"
+                    + f"{resmoke_task}/build_events.json",
+                    "bucket": "mciuploads",
+                },
+            ),
+            BuiltInCommand(
+                "s3.get",
+                {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_file": "resmoke-tests-bazel-invocation.txt",
+                    "remote_file": "${project}/${build_variant}/${revision}/"
+                    + f"bazel-invocation-{resmoke_task}-0.txt",
+                    "bucket": "mciuploads",
+                },
+            ),
+        ],
+        # Between tasks, remove the test logs and outputs. The tasks share hosts and leaving them
+        # can cause the task to include test logs from other bazel targets.
+        setup_task=[BuiltInCommand("shell.exec", {"script": "rm -rf build/ results/ report.json"})],
+        teardown_task=[
+            BuiltInCommand("attach.results", {"file_location": "report.json"}),
+            BuiltInCommand(
+                "s3.put",
+                {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_file": "bazel-invocation.txt",
+                    "remote_file": "${project}/${build_variant}/${revision}/bazel-invocation-${task_id}.txt",
+                    "bucket": "mciuploads",
+                    "permissions": "public-read",
+                    "content_type": "text/plain",
+                    "display_name": "Bazel invocation for local usage",
+                },
+            ),
+            BuiltInCommand(
+                "s3.put",
+                {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_files_include_filter_prefix": "results",
+                    "local_files_include_filter": "**/*outputs.zip",
+                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
+                    "bucket": "mciuploads",
+                    "permissions": "private",
+                    "visibility": "signed",
+                    "preserve_path": "true",
+                    "content_type": "application/zip",
+                },
+            ),
+            BuiltInCommand(
+                "s3.put",
+                {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_files_include_filter_prefix": "results",
+                    "local_files_include_filter": "**/*_MANIFEST",
+                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
+                    "bucket": "mciuploads",
+                    "permissions": "private",
+                    "visibility": "signed",
+                    "preserve_path": "true",
+                    "content_type": "text/plain",
+                },
+            ),
+            BuiltInCommand(
+                "s3.put",
+                {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_files_include_filter_prefix": "results",
+                    "local_files_include_filter": "**/*test.log",
+                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
+                    "bucket": "mciuploads",
+                    "permissions": "private",
+                    "visibility": "signed",
+                    "preserve_path": "true",
+                    "content_type": "text/plain",
+                },
+            ),
+            FunctionCall("generate result task hang analyzer"),
+        ],
+        teardown_group=[
+            FunctionCall("kill processes"),
+            BuiltInCommand("shell.exec", {"script": "rm -rf build/ results/ report.json"}),
+        ],
+    )
+
+    return task_group
+
+
 def get_assignment_tag(target: str) -> Optional[str]:
     # Format is like "assigned_to_jira_team_devprod_build".
     # See also docs/evergreen-testing/yaml_configuration/task_ownership_tags.md
@@ -104,7 +230,7 @@ def resolve_assignment_tags() -> dict[str, str]:
 def resolve_codeowners() -> dict[str, list[str]]:
     try:
         result = subprocess.run(
-            'find * -name "BUILD.bazel" | xargs bazel run @codeowners_binary//:codeowners --',
+            'find * -name "BUILD.bazel" | xargs bazel run --config=local @codeowners_binary//:codeowners --',
             shell=True,
             capture_output=True,
             text=True,
@@ -131,32 +257,290 @@ def resolve_codeowners() -> dict[str, list[str]]:
         return {}
 
 
-def query_targets() -> list[str]:
-    try:
-        result = subprocess.run(
-            ["bazel", "query", RESMOKE_TEST_QUERY],
-            capture_output=True,
-            text=True,
-            check=True,
+def expand_evergreen_variables(text: str, expansions: dict) -> str:
+    """Expand Evergreen ${variable} syntax in a string.
+
+    Args:
+        text: String potentially containing ${var} expansions
+        expansions: Dict of expansion values
+
+    Returns:
+        String with ${var} replaced by expansion values
+    """
+
+    def replace_var(match):
+        var_name = match.group(1)
+        return str(expansions.get(var_name, ""))
+
+    return re.sub(r"\$\{([^}]+)\}", replace_var, text)
+
+
+def get_task_vars(task: EvergreenTask, func_name: str = "execute resmoke tests via bazel") -> dict:
+    """Extract vars from a specific function call in task commands."""
+    for command in task.raw.get("commands", []):
+        if command.get("func") == func_name:
+            return command.get("vars", {})
+    return {}
+
+
+def get_variant_expansion(
+    variant: EvergreenVariant, task: EvergreenTask, expansion_name: str
+) -> str:
+    """Get expansion value from variant or task vars.
+
+    Checks variant expansions first, then task vars, then returns defaults.
+    """
+    task_vars = get_task_vars(task)
+    value = task_vars.get(expansion_name)
+    if value:
+        return value
+
+    value = variant.expansion(expansion_name)
+    if value:
+        return value
+
+    return ""
+
+
+def query_targets(
+    variant,
+    resmoke_task,
+    expansions,
+) -> list[str]:
+    target_pattern = expansions.get("resmoke_test_targets", "//...")
+
+    tag_filter = get_variant_expansion(variant, resmoke_task, RESMOKE_TESTS_TAG_FILTER)
+    tags = [t.strip() for t in tag_filter.split(",") if t.strip()]
+    if not tags:
+        print(
+            f"Warning: No tags found in filter '{tag_filter}' for variant {variant.name}",
+            file=sys.stderr,
         )
-        # Parse the output - each line is a target label
-        return [line.strip() for line in result.stdout.strip().split("\n") if line.strip()]
-    except subprocess.CalledProcessError as e:
-        print(f"Bazel query failed with return code {e.returncode}")
-        print(f"Command: {' '.join(e.cmd)}")
-        print(f"STDOUT:\n{e.stdout}")
-        print(f"STDERR:\n{e.stderr}")
-        raise
+        return []
+
+    bazel_flags = []
+    for flag_name in ["bazel_args", "bazel_compile_flags", "task_compile_flags"]:
+        flag_value = get_variant_expansion(variant, resmoke_task, flag_name)
+        if flag_value:
+            flag_value = expand_evergreen_variables(flag_value, expansions)
+            bazel_flags.extend(shlex.split(flag_value))
+
+    flags_list = list(bazel_flags)
+    flags_list.append("--//bazel/resmoke:skip_deps_for_cquery")
+    flags_list.append("--noincompatible_enable_cc_toolchain_resolution")
+    flags_list.append("--repo_env=no_c++_toolchain=1")
+    flags_list.append("--keep_going")
+
+    # If target_pattern contains multiple space-separated targets, wrap them in set()
+    # to create valid Bazel query syntax
+    if " " in target_pattern and not target_pattern.startswith("set("):
+        target_pattern = f"set({target_pattern})"
+
+    # Query for tests with tags that match the variant. Only py_test rules are considered,
+    # since resmoke_suite_test is a macro for a py_test.
+    if len(tags) == 1:
+        # Single tag - simple query
+        tag = tags[0]
+        query = f"attr(tags, '\\b{tag}(?![a-zA-Z0-9_-])', kind('py_test', {target_pattern}))"
+    else:
+        # Multiple tags - use + operator to combine them in a single query
+        tag_queries = [
+            f"attr(tags, '\\b{tag}(?![a-zA-Z0-9_-])', kind('py_test', {target_pattern}))"
+            for tag in tags
+        ]
+        query = " + ".join(tag_queries)
+
+    cmd = (
+        ["bazel", "cquery"]
+        + flags_list
+        + [
+            query,
+            "--output=starlark",
+            "--starlark:expr",
+            'target.label if "IncompatiblePlatformProvider" not in providers(target) else ""',
+        ]
+    )
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    targets = [
+        line.strip().removeprefix("@@")
+        for line in result.stdout.strip().split("\n")
+        if line.strip()
+    ]
+    print(f"Variant {variant.name}: Found {len(targets)} targets total", file=sys.stderr)
+
+    if target_pattern == "//..." and not targets:
+        error_msg = (
+            f"Bazel cquery failed. No targets found for variant {variant.name}\n"
+            f"Bazel cquery: {query}\n"
+            f"Command: {' '.join(cmd)}\n"
+            f"STDOUT:\n{result.stdout}\n"
+            f"STDERR:\n{result.stderr}"
+        )
+        raise RuntimeError(error_msg)
+
+    return targets
+
+
+def create_task_group_for_variant(variant_name: str, task_name: str, targets: list[str]) -> dict:
+    """Create task group definition for displaying test results.
+
+    Structure is similar to append_result_tasks.py but generated upfront.
+    """
+    return {
+        "name": f"{task_name}_results_{variant_name}",
+        "tasks": targets,
+        "max_hosts": len(targets),
+        "setup_group_can_fail_task": True,
+        "setup_group": [
+            {"func": "git get project and add git tag"},
+            {"func": "get engflow cert"},
+            {"func": "get engflow key"},
+            {
+                "command": "s3.get",
+                "params": {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_file": "build_events.json",
+                    "remote_file": f"${{project}}/${{version_id}}/${{build_variant}}/{task_name}/build_events.json",
+                    "bucket": "mciuploads",
+                },
+            },
+            {
+                "command": "s3.get",
+                "params": {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_file": "resmoke-tests-bazel-invocation.txt",
+                    "remote_file": f"${{project}}/${{build_variant}}/${{revision}}/bazel-invocation-{task_name}-0.txt",
+                    "bucket": "mciuploads",
+                },
+            },
+        ],
+        "setup_task": [
+            {"command": "shell.exec", "params": {"script": "rm -rf build/ results/ report.json"}}
+        ],
+        "teardown_task": [
+            {"command": "attach.results", "params": {"file_location": "report.json"}},
+            {
+                "command": "s3.put",
+                "params": {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_file": "bazel-invocation.txt",
+                    "remote_file": "${project}/${build_variant}/${revision}/bazel-invocation-${task_id}.txt",
+                    "bucket": "mciuploads",
+                    "permissions": "public-read",
+                    "content_type": "text/plain",
+                    "display_name": "Bazel invocation for local usage",
+                },
+            },
+            {
+                "command": "s3.put",
+                "params": {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_files_include_filter_prefix": "results",
+                    "local_files_include_filter": "**/*outputs.zip",
+                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
+                    "bucket": "mciuploads",
+                    "permissions": "private",
+                    "visibility": "signed",
+                    "preserve_path": "true",
+                    "content_type": "application/zip",
+                },
+            },
+            {
+                "command": "s3.put",
+                "params": {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_files_include_filter_prefix": "results",
+                    "local_files_include_filter": "**/*_MANIFEST",
+                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
+                    "bucket": "mciuploads",
+                    "permissions": "private",
+                    "visibility": "signed",
+                    "preserve_path": "true",
+                    "content_type": "text/plain",
+                },
+            },
+            {
+                "command": "s3.put",
+                "params": {
+                    "aws_key": "${aws_key}",
+                    "aws_secret": "${aws_secret}",
+                    "local_files_include_filter_prefix": "results",
+                    "local_files_include_filter": "**/*test.log",
+                    "remote_file": "${project}/${build_variant}/${revision}/${task_id}/",
+                    "bucket": "mciuploads",
+                    "permissions": "private",
+                    "visibility": "signed",
+                    "preserve_path": "true",
+                    "content_type": "text/plain",
+                },
+            },
+            {"func": "generate result task hang analyzer"},
+        ],
+        "teardown_group": [
+            {"func": "kill processes"},
+            {"command": "shell.exec", "params": {"script": "rm -rf build/ results/ report.json"}},
+        ],
+    }
+
+
+def get_evergreen_config_path(project_name: str) -> str:
+    if project_name == MASTER_PROJECT_NAME:
+        return MASTER_PROJECT_CONFIG
+    return NIGHTLY_PROJECT_CONFIG
 
 
 @app.command()
 def main(outfile: Annotated[str, typer.Option()]):
     os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
 
-    test_targets = query_targets()
+    expansions = read_config_file("../expansions.yml")
+    project_name = expansions.get("project", MASTER_PROJECT_NAME)
+    evg_config_path = get_evergreen_config_path(project_name)
 
-    tasks = [make_results_task(target) for target in test_targets]
-    project = {"tasks": [task for task in tasks]}
+    print(f"Parsing Evergreen configuration from {evg_config_path}...", file=sys.stderr)
+    evg_config = parse_evergreen_file(evg_config_path)
+
+    project = {"tasks": [], "task_groups": [], "buildvariants": []}
+
+    targets_all = set()
+    for variant in evg_config.variants:
+        resmoke_task = variant.get_task("resmoke_tests")
+        if not resmoke_task:
+            continue
+
+        targets = query_targets(variant, resmoke_task, expansions)
+        if not targets:
+            continue
+        targets_all.update(targets)
+
+        task_group = make_task_group("resmoke_tests", variant.name, targets).as_dict()
+        task_group["tasks"] = targets
+        project["task_groups"].append(task_group)
+
+        build_variant = BuildVariant(name=variant.name).as_dict()
+        # Typical variants running resmoke tests set a variant-wide dependency. During conversion,
+        # these are not a dependency for the `resmoke_tests` task or the results tasks added here.
+        # Set an explicitly depends_on in the task group's reference to override it.
+        # The task that generated the task is used as a no-op dependency, as a workaround for not
+        # being able to set an empty depends_on. Remove with SERVER-119809.
+        build_variant["tasks"] = {
+            "name": task_group["name"],
+            "activate": False,
+            "depends_on": {
+                "name": "bazel_result_tasks_gen",
+                "variant": "generate-tasks-for-version",
+                "omit_generated_tasks": True,
+            },
+        }
+        project["buildvariants"].append(build_variant)
+
+        project["tasks"] = [make_results_task(target) for target in targets_all]
 
     with open(outfile, "w") as f:
         f.write(json.dumps(project, indent=4))

evergreen/fetch_remote_test_results.sh

@@ -23,6 +23,11 @@ function is_failure() {
     jq --exit-status '.testResult | select(.status != "PASSED")' <<<$1 >/dev/null
 }
 
+# Checks if a test result record indicates that the test timed out.
+function is_timeout() {
+    jq --exit-status '.testResult | select(.status == "TIMEOUT")' <<<$1 >/dev/null
+}
+
 # Returns a file-path safe prefix for an individual test execution.
 function target_prefix() {
     jq --raw-output '.id.testResult as $id | .testResult | "\(($id.label | ltrimstr("//") | gsub(":";"\/")))/shard_\($id.shard)"' <<<$1
@@ -34,14 +39,17 @@ function download_outputs() {
     local is_failure=$2
 
     jq --raw-output '.id.testResult as $id | .testResult.testActionOutput[] | "\t\($id.shard)\t\(.name)\t\(.uri)"' <<<"$test_result" | while IFS=$'\t' read -r shard name uri; do
-        # Always download test.outputs (zip file)
-        # If test failed, also download test.log and manifest
+        # Always download test.outputs (zip file) and test.log
+        # If test failed, also download manifest
         should_download=false
         if [[ "$name" == *'test.outputs'* && "$name" != *'manifest'* ]]; then
             should_download=true
         fi
+        if [[ "$name" == *'test.log'* ]]; then
+            should_download=true
+        fi
         if [[ "$is_failure" == "1" ]]; then
-            if [[ "$name" == *'test.log'* || "$name" == *'manifest__MANIFEST'* ]]; then
+            if [[ "$name" == *'manifest__MANIFEST'* ]]; then
                 should_download=true
             fi
         fi
@@ -74,12 +82,10 @@ function unzip_outputs() {
     if [[ -n "$zip_file" ]]; then
         local output_dir='test.outputs'
         mkdir -p "$output_dir"
-        echo "  Unzipping: $zip_file -> $output_dir/"
         unzip -o -q "$zip_file" -d "$output_dir"
 
         # If test passed, remove the zip file. If it failed, the whole thing is going to be attached to the task in Evergreen.
         if [[ "$is_failure" != '1' ]]; then
-            echo "  Removing: $zip_file"
             rm "$zip_file"
         fi
     fi
@@ -93,7 +99,6 @@ function symlink_test_logs() {
         return
     fi
 
-    echo "  Creating symlinks in ${workdir}/build/..."
     find "$build_dir" -type f | while read -r file; do
         # Get the relative path from the build directory
         rel_path="${file#$build_dir/}"
@@ -107,6 +112,59 @@ function symlink_test_logs() {
     done
 }
 
+# Displays a formatted summary of test results.
+function display_test_summary() {
+    echo "================================================================================"
+    echo "Test Results Summary"
+    echo "================================================================================"
+    echo "Target: ${test_label}"
+    echo "Total Shards: ${#shard_names[@]}"
+    echo "--------------------------------------------------------------------------------"
+
+    # Create a sorted list of indices based on shard names
+    local sorted_indices=()
+    for i in "${!shard_names[@]}"; do
+        sorted_indices+=("$i")
+    done
+
+    # Sort indices by extracting and comparing shard numbers
+    IFS=$'\n' sorted_indices=($(
+        for i in "${sorted_indices[@]}"; do
+            local shard_num=$(echo "${shard_names[$i]}" | grep -oP 'shard_\K\d+$')
+            echo "$shard_num $i"
+        done | sort -n | cut -d' ' -f2
+    ))
+
+    for i in "${sorted_indices[@]}"; do
+        local shard="${shard_names[$i]}"
+        local status="${shard_statuses[$i]}"
+        local test_counts="${shard_test_counts[$i]}"
+
+        # Format status with color indicators
+        case "$status" in
+        "PASSED")
+            echo "  ✓ $shard: PASSED ($test_counts tests passed)"
+            ;;
+        "FAILED")
+            if [[ "$test_counts" == "0/0" ]]; then
+                echo "  ✗ $shard: FAILED (no report generated)"
+            else
+                echo "  ✗ $shard: FAILED ($test_counts tests passed)"
+            fi
+            ;;
+        "TIMEOUT")
+            echo "  ⏱ $shard: TIMEOUT"
+            ;;
+        "NO_REPORT")
+            echo "  ✗ $shard: NO REPORT (no tests may have been run)"
+            ;;
+        esac
+    done
+
+    echo "================================================================================"
+    echo ""
+}
+
 # Combine all resmoke telemetry and place it where Evergreen expects it: ${workdir}/build/OTelTraces.
 # Metrics are batched into line-separated JSON files no greater than 4MB each. Evergreen processes
 # fewer files faster, but hits message size limitations if they are too large.
@@ -142,8 +200,6 @@ function combine_metrics() {
         # Update current size
         current_size=$((current_size + file_size + newline_size))
     done
-
-    echo 'Combined OTel metrics json'
 }
 
 # Combines all Resmoke test report JSONs into a single JSON.
@@ -164,11 +220,13 @@ function combine_reports() {
 
     local combined_report_file="${workdir}/report.json"
     echo "$combined_report" >"$combined_report_file"
-    echo "Combined report written to: $combined_report_file"
 
     local total_tests=$(echo "$combined_report" | jq '.results | length')
     local failures=$(echo "$combined_report" | jq '.failures')
-    echo "Summary: $total_tests tests, $failures failures"
+
+    echo ""
+    echo "Combined Report: ${total_tests} tests, ${failures} failures"
+    echo "Report written to: $combined_report_file"
 }
 
 # Writes a user-friendly bazel invocation for re-running this test target.
@@ -186,10 +244,33 @@ function write_test_failures_expansion() {
     echo "test_failures_exist: true" >"$output_file"
 }
 
-# Print the contents of all *test.log files.
+# Print the contents of all *test.log files with headers per shard.
 function print_executor_logs() {
-    echo "Executor logs for all failed shards:"
-    find "${workdir}/results" -name '*test.log' -type f -exec cat {} +
+    local log_files=$(find "${workdir}/results" -name '*test.log' -type f 2>/dev/null)
+
+    if [[ -z "$log_files" ]]; then
+        return
+    fi
+
+    # Sort log files by shard number
+    local sorted_log_files=$(echo "$log_files" | while IFS= read -r log_file; do
+        # Extract shard number from path (e.g., /workdir/results/foo/bar/shard_1/test.log -> 1)
+        local shard_num=$(echo "$log_file" | grep -oP 'shard_\K\d+(?=/)')
+        echo "$shard_num $log_file"
+    done | sort -n | cut -d' ' -f2-)
+
+    while IFS= read -r log_file; do
+        # Extract shard name from path (e.g., /workdir/results/foo/bar/shard_1/test.log -> foo/bar/shard_1)
+        local shard_path=$(echo "$log_file" | sed "s|${workdir}/results/||" | sed 's|/[^/]*$||')
+
+        echo "================================================================================"
+        echo "Shard $shard_path log:"
+        echo "================================================================================"
+        cat "$log_file"
+        echo ""
+        echo "================================================================================"
+        echo ""
+    done <<<"$sorted_log_files"
 }
 
 # Resolves a file path from a list of candidate locations. Returns the first existing file path found.
@@ -238,15 +319,29 @@ if [ ! -f "$ENGFLOW_KEY" ]; then
 fi
 
 fail_task=0
+result_count=0
+missing_report=0
+shard_names=()
+shard_statuses=()
+shard_test_counts=()
+
+echo "Fetching test results for ${test_label}..."
+
 while IFS= read -r test_result; do
+    ((result_count++))
     target_prefix=$(target_prefix "$test_result")
     target_dir="${workdir}/results/$target_prefix"
-    echo "Fetching results for $target_prefix"
     mkdir -p "$target_dir"
     pushd "$target_dir" >/dev/null
 
     is_failure_flag=0
-    if is_failure "$test_result"; then
+    is_timeout_flag=0
+    if is_timeout "$test_result"; then
+        is_timeout_flag=1
+        is_failure_flag=1
+        fail_task=1
+        write_test_failures_expansion
+    elif is_failure "$test_result"; then
         is_failure_flag=1
         fail_task=1
         write_test_failures_expansion
@@ -256,26 +351,78 @@ while IFS= read -r test_result; do
     unzip_outputs "$is_failure_flag"
     symlink_test_logs
 
+    # Record shard information
+    shard_names+=("$target_prefix")
+    # Check if any report*.json files exist
+    if compgen -G "test.outputs/report*.json" >/dev/null; then
+        # Extract test counts from the report
+        report_file=$(compgen -G "test.outputs/report*.json" | head -n 1)
+        total_tests=$(jq '.results | length' "$report_file" 2>/dev/null || echo "0")
+        failed_tests=$(jq '.results | map(select(.status == "fail" or .status == "timeout")) | length' "$report_file" 2>/dev/null || echo "0")
+        passed_tests=$(jq '.results | map(select(.status == "pass")) | length' "$report_file" 2>/dev/null || echo "0")
+
+        shard_test_counts+=("$passed_tests/$total_tests")
+
+        if [[ "$is_timeout_flag" -eq 1 ]]; then
+            shard_statuses+=("TIMEOUT")
+        elif [[ "$is_failure_flag" -eq 1 ]]; then
+            shard_statuses+=("FAILED")
+        else
+            shard_statuses+=("PASSED")
+        fi
+    else
+        # No report file found - check if we have bazel-level status information
+        if [[ "$is_timeout_flag" -eq 1 ]]; then
+            shard_statuses+=("TIMEOUT")
+            shard_test_counts+=("0/0")
+        else
+            shard_statuses+=("NO_REPORT")
+            shard_test_counts+=("0/0")
+            missing_report=1
+        fi
+    fi
+
     popd >/dev/null
 done < <(enumerate_test_results)
 
+# Check if any results were found
+if [[ "$result_count" -eq 0 ]]; then
+    echo "Error: No test results found for target '${test_label}' in '$BEP_FILE'." >&2
+    echo "The test may have failed to build. Check the logs from the resmoke_tests task." >&2
+    exit 1
+fi
+
+print_executor_logs
+
+display_test_summary
+
 combine_metrics
 
 failures=$(combine_reports)
 
 write_bazel_invocation
 
-# If there are failures, let Evergreen mark the test as failed by the test results by exiting 0 here.
-# If there are no failures in the combined report, but the bazel test failed, report
-# it as a system failure by returning $fail_task.
-if [[ "$failures" == 'No report.json files found' ]]; then
-    if [[ "$fail_task" -eq 1 ]]; then
-        echo 'No report/test logs were found, but the bazel test failed. Check the test executor logs below.'
+# Check for system-level failures (TIMEOUT or NO_REPORT)
+for status in "${shard_statuses[@]}"; do
+    if [[ "$status" == "TIMEOUT" || "$status" == "NO_REPORT" ]]; then
+        echo "Error: One or more shards had TIMEOUT or NO_REPORT status. Not all tests ran or were reported." >&2
+        write_test_failures_expansion
+        exit 1
     fi
+done
+
+# Check for test failures
+# If there are test failures, write the expansion and exit 0 to let Evergreen mark as failed via test results
+has_test_failures=0
+for status in "${shard_statuses[@]}"; do
+    if [[ "$status" == "FAILED" ]]; then
+        has_test_failures=1
+        break
+    fi
+done
+
+if [[ "$has_test_failures" -eq 1 ]]; then
     write_test_failures_expansion
-    print_executor_logs
-    exit $fail_task
-else
-    print_executor_logs
-    exit 0
 fi
+
+exit 0

evergreen/resmoke_tests_execute_bazel.sh

@@ -112,7 +112,10 @@ if [[ "$RET" != "0" ]]; then
     eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_tests || true
 fi
 
-eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:append_result_tasks -- --outfile=generated_tasks.json
+if [ "${generate_burn_in_targets}" != "true" ]; then
+    echo "Activating result task group..."
+    python buildscripts/evergreen_activate_result_tasks.py --expansion-file ../expansions.yml --build-events-file build_events.json
+fi
 
 eval ${BAZEL_BINARY} shutdown # Explicitly shutdown the bazel server in case the Evergreen agent is tracking it for completion of this process.
 

poetry.lock

@@ -1009,21 +1009,22 @@ typing-extensions = ">=3.10.0"
 
 [[package]]
 name = "evergreen-py"
-version = "3.13.0"
+version = "3.15.0"
 description = "Python client for the Evergreen API"
 optional = false
-python-versions = "<4.0,>=3.9"
+python-versions = "<3.14,>=3.9"
 groups = ["testing"]
-markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\""
+markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and python_version < \"3.14\""
 files = [
-    {file = "evergreen_py-3.13.0-py3-none-any.whl", hash = "sha256:7728ce5bdbf27066879c4b59f9617197c76fa97d08bed9eb3cd67a2b2fd26bd8"},
-    {file = "evergreen_py-3.13.0.tar.gz", hash = "sha256:cde2802af94eb5245d9670643fccf9dea777a33bf1cceb5b4cfd57c1bc72054e"},
+    {file = "evergreen_py-3.15.0-py3-none-any.whl", hash = "sha256:ed172814276e2e62ecec040df3035735ea8feb933b51795f8e88504d919b1b4f"},
+    {file = "evergreen_py-3.15.0.tar.gz", hash = "sha256:9c000f4b3a3e14ca6b644062479607db6f39da2dc51b1ccba9879aa7bdcb115e"},
 ]
 
 [package.dependencies]
 Click = ">=8,<9"
 packaging = ">=25.0,<26.0"
 pydantic = ">=1"
+PyJWT = ">=2.0,<3.0"
 python-dateutil = ">=2"
 PyYAML = ">=5"
 requests = ">=2"
@@ -5918,4 +5919,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<4.0"
-content-hash = "c1531f3ed6e9a5a5492221184f226041f98cfefc2b0928e5ad87e49a4d707e76"
+content-hash = "b3c0b08dcf9577f0717ef413ec56b951c45d8321d2a9a6715acfd307e2eb1ea6"

pyproject.toml

@@ -147,7 +147,7 @@ curatorbin = "^1.2.4"
 PyKMIP = {git = "https://github.com/mongodb-forks/PyKMIP.git", rev = "c48cb01635819e478b573e3245ef840a11d78865"}
 kafka-python = "^2.0.2"
 avro-python3 = "^1.10.2"
-evergreen-py = ">=3.13.0,<3.14.0"
+evergreen-py = { version = "^3.15.0", python = ">=3.10,<3.14"}
 mock = "^5.1.0"
 shrub-py = "^3.1.4"
 ocspresponder = "^0.5.0"