diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 462f5743c58..c6ca7fbb06c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -78,6 +78,11 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot /buildscripts/golden_test.py @10gen/query-optimization @svc-auto-approve-bot /buildscripts/evergreen_gen_streams* @10gen/streams-engine @svc-auto-approve-bot /buildscripts/compare_evergreen_versions.py @10gen/devprod-correctness @svc-auto-approve-bot +/buildscripts/ciconfig @10gen/devprod-correctness @svc-auto-approve-bot +/buildscripts/append_result_tasks.py @10gen/devprod-correctness @svc-auto-approve-bot +/buildscripts/generate_result_tasks.py @10gen/devprod-correctness @svc-auto-approve-bot +/buildscripts/evergreen_activate_gen_tasks.py @10gen/devprod-correctness @svc-auto-approve-bot +/buildscripts/bazel_burn_in.py @10gen/devprod-correctness @svc-auto-approve-bot # The following patterns are parsed from ./buildscripts/antithesis/OWNERS.yml /buildscripts/antithesis/ @10gen/devprod-correctness @svc-auto-approve-bot @@ -331,8 +336,8 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot /buildscripts/tests/test_evergreen_task_timeout.py @10gen/devprod-correctness @svc-auto-approve-bot /buildscripts/tests/test_generate_sbom.py @10gen/code-review-team-ssdlc @svc-auto-approve-bot -# The following patterns are parsed from ./buildscripts/tests/burn_in_tests_end2end/OWNERS.yml -/buildscripts/tests/burn_in_tests_end2end/ @10gen/devprod-correctness @svc-auto-approve-bot +# The following patterns are parsed from ./buildscripts/tests/burn_in/OWNERS.yml +/buildscripts/tests/burn_in/ @10gen/devprod-correctness @svc-auto-approve-bot # The following patterns are parsed from ./buildscripts/tests/monitor_build_status/OWNERS.yml /buildscripts/tests/monitor_build_status/ @10gen/devprod-correctness @svc-auto-approve-bot diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index 2d4bbce6487..639781b15ac 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -263,12 +263,48 @@ py_binary( ], ) +py_binary( + name = "burn_in_tests", + srcs = ["burn_in_tests.py"], + visibility = ["//visibility:public"], + deps = [ + "//buildscripts/patch_builds", + "//buildscripts/resmokelib", + dependency( + "structlog", + group = "evergreen", + ), + dependency( + "gitpython", + group = "evergreen", + ), + ], +) + +py_binary( + name = "bazel_burn_in", + srcs = ["bazel_burn_in.py"], + visibility = ["//visibility:public"], + deps = [ + "//buildscripts:burn_in_tests", + "//buildscripts:generate_result_tasks", + "//buildscripts/util", + dependency( + "typer", + group = "core", + ), + dependency( + "shrub-py", + group = "testing", + ), + ], +) + py_binary( name = "generate_result_tasks", srcs = ["generate_result_tasks.py"], visibility = ["//visibility:public"], deps = [ - "//buildscripts:gather_failed_tests", dependency( "typer", group = "core", @@ -285,6 +321,7 @@ py_binary( srcs = ["append_result_tasks.py"], visibility = ["//visibility:public"], deps = [ + "//buildscripts:bazel_burn_in", "//buildscripts:gather_failed_tests", dependency( "typer", diff --git a/buildscripts/OWNERS.yml b/buildscripts/OWNERS.yml index a52109a7732..20c130f0994 100644 --- a/buildscripts/OWNERS.yml +++ b/buildscripts/OWNERS.yml @@ -47,3 +47,18 @@ filters: - "compare_evergreen_versions.py": approvers: - 10gen/devprod-correctness + - "ciconfig": + approvers: + - 10gen/devprod-correctness + - "append_result_tasks.py": + approvers: + - 10gen/devprod-correctness + - "generate_result_tasks.py": + approvers: + - 10gen/devprod-correctness + - "evergreen_activate_gen_tasks.py": + approvers: + - 10gen/devprod-correctness + - "bazel_burn_in.py": + approvers: + - 10gen/devprod-correctness diff --git a/buildscripts/append_result_tasks.py b/buildscripts/append_result_tasks.py index f262725613d..4b355a797db 100644 --- a/buildscripts/append_result_tasks.py +++ b/buildscripts/append_result_tasks.py @@ -20,12 +20,15 @@ Options: import json import os +import re import typer from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, TaskGroup from shrub.v2.command import BuiltInCommand +from shrub.v2.task import ExistingTask from typing_extensions import Annotated +from buildscripts.bazel_burn_in import BAZEL_BURN_IN_TESTS from buildscripts.gather_failed_tests import process_bep from buildscripts.util.read_config import read_config_file @@ -38,6 +41,7 @@ def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_eve expansions = read_config_file("../expansions.yml") build_variant = expansions.get("build_variant") + task_name = expansions.get("task_name") failed_tests, successful_tests = process_bep(build_events) tasks = failed_tests + successful_tests @@ -45,21 +49,33 @@ def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_eve # Shrub's TaskGroup doesn't supporting adding existing tasks, so leave `tasks` empty and patch # the real list in later. task_group = TaskGroup( - name=f"resmoke_tests_results_{build_variant}", + name=f"{task_name}_results_{build_variant}", tasks=[], max_hosts=len(tasks), + setup_group_can_fail_task=True, setup_group=[ FunctionCall("git get project and add git tag"), FunctionCall("get engflow cert"), FunctionCall("get engflow key"), - FunctionCall("download build events json"), + BuiltInCommand( + "s3.get", + { + "aws_key": "${aws_key}", + "aws_secret": "${aws_secret}", + "local_file": "build_events.json", + "remote_file": "${project}/${version_id}/${build_variant}/" + + f"{task_name}/build_events.json", + "bucket": "mciuploads", + }, + ), BuiltInCommand( "s3.get", { "aws_key": "${aws_key}", "aws_secret": "${aws_secret}", "local_file": "resmoke-tests-bazel-invocation.txt", - "remote_file": "${project}/${build_variant}/${revision}/bazel-invocation-resmoke_tests-0.txt", + "remote_file": "${project}/${build_variant}/${revision}/" + + f"bazel-invocation-{task_name}-0.txt", "bucket": "mciuploads", }, ), @@ -135,6 +151,13 @@ def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_eve ) build_variant = BuildVariant(name=build_variant) + if re.match(BAZEL_BURN_IN_TESTS, task_name): + build_variant.display_task( + display_name="burn_in_tests", + execution_existing_tasks=[ExistingTask(task_name)] + + [ExistingTask(task) for task in tasks], + ) + build_variant.add_task_group(task_group) shrub_project = ShrubProject.empty() shrub_project.add_build_variant(build_variant) diff --git a/buildscripts/bazel_burn_in.py b/buildscripts/bazel_burn_in.py new file mode 100644 index 00000000000..8bf6c608c86 --- /dev/null +++ b/buildscripts/bazel_burn_in.py @@ -0,0 +1,349 @@ +"""Burn-in generator for bazel-based resmoke test suites. + +This module provides functionality to generate burn-in tests for changed test files +using Bazel targets. It identifies tests that have been modified in a git revision, +creates duplicate test targets with burn-in configurations (repeated execution), and +generates Evergreen task configurations to run these burn-in tests across build variants. + +The two commands are: +generate-targets: generates burn-in test targets in BUILD.bazel files for changed tests +generate-tasks: generates Evergreen task configurations to execute burn-in tests + +Usage: + # First, generate resmoke configs: + bazel build //... --build_tag_filters=resmoke_config + bazel cquery "kind(resmoke_config, //...)" --output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" > resmoke_suite_configs.yml + + # Generate burn-in test targets in BUILD.bazel files: + python buildscripts/bazel_burn_in.py generate-targets + + # Generate Evergreen tasks for burn-in tests: + python buildscripts/bazel_burn_in.py generate-tasks --outfile=generated_tasks.json +""" + +import json +import os +import re +import subprocess +import sys +from functools import cache +from typing import List, NamedTuple + +import typer +import yaml +from git import Repo +from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskGroup +from shrub.v2.command import BuiltInCommand +from typing_extensions import Annotated + +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from buildscripts.burn_in_tests import ( + SELECTOR_FILE, + SUPPORTED_TEST_KINDS, + LocalFileChangeDetector, +) +from buildscripts.ciconfig.evergreen import parse_evergreen_file +from buildscripts.generate_result_tasks import make_results_task +from buildscripts.util import buildozer_utils as buildozer + +BAZEL_BURN_IN_TESTS = r"resmoke_tests_burn_in_*" + + +def parse_bazel_target(target: str) -> tuple[str, str]: + """ + Parse a bazel target to get the BUILD.bazel file path and target name. + + Args: + target: Bazel target like "//buildscripts/resmokeconfig:core_config" + + Returns: + Tuple of (build_file_path, target_name without _config suffix) + """ + # Remove leading // + target = target.removeprefix("//") + + # Split on : + if ":" in target: + package_path, target_name = target.split(":", 1) + else: + package_path = target + target_name = target.split("/")[-1] + + # Remove _config suffix if present + if target_name.endswith("_config"): + target_name = target_name.removesuffix("_config") + + # Construct BUILD.bazel path + package_parts = package_path.split("/") + build_file_path = os.path.join(*package_parts, "BUILD.bazel") + + return build_file_path, target_name + + +def create_burn_in_target(target_original: str, target_burn_in: str, test: str): + """ """ + # Create the label "//jstests:foo.js" from jstests/foo.js + test_label = "//" + ":".join(test.rsplit("/", 1)) + + build_file, name_original = parse_bazel_target(target_original) + _, name_burn_in = parse_bazel_target(target_burn_in) + + # Buildozer does not provide a convenient way to clone an entire rule, so we print the original, + # replace the "name" attribute, and then write it back to the BUILD.bazel. + # To reduce the likelihood of errors, all other edits are made using buildozer. + rule_original = buildozer.bd_print([target_original], ["rule"]) + rule_new = re.sub(rf'(name\s*=\s*"){name_original}(")', rf"\1{name_burn_in}\2", rule_original) + with open(build_file, "a") as f: + f.write(rule_new) + + # Set the suite to only run the burn-in test, with only one shard. + buildozer.bd_set([target_burn_in], "srcs", test_label) + buildozer.bd_set([target_burn_in], "shard_count", "1") + + # Add burn-in arguments to the suite to repeat the test + resmoke_args_str = buildozer.bd_print([target_original], ["resmoke_args"]) + resmoke_args = resmoke_args_str.strip().removeprefix("[").removesuffix("]").split() + + # "(missing)" is buildozer's response if an attribute is not present + if "(missing)" in resmoke_args: + resmoke_args.remove("(missing)") + resmoke_args.extend( + [ + "--repeatTestsMax=1000", + "--repeatTestsMin=2", + "--repeatTestsSecs=600.0", + ] + ) + resmoke_args_str = "[" + ",".join(['"' + arg + '"' for arg in resmoke_args]) + "]" + buildozer.bd_set([target_burn_in], "resmoke_args", resmoke_args_str) + + +class BurnInTargetInfo(NamedTuple): + burn_in_target: str + original_target: str + test: str + + +def get_resmoke_configs(): + with open("resmoke_suite_configs.yml", "r") as f: + return yaml.safe_load(f) + + +def query_targets_to_burn_in(origin_rev: str) -> List[BurnInTargetInfo]: + change_detector = LocalFileChangeDetector(origin_rev) + tests_changed = change_detector.find_changed_tests([Repo(".")]) + + with open(SELECTOR_FILE, "r") as f: + exclusions = yaml.safe_load(f) + + targets = [] + for config_label, config_path in get_resmoke_configs().items(): + test_label = config_label.removeprefix("@@").removesuffix("_config") + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + test_kind = config["test_kind"] + if test_kind not in SUPPORTED_TEST_KINDS: + continue + + if test_label in exclusions["selector"].get(test_kind, {}).get("exclude_suites", []): + continue + + for test in tests_changed: + if test in exclusions["selector"].get(test_kind, {}).get("exclude_tests", []): + continue + if test not in config["selector"].get("roots"): + continue + + burn_in_target = ( + test_label + + "_burn_in_" + + test.replace("/", "_").replace("\\", "_").removeprefix("_") + ) + + targets.append( + BurnInTargetInfo( + burn_in_target=burn_in_target, original_target=test_label, test=test + ) + ) + + return targets + + +@cache +def get_targets_with_tag(tag: str) -> List[str]: + try: + query = f"attr(tags, '\\b{tag}(?![a-zA-Z0-9_-])', //...)" + result = subprocess.run( + ["bazel", "query", query], + capture_output=True, + text=True, + check=True, + ) + return [line.strip() for line in result.stdout.strip().split("\n") if line.strip()] + except subprocess.CalledProcessError as e: + print(f"Failed to query bazel targets with tag '{tag}': {e}") + print(f"stdout: {e.stdout}") + print(f"stderr: {e.stderr}") + raise + + +def make_task(targets_to_run, variant_name): + task = Task( + name=f"resmoke_tests_burn_in_{variant_name}", + commands=[ + FunctionCall( + "execute resmoke tests via bazel", + { + "targets": " ".join(targets_to_run), + "bazel_args": ( + "--test_tag_filters=${resmoke_tests_tag_filter},-incompatible_with_bazel_remote_test " + "--test_arg=--testTimeout=960 " + "--test_timeout=1500 " + "--test_sharding_strategy=disabled " + "--test_arg=--sanityCheck" + ), + "task_compile_flags": ( + "--keep_going " + "--verbose_failures " + "--simple_build_id=True " + "--define=MONGO_VERSION=${version} " + "--config=evg " + "--features=strip_debug " + "--separate_debug=False " + "--remote_download_outputs=minimal " + "--zip_undeclared_test_outputs" + ), + "generate_burn_in_targets": True, + }, + ), + ], + ) + return TaskGroup( + name=f"resmoke_tests_burn_in_{variant_name}-TG", + tasks=[task], + max_hosts=-1, + setup_task=[ + BuiltInCommand("manifest.load", {}), + FunctionCall("git get project and add git tag"), + FunctionCall("set task expansion macros"), + FunctionCall("f_expansions_write"), + FunctionCall("kill processes"), + FunctionCall("cleanup environment"), + FunctionCall("set up venv"), + FunctionCall("configure evergreen api credentials"), + FunctionCall("set up credentials"), + FunctionCall("get engflow creds"), + ], + teardown_task=[ + BuiltInCommand("generate.tasks", {"optional": True, "files": ["generated_tasks.json"]}), + BuiltInCommand( + "s3.put", + { + "optional": True, + "aws_key": "${aws_key}", + "aws_secret": "${aws_secret}", + "local_file": "src/generated_tasks.json", + "remote_file": "${project}/${version_id}/${build_variant}/${task_name}/generated_tasks.json", + "bucket": "mciuploads", + "permissions": "private", + "visibility": "signed", + "content_type": "application/json", + }, + ), + BuiltInCommand( + "s3.put", + { + "optional": True, + "aws_key": "${aws_key}", + "aws_secret": "${aws_secret}", + "local_file": "src/build_events.json", + "remote_file": "${project}/${version_id}/${build_variant}/${task_name}/build_events.json", + "bucket": "mciuploads", + "permissions": "private", + "visibility": "signed", + "content_type": "application/json", + }, + ), + FunctionCall("debug full disk"), + FunctionCall("attach bazel invocation"), + FunctionCall("save failed tests"), + FunctionCall("f_expansions_write"), + FunctionCall("kill processes"), + ], + setup_group_can_fail_task=True, + ) + + +app = typer.Typer(pretty_exceptions_show_locals=False) + + +@app.command() +def generate_targets(origin_rev: str): + """Generate burn-in test targets for changed test files.""" + os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) + + targets = query_targets_to_burn_in(origin_rev) + print(f"\nFound {len(targets)} burn-in targets to generate\n") + + for burn_in_name, original_target, test in targets: + print(f"Creating: {original_target} -> {burn_in_name}") + + create_burn_in_target(original_target, burn_in_name, test) + + +@app.command() +def generate_tasks(origin_rev: str, outfile: Annotated[str, typer.Option()]): + os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) + + targets = query_targets_to_burn_in(origin_rev) + + evg_conf = parse_evergreen_file("etc/evergreen.yml") + + shrub_project = ShrubProject.empty() + + results_tasks = [] + for variant_name in evg_conf.variant_names: + variant = evg_conf.get_variant(variant_name) + if not (variant.is_required_variant() or variant.is_suggested_variant()): + continue + task = variant.get_task("resmoke_tests") + if task: + tags = variant.expansion("resmoke_tests_tag_filter").split(",") + targets_with_tag = [] + for tag in tags: + targets_with_tag += get_targets_with_tag(tag) + + burn_in_targets_to_run = [ + target.burn_in_target + for target in targets + if target.original_target in targets_with_tag + ] + if burn_in_targets_to_run: + burn_in_task = make_task(burn_in_targets_to_run, variant_name) + + results_tasks.extend( + [make_results_task(target) for target in burn_in_targets_to_run] + ) + + build_variant = BuildVariant(name=variant_name) + build_variant.add_task_group(burn_in_task) + shrub_project.add_build_variant(build_variant) + + # Patch in fields that not supported by shrub + project = shrub_project.as_dict() + project["tasks"] = project.get("tasks", []) + for variant in project.get("buildvariants", []): + for task in variant.get("tasks", []): + task["activate"] = False + for task in project["tasks"]: + task["exec_timeout_secs"] = 1800 + project["tasks"].extend([task.as_dict() for task in results_tasks]) + + with open(outfile, "w") as f: + f.write(json.dumps(project, indent=4)) + + +if __name__ == "__main__": + app() diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 633e12d07e4..30b9ff60296 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -363,6 +363,10 @@ class Variant(object): """Return True if the variant is a required variant.""" return self.display_name.startswith("!") + def is_suggested_variant(self) -> bool: + """Return True if the variant is a suggested variant.""" + return self.display_name.startswith("*") + def get_task(self, task_name): """Return the task with the given name as an instance of VariantTask. diff --git a/buildscripts/evergreen_activate_gen_tasks.py b/buildscripts/evergreen_activate_gen_tasks.py index 42a8b93c33f..3cdceab62f6 100755 --- a/buildscripts/evergreen_activate_gen_tasks.py +++ b/buildscripts/evergreen_activate_gen_tasks.py @@ -2,6 +2,7 @@ """Activate an evergreen task in the existing build.""" import os +import re import sys import click @@ -32,6 +33,7 @@ LOGGER = structlog.getLogger(__name__) EVG_CONFIG_FILE = "./.evergreen.yml" BURN_IN_TAGS = "burn_in_tags" BURN_IN_TESTS = "burn_in_tests" +BAZEL_BURN_IN_TESTS = r"resmoke_tests_burn_in*" BURN_IN_VARIANT_SUFFIX = "generated-by-burn-in-tags" @@ -69,30 +71,41 @@ def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None: tasks_not_activated = [] if expansions.task == BURN_IN_TAGS: version = evg_api.version_by_id(expansions.version_id) - burn_in_build_variants = [ - variant - for variant in version.build_variants_map.keys() - if variant.endswith(BURN_IN_VARIANT_SUFFIX) - ] - for build_variant in burn_in_build_variants: + for build_variant in version.build_variants_map.keys(): build_id = version.build_variants_map[build_variant] task_list = evg_api.tasks_by_build(build_id) - - for task in task_list: - if task.display_name == BURN_IN_TESTS: - LOGGER.info( - "Activating task", task_id=task.task_id, task_name=task.display_name - ) - try: - evg_api.configure_task(task.task_id, activated=True) - except Exception: - LOGGER.error( - "Could not activate task", - task_id=task.task_id, - task_name=task.display_name, - exc_info=True, + if build_variant.endswith(BURN_IN_VARIANT_SUFFIX): + for task in task_list: + if task.display_name == BURN_IN_TESTS: + LOGGER.info( + "Activating task", task_id=task.task_id, task_name=task.display_name ) - tasks_not_activated.append(task.task_id) + try: + evg_api.configure_task(task.task_id, activated=True) + except Exception: + LOGGER.error( + "Could not activate task", + task_id=task.task_id, + task_name=task.display_name, + exc_info=True, + ) + tasks_not_activated.append(task.task_id) + else: + for task in task_list: + if re.match(BAZEL_BURN_IN_TESTS, task.display_name): + LOGGER.info( + "Activating task", task_id=task.task_id, task_name=task.display_name + ) + try: + evg_api.configure_task(task.task_id, activated=True) + except Exception: + LOGGER.error( + "Could not activate task", + task_id=task.task_id, + task_name=task.display_name, + exc_info=True, + ) + tasks_not_activated.append(task.task_id) else: task_list = retry_call( diff --git a/buildscripts/generate_result_tasks.py b/buildscripts/generate_result_tasks.py index dd47f58e059..dcfbf5c3c9d 100644 --- a/buildscripts/generate_result_tasks.py +++ b/buildscripts/generate_result_tasks.py @@ -28,8 +28,7 @@ RESMOKE_TEST_QUERY = 'attr(tags, "resmoke_suite_test", //...)' app = typer.Typer(pretty_exceptions_show_locals=False) -def make_task(target: str) -> Task: - print(f"Generating task for {target}") +def make_results_task(target: str) -> Task: commands = [ FunctionCall("fetch remote test results", {"test_label": target}), ] @@ -61,7 +60,7 @@ def main(outfile: Annotated[str, typer.Option()]): test_targets = query_targets() - tasks = [make_task(target) for target in test_targets] + tasks = [make_results_task(target) for target in test_targets] project = {"tasks": [task.as_dict() for task in tasks]} with open(outfile, "w") as f: diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml index 4cd09f6b37d..09967fb44ff 100644 --- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml +++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml @@ -6,7 +6,7 @@ selector: - buildscripts/idl/tests/**/test_*.py - buildscripts/bazel_rules_mongo/tests/test_*.py exclude_files: - - buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py # Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783. + - buildscripts/tests/burn_in/test_burn_in_end2end.py # Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783. # These tests are also @unittest.skip'ed. SERVER-48969 tracks re-enabling them. - buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path. - buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3. diff --git a/buildscripts/tests/burn_in/BUILD.bazel b/buildscripts/tests/burn_in/BUILD.bazel new file mode 100644 index 00000000000..1f6f82dbe7f --- /dev/null +++ b/buildscripts/tests/burn_in/BUILD.bazel @@ -0,0 +1,47 @@ +py_library( + name = "all_python_files", + srcs = glob(["*.py"]), + visibility = ["//visibility:public"], +) + +py_test( + name = "test_burn_in", + srcs = [ + "test_burn_in.py", + ], + data = [ + "//buildscripts/resmokeconfig:all_files", + "//buildscripts/resmokeconfig/loggers:all_files", + "//etc:burn_in_tests.yml", + ], + deps = [ + "//buildscripts:burn_in_tests", + "//buildscripts/resmokelib", + ], +) + +py_test( + name = "test_burn_in_end2end", + srcs = [ + "test_burn_in_end2end.py", + ], + data = [ + "//buildscripts/resmokeconfig:all_files", + "//buildscripts/resmokeconfig/loggers:all_files", + "//etc:burn_in_tests.yml", + ], + deps = [ + "//buildscripts:burn_in_tests", + "//buildscripts/resmokelib", + ], +) + +py_test( + name = "test_bazel_burn_in", + srcs = [ + "test_bazel_burn_in.py", + ], + deps = [ + "//buildscripts:bazel_burn_in", + ], +) diff --git a/buildscripts/tests/burn_in_tests_end2end/OWNERS.yml b/buildscripts/tests/burn_in/OWNERS.yml similarity index 100% rename from buildscripts/tests/burn_in_tests_end2end/OWNERS.yml rename to buildscripts/tests/burn_in/OWNERS.yml diff --git a/buildscripts/tests/burn_in/__init__.py b/buildscripts/tests/burn_in/__init__.py new file mode 100644 index 00000000000..4b7a2bb941b --- /dev/null +++ b/buildscripts/tests/burn_in/__init__.py @@ -0,0 +1 @@ +"""Empty.""" diff --git a/buildscripts/tests/burn_in/test_bazel_burn_in.py b/buildscripts/tests/burn_in/test_bazel_burn_in.py new file mode 100644 index 00000000000..2aec36e0c18 --- /dev/null +++ b/buildscripts/tests/burn_in/test_bazel_burn_in.py @@ -0,0 +1,228 @@ +"""Unit tests for buildscripts/bazel_burn_in.py.""" + +import os +import unittest +from unittest.mock import mock_open, patch + +import buildscripts.bazel_burn_in as under_test + +NS = "buildscripts.bazel_burn_in" + + +def ns(relative_name): + """Return a full name from a name relative to the test module's namespace.""" + return NS + "." + relative_name + + +# Mock data fixtures +MOCK_RESMOKE_CONFIG = {"test_kind": "js_test", "selector": {"roots": ["jstests/**/*.js"]}} + +MOCK_EXCLUSIONS = {"selector": {"js_test": {"exclude_suites": [], "exclude_tests": []}}} + +MOCK_BUILDOZER_RULE = """resmoke_suite_test( + name = "core_config", + resmoke_args = ["--log=info"], + srcs = ["//jstests/core:all"], + shard_count = 4, +)""" + + +class TestParseBazelTarget(unittest.TestCase): + """Tests for parse_bazel_target function.""" + + def test_parse_basic_target_with_colon(self): + """Test parsing a basic target with colon and _config suffix.""" + result = under_test.parse_bazel_target("//buildscripts/resmokeconfig:core_config") + self.assertEqual( + (os.path.join("buildscripts", "resmokeconfig", "BUILD.bazel"), "core"), result + ) + + def test_parse_target_without_colon(self): + """Test parsing a target without colon.""" + result = under_test.parse_bazel_target("//jstests/core") + self.assertEqual((os.path.join("jstests", "core", "BUILD.bazel"), "core"), result) + + def test_parse_target_without_config_suffix(self): + """Test parsing a target without _config suffix.""" + result = under_test.parse_bazel_target("//buildscripts:test_target") + self.assertEqual((os.path.join("buildscripts", "BUILD.bazel"), "test_target"), result) + + def test_parse_target_with_double_slash_prefix(self): + """Test parsing a target with // prefix.""" + result = under_test.parse_bazel_target("//path/to/package:target_name_config") + self.assertEqual( + (os.path.join("path", "to", "package", "BUILD.bazel"), "target_name"), result + ) + + def test_parse_target_without_double_slash(self): + """Test parsing a target without // prefix.""" + result = under_test.parse_bazel_target("buildscripts:target") + self.assertEqual((os.path.join("buildscripts", "BUILD.bazel"), "target"), result) + + def test_parse_target_with_nested_path(self): + """Test parsing a target with deeply nested path.""" + result = under_test.parse_bazel_target("//a/b/c/d:target_config") + self.assertEqual((os.path.join("a", "b", "c", "d", "BUILD.bazel"), "target"), result) + + def test_parse_target_edge_case_single_directory(self): + """Test parsing a target with single directory.""" + result = under_test.parse_bazel_target("//jstests:test_config") + self.assertEqual((os.path.join("jstests", "BUILD.bazel"), "test"), result) + + def test_parse_target_with_underscores_in_name(self): + """Test parsing a target with underscores in name.""" + result = under_test.parse_bazel_target("//path:my_test_target_config") + self.assertEqual((os.path.join("path", "BUILD.bazel"), "my_test_target"), result) + + def test_parse_target_config_in_middle_of_name(self): + """Test that config in middle of name is NOT removed.""" + result = under_test.parse_bazel_target("//path:config_test_target") + self.assertEqual((os.path.join("path", "BUILD.bazel"), "config_test_target"), result) + + +class TestCreateBurnInTarget(unittest.TestCase): + """Unit tests for create_burn_in_target function.""" + + @patch(ns("buildozer.bd_set")) + @patch(ns("buildozer.bd_print")) + @patch("builtins.open", new_callable=mock_open) + @patch(ns("parse_bazel_target")) + def test_create_burn_in_target_basic( + self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set + ): + """Test basic burn-in target creation.""" + # Setup + target_original = "//jstests:core_config" + target_burn_in = "//jstests:core_burn_in_find_js" + test = "jstests/core/find.js" + + mock_parse.side_effect = [ + ("jstests/BUILD.bazel", "core"), + ("jstests/BUILD.bazel", "core_burn_in_find_js"), + ] + + mock_rule = 'resmoke_suite_test(\n name = "core",\n resmoke_args = [],\n)' + mock_bd_print.side_effect = [mock_rule, "[]"] + + # Execute + under_test.create_burn_in_target(target_original, target_burn_in, test) + + # Assert + mock_parse.assert_any_call(target_original) + mock_parse.assert_any_call(target_burn_in) + + mock_open_file.assert_called_once_with("jstests/BUILD.bazel", "a") + + mock_bd_set.assert_any_call([target_burn_in], "srcs", "//jstests/core:find.js") + mock_bd_set.assert_any_call([target_burn_in], "shard_count", "1") + + # Verify resmoke_args includes repeat parameters + resmoke_args_calls = [ + call for call in mock_bd_set.call_args_list if call[0][1] == "resmoke_args" + ] + self.assertEqual(len(resmoke_args_calls), 1) + resmoke_args_value = resmoke_args_calls[0][0][2] + + self.assertIn("--repeatTestsMax=1000", resmoke_args_value) + self.assertIn("--repeatTestsMin=2", resmoke_args_value) + self.assertIn("--repeatTestsSecs=600.0", resmoke_args_value) + + @patch(ns("buildozer.bd_set")) + @patch(ns("buildozer.bd_print")) + @patch("builtins.open", new_callable=mock_open) + @patch(ns("parse_bazel_target")) + def test_create_burn_in_target_with_existing_resmoke_args( + self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set + ): + """Test burn-in target creation preserves existing resmoke args.""" + # Setup + target_original = "//jstests:core_config" + target_burn_in = "//jstests:core_burn_in_find_js" + test = "jstests/core/find.js" + + mock_parse.side_effect = [ + ("jstests/BUILD.bazel", "core"), + ("jstests/BUILD.bazel", "core_burn_in_find_js"), + ] + + mock_bd_print.side_effect = [ + 'resmoke_suite_test(name = "core")', + '["--log=debug" "--storageEngine=wiredTiger"]', + ] + + # Execute + under_test.create_burn_in_target(target_original, target_burn_in, test) + + # Assert - verify existing args preserved and new args added + resmoke_args_calls = [ + call for call in mock_bd_set.call_args_list if call[0][1] == "resmoke_args" + ] + resmoke_args_value = resmoke_args_calls[0][0][2] + + self.assertIn("--log=debug", resmoke_args_value) + self.assertIn("--storageEngine=wiredTiger", resmoke_args_value) + self.assertIn("--repeatTestsMax=1000", resmoke_args_value) + + @patch(ns("buildozer.bd_set")) + @patch(ns("buildozer.bd_print")) + @patch("builtins.open", new_callable=mock_open) + @patch(ns("parse_bazel_target")) + def test_create_burn_in_target_with_missing_resmoke_args( + self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set + ): + """Test burn-in target handles missing resmoke_args.""" + # Setup + target_original = "//jstests:core_config" + target_burn_in = "//jstests:core_burn_in_find_js" + test = "jstests/core/find.js" + + mock_parse.side_effect = [ + ("jstests/BUILD.bazel", "core"), + ("jstests/BUILD.bazel", "core_burn_in_find_js"), + ] + + mock_bd_print.side_effect = ['resmoke_suite_test(name = "core")', "(missing)"] + + # Execute + under_test.create_burn_in_target(target_original, target_burn_in, test) + + # Assert - verify (missing) is not in the args + resmoke_args_calls = [ + call for call in mock_bd_set.call_args_list if call[0][1] == "resmoke_args" + ] + resmoke_args_value = resmoke_args_calls[0][0][2] + + self.assertNotIn("(missing)", resmoke_args_value) + self.assertIn("--repeatTestsMax=1000", resmoke_args_value) + + @patch(ns("buildozer.bd_set")) + @patch(ns("buildozer.bd_print")) + @patch("builtins.open", new_callable=mock_open) + @patch(ns("parse_bazel_target")) + def test_create_burn_in_target_sets_correct_attributes( + self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set + ): + """Test that bd_set is called with correct attributes.""" + # Setup + target_original = "//jstests:core_config" + target_burn_in = "//jstests:core_burn_in_find_js" + test = "jstests/core/find.js" + + mock_parse.side_effect = [ + ("jstests/BUILD.bazel", "core"), + ("jstests/BUILD.bazel", "core_burn_in_find_js"), + ] + + mock_bd_print.side_effect = ['resmoke_suite_test(name = "core")', "[]"] + + # Execute + under_test.create_burn_in_target(target_original, target_burn_in, test) + + # Assert - bd_set called 3 times for srcs, shard_count, and resmoke_args + self.assertEqual(mock_bd_set.call_count, 3) + mock_bd_set.assert_any_call([target_burn_in], "srcs", "//jstests/core:find.js") + mock_bd_set.assert_any_call([target_burn_in], "shard_count", "1") + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/burn_in/test_burn_in.py similarity index 99% rename from buildscripts/tests/test_burn_in_tests.py rename to buildscripts/tests/burn_in/test_burn_in.py index e8ac586bbe9..52a56a8a928 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/burn_in/test_burn_in.py @@ -9,9 +9,9 @@ import subprocess import sys import unittest from io import StringIO +from unittest.mock import MagicMock, Mock, patch import yaml -from mock import MagicMock, Mock, patch import buildscripts.burn_in_tests as under_test import buildscripts.resmokelib.parser as _parser @@ -514,3 +514,7 @@ class TestYamlBurnInExecutor(unittest.TestCase): results = yaml.safe_load(yaml_raw) self.assertEqual(n_tasks, len(results["discovered_tasks"])) self.assertEqual(n_tests, len(results["discovered_tasks"][0]["suites"][0]["test_list"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/burn_in/test_burn_in_end2end.py b/buildscripts/tests/burn_in/test_burn_in_end2end.py new file mode 100644 index 00000000000..4578338d8c1 --- /dev/null +++ b/buildscripts/tests/burn_in/test_burn_in_end2end.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys +import unittest + +import yaml + +import buildscripts.burn_in_tests as under_test + + +class TestBurnInTestsEnd2End(unittest.TestCase): + @unittest.skip( + "Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783." + ) + @classmethod + def setUpClass(cls): + subprocess.run( + [ + sys.executable, + "buildscripts/burn_in_tests.py", + "generate-test-membership-map-file-for-ci", + ] + ) + + @classmethod + def tearDownClass(cls): + if os.path.exists(under_test.BURN_IN_TEST_MEMBERSHIP_FILE): + os.remove(under_test.BURN_IN_TEST_MEMBERSHIP_FILE) + + def test_valid_yaml_output(self): + process = subprocess.run( + [ + sys.executable, + "buildscripts/burn_in_tests.py", + "run", + "--yaml", + ], + text=True, + capture_output=True, + ) + self.assertEqual( + 0, + process.returncode, + process.stderr, + ) + + output = process.stdout + try: + yaml.safe_load(output) + except Exception: + self.fail(msg="burn_in_tests.py does not output valid yaml.") + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/burn_in_tests_end2end/BUILD.bazel b/buildscripts/tests/burn_in_tests_end2end/BUILD.bazel deleted file mode 100644 index fa7edff1bc5..00000000000 --- a/buildscripts/tests/burn_in_tests_end2end/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -# TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets -py_library( - name = "all_python_files", - srcs = glob(["*.py"]), - visibility = ["//visibility:public"], -) diff --git a/buildscripts/util/buildozer_utils.py b/buildscripts/util/buildozer_utils.py index 0c11c4cc137..3779b692f12 100644 --- a/buildscripts/util/buildozer_utils.py +++ b/buildscripts/util/buildozer_utils.py @@ -39,8 +39,9 @@ def bd_comment(labels: List[str], comment: str, attr: str = "", value: str = "") _bd_command(f"comment {attr} {value} {comment}", labels) -def bd_print(labels: List[str], attrs: List[str]) -> None: - _bd_command(f'print {" ".join(attrs)}', labels) +def bd_print(labels: List[str], attrs: List[str]) -> str: + p = _bd_command(f'print {" ".join(attrs)}', labels) + return p.stdout def bd_new_load(packages: List[str], path: str, rules: List[str]) -> None: diff --git a/etc/BUILD.bazel b/etc/BUILD.bazel index e11f6db72c1..ad27f7c1293 100644 --- a/etc/BUILD.bazel +++ b/etc/BUILD.bazel @@ -6,6 +6,7 @@ exports_files([ "lsan.suppressions", "tsan.suppressions", "extensions.yml", + "burn_in_tests.yml", ]) # This is a hack to work around the fact that the cc_library flag additional_compiler_inputs doesn't diff --git a/etc/burn_in_tests.yml b/etc/burn_in_tests.yml index 89ee333cf85..c0d0845d9b3 100644 --- a/etc/burn_in_tests.yml +++ b/etc/burn_in_tests.yml @@ -1,7 +1,7 @@ # This file is used to exclude suites, tasks or tests from running in the burn_in_test task. selector: js_test: - # Exclude list of resmoke.py suite names. + # Exclude list of resmoke.py suite names or resmoke_suite_test target names. exclude_suites: # Requires an HTTP server to be running in the background. - queryable_wt diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 6c2cf49e521..6086bf0fc5f 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1265,6 +1265,7 @@ functions: build_variant: ${build_variant} distro_id: ${distro_id} execution: ${execution} + generate_burn_in_targets: ${generate_burn_in_targets} otel_parent_id: ${otel_parent_id} otel_trace_id: ${otel_trace_id} project: ${project} @@ -1283,16 +1284,6 @@ functions: - *f_expansions_write - *execute_resmoke_tests_via_bazel_sh - "download build events json": - - command: s3.get - display_name: "download build events json" - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - bucket: mciuploads - remote_file: ${project}/${version_id}/${build_variant}/resmoke_tests/build_events.json - local_file: "build_events.json" - "fetch remote test results": - command: subprocess.exec params: @@ -1410,6 +1401,7 @@ functions: optional: true files: - src/generated_resmoke_config/*.json + - src/generated_bazel_tasks.json "generate version": - *f_expansions_write @@ -1428,6 +1420,8 @@ functions: - *validate_generate_tasks_config "generate version burn in": + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - *configure_evergreen_api_credentials - command: subprocess.exec @@ -1441,6 +1435,17 @@ functions: - *validate_generate_tasks_config - *upload_burn_in_generate_tasks_config - *generate_resmoke_tasks_config + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/generated_bazel_tasks.json + remote_file: ${project}/${version_id}/${build_variant}/{task_name}/generated_bazel_tasks.json + bucket: mciuploads + permissions: private + visibility: signed + content_type: application/json "initialize multiversion tasks": - *f_expansions_write diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index b10598fcad6..f8fd5fda88f 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -2136,6 +2136,9 @@ tasks: - name: version_burn_in_gen run_on: ubuntu2404-medium tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - command: manifest.load - func: "git get shallow project" @@ -2146,6 +2149,14 @@ tasks: - func: "cleanup environment" - func: "set up venv" - func: "upload pip requirements" + - func: "get engflow creds" + - func: "build all resmoke configs" + vars: + targets: //... + bazel_args: >- + --build_tag_filters=resmoke_config + --noincompatible_enable_cc_toolchain_resolution + --repo_env=no_c++_toolchain=1 - func: "generate version burn in" - name: version_gen diff --git a/evergreen/bazel_evergreen_shutils.sh b/evergreen/bazel_evergreen_shutils.sh index cab4340de42..7dbad3b5895 100644 --- a/evergreen/bazel_evergreen_shutils.sh +++ b/evergreen/bazel_evergreen_shutils.sh @@ -351,3 +351,19 @@ bazel_evergreen_shutils::maybe_scale_test_timeout_and_append() { bazel_args="${bazel_args:-} --test_timeout=${scaled}" fi } + +# Queries all resmoke_config targets and outputs YAML key-value pairs. +# Usage: bazel_evergreen_shutils::query_resmoke_configs +# example: bazel_evergreen_shutils::query_resmoke_configs "$BAZEL_BINARY" "${CONFIG_FLAGS}" "resmoke_suite_configs.yml" +# Outputs YAML entries like: +# //buildscripts/resmokeconfig:core_config: bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml +bazel_evergreen_shutils::query_resmoke_configs() { + local BAZEL_BINARY="$1" + local FLAGS="$2" + local OUTPUT_FILE="$3" + + ${BAZEL_BINARY} cquery ${FLAGS} 'kind(resmoke_config, //...)' \ + --output=starlark \ + --starlark:expr='": ".join([str(target.label).replace("@@","")] + [f.path for f in target.files.to_list()])' \ + >"${OUTPUT_FILE}" +} diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh index fb717c96e64..4c1aad7450d 100644 --- a/evergreen/generate_version_burn_in.sh +++ b/evergreen/generate_version_burn_in.sh @@ -24,3 +24,5 @@ RUST_BACKTRACE=full PATH=$PATH:$HOME:/ ./mongo-task-generator \ --burn-in \ --burn-in-tests-command "python buildscripts/burn_in_tests.py run --origin-rev=$base_revision" \ $@ + +$python buildscripts/bazel_burn_in.py generate-tasks "$base_revision" --outfile=generated_bazel_tasks.json diff --git a/evergreen/get_all_resmoke_suite_configs.sh b/evergreen/get_all_resmoke_suite_configs.sh index e0877a63676..baf3108d402 100644 --- a/evergreen/get_all_resmoke_suite_configs.sh +++ b/evergreen/get_all_resmoke_suite_configs.sh @@ -15,10 +15,8 @@ set -o verbose source ./evergreen/bazel_evergreen_shutils.sh BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path) -# Queries all resmoke_config targets: kind(resmoke_config, //...) -# and outputs YAML key-value pair created by the starlark expression for each target. -# str(target.label).replace('@@','') -> the target name, like //buildscripts/resmokeconfig:core_config -# f.path for f in target.files.to_list() -> the path to the config file, like bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml -${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ - --define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \ - --output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" >resmoke_suite_configs.yml +# Queries all resmoke_config targets and outputs YAML key-value pairs mapping targets to their config files. +bazel_evergreen_shutils::query_resmoke_configs \ + "${BAZEL_BINARY}" \ + "${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}" \ + "resmoke_suite_configs.yml" diff --git a/evergreen/resmoke_tests_execute_bazel.sh b/evergreen/resmoke_tests_execute_bazel.sh index e3d36255c69..bcd73c09f39 100644 --- a/evergreen/resmoke_tests_execute_bazel.sh +++ b/evergreen/resmoke_tests_execute_bazel.sh @@ -53,12 +53,21 @@ for strategy in "${strategies[@]}"; do done ALL_FLAGS="${ci_flags} ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} ${patch_compile_flags:-}" +CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")" echo "${ALL_FLAGS}" >.bazel_build_flags # Save the invocation, intentionally excluding CI specific flags. echo "python buildscripts/install_bazel.py" >bazel-invocation.txt echo "bazel test ${bazel_args} ${targets}" >>bazel-invocation.txt +if [ "${generate_burn_in_targets}" = "true" ]; then + echo "Generating burn-in test targets..." + base_revision="$(git merge-base ${revision} HEAD)" + ${BAZEL_BINARY} build ${CONFIG_FLAGS} //... --build_tag_filters=resmoke_config + bazel_evergreen_shutils::query_resmoke_configs "${BAZEL_BINARY}" "${CONFIG_FLAGS}" "resmoke_suite_configs.yml" + ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:bazel_burn_in -- generate-targets "$base_revision" || echo "Failed to generate burn-in targets" +fi + set +o errexit # Fetch then test with retries. @@ -92,12 +101,13 @@ if [[ "$RET" != "0" ]]; then # The --config flag needs to stay consistent for the `bazel run` to avoid evicting the previous results. # Strip out anything that isn't a --config flag that could interfere with the run command. - CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")" eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_tests || true fi eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:append_result_tasks -- --outfile=generated_tasks.json +eval ${BAZEL_BINARY} shutdown # Explicitly shutdown the bazel server in case the Evergreen agent is tracking it for completion of this process. + # Return code 3 from `bazel test` indicates that the build was OK, but some tests failed or timed out. # The test failures are reported in individual results tasks, so don't fail the task here. if [[ "$RET" -eq 3 ]]; then diff --git a/evergreen/spawnhost/download_test_binaries.py b/evergreen/spawnhost/download_test_binaries.py index 9c07058221b..971348ed197 100644 --- a/evergreen/spawnhost/download_test_binaries.py +++ b/evergreen/spawnhost/download_test_binaries.py @@ -16,8 +16,15 @@ def main(): task = evg_api.task_by_id(task_id) tasks_in_variant = evg_api.tasks_by_build(task.build_id) - resmoke_tests_task = list(filter(lambda t: t.display_name == "resmoke_tests", tasks_in_variant)) - assert len(resmoke_tests_task) == 1, "Could not find a unique resmoke_tests task" + if "_burn_in_" in task.display_name: + resmoke_tests_task = list( + filter(lambda t: t.display_name.startswith("resmoke_tests_burn_in"), tasks_in_variant) + ) + else: + resmoke_tests_task = list( + filter(lambda t: t.display_name == "resmoke_tests", tasks_in_variant) + ) + assert len(resmoke_tests_task) == 1, "Could not find a unique resmoke test task" resmoke_tests_task = resmoke_tests_task[0] output_dir = "/data/mci/artifacts-resmoke_tests"