mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-106100 Generate burn-in from bazel-based resmoke suites (#47357)

Browse Source 
GitOrigin-RevId: 88f6e186a31bc637126ea0904b0136ef0c0882bb
This commit is contained in:
Sean Lyons 2026-02-03 15:02:08 -05:00 committed by MongoDB Bot
parent 750f719c19
commit df8c894b19
26 changed files with 886 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/CODEOWNERS vendored
View File

@ -78,6 +78,11 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
/buildscripts/golden_test.py @10gen/query-optimization @svc-auto-approve-bot
/buildscripts/evergreen_gen_streams* @10gen/streams-engine @svc-auto-approve-bot
/buildscripts/compare_evergreen_versions.py @10gen/devprod-correctness @svc-auto-approve-bot
/buildscripts/ciconfig @10gen/devprod-correctness @svc-auto-approve-bot
/buildscripts/append_result_tasks.py @10gen/devprod-correctness @svc-auto-approve-bot
/buildscripts/generate_result_tasks.py @10gen/devprod-correctness @svc-auto-approve-bot
/buildscripts/evergreen_activate_gen_tasks.py @10gen/devprod-correctness @svc-auto-approve-bot
/buildscripts/bazel_burn_in.py @10gen/devprod-correctness @svc-auto-approve-bot
# The following patterns are parsed from ./buildscripts/antithesis/OWNERS.yml
/buildscripts/antithesis/ @10gen/devprod-correctness @svc-auto-approve-bot
@ -331,8 +336,8 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
/buildscripts/tests/test_evergreen_task_timeout.py @10gen/devprod-correctness @svc-auto-approve-bot
/buildscripts/tests/test_generate_sbom.py @10gen/code-review-team-ssdlc @svc-auto-approve-bot
# The following patterns are parsed from ./buildscripts/tests/burn_in_tests_end2end/OWNERS.yml
/buildscripts/tests/burn_in_tests_end2end/ @10gen/devprod-correctness @svc-auto-approve-bot
# The following patterns are parsed from ./buildscripts/tests/burn_in/OWNERS.yml
/buildscripts/tests/burn_in/ @10gen/devprod-correctness @svc-auto-approve-bot
# The following patterns are parsed from ./buildscripts/tests/monitor_build_status/OWNERS.yml
/buildscripts/tests/monitor_build_status/ @10gen/devprod-correctness @svc-auto-approve-bot

39
buildscripts/BUILD.bazel
View File

@ -263,12 +263,48 @@ py_binary(
],
)
py_binary(
name = "burn_in_tests",
srcs = ["burn_in_tests.py"],
visibility = ["//visibility:public"],
deps = [
"//buildscripts/patch_builds",
"//buildscripts/resmokelib",
dependency(
"structlog",
group = "evergreen",
),
dependency(
"gitpython",
group = "evergreen",
),
],
)
py_binary(
name = "bazel_burn_in",
srcs = ["bazel_burn_in.py"],
visibility = ["//visibility:public"],
deps = [
"//buildscripts:burn_in_tests",
"//buildscripts:generate_result_tasks",
"//buildscripts/util",
dependency(
"typer",
group = "core",
),
dependency(
"shrub-py",
group = "testing",
),
],
)
py_binary(
name = "generate_result_tasks",
srcs = ["generate_result_tasks.py"],
visibility = ["//visibility:public"],
deps = [
"//buildscripts:gather_failed_tests",
dependency(
"typer",
group = "core",
@ -285,6 +321,7 @@ py_binary(
srcs = ["append_result_tasks.py"],
visibility = ["//visibility:public"],
deps = [
"//buildscripts:bazel_burn_in",
"//buildscripts:gather_failed_tests",
dependency(
"typer",

15
buildscripts/OWNERS.yml
View File

@ -47,3 +47,18 @@ filters:
- "compare_evergreen_versions.py":
approvers:
- 10gen/devprod-correctness
- "ciconfig":
approvers:
- 10gen/devprod-correctness
- "append_result_tasks.py":
approvers:
- 10gen/devprod-correctness
- "generate_result_tasks.py":
approvers:
- 10gen/devprod-correctness
- "evergreen_activate_gen_tasks.py":
approvers:
- 10gen/devprod-correctness
- "bazel_burn_in.py":
approvers:
- 10gen/devprod-correctness

29
buildscripts/append_result_tasks.py
View File

@ -20,12 +20,15 @@ Options:
import json
import os
import re
import typer
from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, TaskGroup
from shrub.v2.command import BuiltInCommand
from shrub.v2.task import ExistingTask
from typing_extensions import Annotated
from buildscripts.bazel_burn_in import BAZEL_BURN_IN_TESTS
from buildscripts.gather_failed_tests import process_bep
from buildscripts.util.read_config import read_config_file
@ -38,6 +41,7 @@ def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_eve
expansions = read_config_file("../expansions.yml")
build_variant = expansions.get("build_variant")
task_name = expansions.get("task_name")
failed_tests, successful_tests = process_bep(build_events)
tasks = failed_tests + successful_tests
@ -45,21 +49,33 @@ def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_eve
# Shrub's TaskGroup doesn't supporting adding existing tasks, so leave `tasks` empty and patch
# the real list in later.
task_group = TaskGroup(
name=f"resmoke_tests_results_{build_variant}",
name=f"{task_name}_results_{build_variant}",
tasks=[],
max_hosts=len(tasks),
setup_group_can_fail_task=True,
setup_group=[
FunctionCall("git get project and add git tag"),
FunctionCall("get engflow cert"),
FunctionCall("get engflow key"),
FunctionCall("download build events json"),
BuiltInCommand(
"s3.get",
{
"aws_key": "${aws_key}",
"aws_secret": "${aws_secret}",
"local_file": "build_events.json",
"remote_file": "${project}/${version_id}/${build_variant}/"
+ f"{task_name}/build_events.json",
"bucket": "mciuploads",
},
),
BuiltInCommand(
"s3.get",
{
"aws_key": "${aws_key}",
"aws_secret": "${aws_secret}",
"local_file": "resmoke-tests-bazel-invocation.txt",
"remote_file": "${project}/${build_variant}/${revision}/bazel-invocation-resmoke_tests-0.txt",
"remote_file": "${project}/${build_variant}/${revision}/"
+ f"bazel-invocation-{task_name}-0.txt",
"bucket": "mciuploads",
},
),
@ -135,6 +151,13 @@ def main(outfile: Annotated[str, typer.Option()], build_events: str = "build_eve
)
build_variant = BuildVariant(name=build_variant)
if re.match(BAZEL_BURN_IN_TESTS, task_name):
build_variant.display_task(
display_name="burn_in_tests",
execution_existing_tasks=[ExistingTask(task_name)]
+ [ExistingTask(task) for task in tasks],
)
build_variant.add_task_group(task_group)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)

349
buildscripts/bazel_burn_in.py Normal file
View File

@ -0,0 +1,349 @@
"""Burn-in generator for bazel-based resmoke test suites.
This module provides functionality to generate burn-in tests for changed test files
using Bazel targets. It identifies tests that have been modified in a git revision,
creates duplicate test targets with burn-in configurations (repeated execution), and
generates Evergreen task configurations to run these burn-in tests across build variants.
The two commands are:
generate-targets: generates burn-in test targets in BUILD.bazel files for changed tests
generate-tasks: generates Evergreen task configurations to execute burn-in tests
Usage:
# First, generate resmoke configs:
bazel build //... --build_tag_filters=resmoke_config
bazel cquery "kind(resmoke_config, //...)" --output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" > resmoke_suite_configs.yml
# Generate burn-in test targets in BUILD.bazel files:
python buildscripts/bazel_burn_in.py generate-targets <origin_rev>
# Generate Evergreen tasks for burn-in tests:
python buildscripts/bazel_burn_in.py generate-tasks <origin_rev> --outfile=generated_tasks.json
"""
import json
import os
import re
import subprocess
import sys
from functools import cache
from typing import List, NamedTuple
import typer
import yaml
from git import Repo
from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskGroup
from shrub.v2.command import BuiltInCommand
from typing_extensions import Annotated
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from buildscripts.burn_in_tests import (
SELECTOR_FILE,
SUPPORTED_TEST_KINDS,
LocalFileChangeDetector,
)
from buildscripts.ciconfig.evergreen import parse_evergreen_file
from buildscripts.generate_result_tasks import make_results_task
from buildscripts.util import buildozer_utils as buildozer
BAZEL_BURN_IN_TESTS = r"resmoke_tests_burn_in_*"
def parse_bazel_target(target: str) -> tuple[str, str]:
"""
Parse a bazel target to get the BUILD.bazel file path and target name.
Args:
target: Bazel target like "//buildscripts/resmokeconfig:core_config"
Returns:
Tuple of (build_file_path, target_name without _config suffix)
"""
# Remove leading //
target = target.removeprefix("//")
# Split on :
if ":" in target:
package_path, target_name = target.split(":", 1)
else:
package_path = target
target_name = target.split("/")[-1]
# Remove _config suffix if present
if target_name.endswith("_config"):
target_name = target_name.removesuffix("_config")
# Construct BUILD.bazel path
package_parts = package_path.split("/")
build_file_path = os.path.join(*package_parts, "BUILD.bazel")
return build_file_path, target_name
def create_burn_in_target(target_original: str, target_burn_in: str, test: str):
""" """
# Create the label "//jstests:foo.js" from jstests/foo.js
test_label = "//" + ":".join(test.rsplit("/", 1))
build_file, name_original = parse_bazel_target(target_original)
_, name_burn_in = parse_bazel_target(target_burn_in)
# Buildozer does not provide a convenient way to clone an entire rule, so we print the original,
# replace the "name" attribute, and then write it back to the BUILD.bazel.
# To reduce the likelihood of errors, all other edits are made using buildozer.
rule_original = buildozer.bd_print([target_original], ["rule"])
rule_new = re.sub(rf'(name\s*=\s*"){name_original}(")', rf"\1{name_burn_in}\2", rule_original)
with open(build_file, "a") as f:
f.write(rule_new)
# Set the suite to only run the burn-in test, with only one shard.
buildozer.bd_set([target_burn_in], "srcs", test_label)
buildozer.bd_set([target_burn_in], "shard_count", "1")
# Add burn-in arguments to the suite to repeat the test
resmoke_args_str = buildozer.bd_print([target_original], ["resmoke_args"])
resmoke_args = resmoke_args_str.strip().removeprefix("[").removesuffix("]").split()
# "(missing)" is buildozer's response if an attribute is not present
if "(missing)" in resmoke_args:
resmoke_args.remove("(missing)")
resmoke_args.extend(
[
"--repeatTestsMax=1000",
"--repeatTestsMin=2",
"--repeatTestsSecs=600.0",
]
)
resmoke_args_str = "[" + ",".join(['"' + arg + '"' for arg in resmoke_args]) + "]"
buildozer.bd_set([target_burn_in], "resmoke_args", resmoke_args_str)
class BurnInTargetInfo(NamedTuple):
burn_in_target: str
original_target: str
test: str
def get_resmoke_configs():
with open("resmoke_suite_configs.yml", "r") as f:
return yaml.safe_load(f)
def query_targets_to_burn_in(origin_rev: str) -> List[BurnInTargetInfo]:
change_detector = LocalFileChangeDetector(origin_rev)
tests_changed = change_detector.find_changed_tests([Repo(".")])
with open(SELECTOR_FILE, "r") as f:
exclusions = yaml.safe_load(f)
targets = []
for config_label, config_path in get_resmoke_configs().items():
test_label = config_label.removeprefix("@@").removesuffix("_config")
with open(config_path, "r") as f:
config = yaml.safe_load(f)
test_kind = config["test_kind"]
if test_kind not in SUPPORTED_TEST_KINDS:
continue
if test_label in exclusions["selector"].get(test_kind, {}).get("exclude_suites", []):
continue
for test in tests_changed:
if test in exclusions["selector"].get(test_kind, {}).get("exclude_tests", []):
continue
if test not in config["selector"].get("roots"):
continue
burn_in_target = (
test_label
+ "_burn_in_"
+ test.replace("/", "_").replace("\\", "_").removeprefix("_")
)
targets.append(
BurnInTargetInfo(
burn_in_target=burn_in_target, original_target=test_label, test=test
)
)
return targets
@cache
def get_targets_with_tag(tag: str) -> List[str]:
try:
query = f"attr(tags, '\\b{tag}(?![a-zA-Z0-9_-])', //...)"
result = subprocess.run(
["bazel", "query", query],
capture_output=True,
text=True,
check=True,
)
return [line.strip() for line in result.stdout.strip().split("\n") if line.strip()]
except subprocess.CalledProcessError as e:
print(f"Failed to query bazel targets with tag '{tag}': {e}")
print(f"stdout: {e.stdout}")
print(f"stderr: {e.stderr}")
raise
def make_task(targets_to_run, variant_name):
task = Task(
name=f"resmoke_tests_burn_in_{variant_name}",
commands=[
FunctionCall(
"execute resmoke tests via bazel",
{
"targets": " ".join(targets_to_run),
"bazel_args": (
"--test_tag_filters=${resmoke_tests_tag_filter},-incompatible_with_bazel_remote_test "
"--test_arg=--testTimeout=960 "
"--test_timeout=1500 "
"--test_sharding_strategy=disabled "
"--test_arg=--sanityCheck"
),
"task_compile_flags": (
"--keep_going "
"--verbose_failures "
"--simple_build_id=True "
"--define=MONGO_VERSION=${version} "
"--config=evg "
"--features=strip_debug "
"--separate_debug=False "
"--remote_download_outputs=minimal "
"--zip_undeclared_test_outputs"
),
"generate_burn_in_targets": True,
},
),
],
)
return TaskGroup(
name=f"resmoke_tests_burn_in_{variant_name}-TG",
tasks=[task],
max_hosts=-1,
setup_task=[
BuiltInCommand("manifest.load", {}),
FunctionCall("git get project and add git tag"),
FunctionCall("set task expansion macros"),
FunctionCall("f_expansions_write"),
FunctionCall("kill processes"),
FunctionCall("cleanup environment"),
FunctionCall("set up venv"),
FunctionCall("configure evergreen api credentials"),
FunctionCall("set up credentials"),
FunctionCall("get engflow creds"),
],
teardown_task=[
BuiltInCommand("generate.tasks", {"optional": True, "files": ["generated_tasks.json"]}),
BuiltInCommand(
"s3.put",
{
"optional": True,
"aws_key": "${aws_key}",
"aws_secret": "${aws_secret}",
"local_file": "src/generated_tasks.json",
"remote_file": "${project}/${version_id}/${build_variant}/${task_name}/generated_tasks.json",
"bucket": "mciuploads",
"permissions": "private",
"visibility": "signed",
"content_type": "application/json",
},
),
BuiltInCommand(
"s3.put",
{
"optional": True,
"aws_key": "${aws_key}",
"aws_secret": "${aws_secret}",
"local_file": "src/build_events.json",
"remote_file": "${project}/${version_id}/${build_variant}/${task_name}/build_events.json",
"bucket": "mciuploads",
"permissions": "private",
"visibility": "signed",
"content_type": "application/json",
},
),
FunctionCall("debug full disk"),
FunctionCall("attach bazel invocation"),
FunctionCall("save failed tests"),
FunctionCall("f_expansions_write"),
FunctionCall("kill processes"),
],
setup_group_can_fail_task=True,
)
app = typer.Typer(pretty_exceptions_show_locals=False)
@app.command()
def generate_targets(origin_rev: str):
"""Generate burn-in test targets for changed test files."""
os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
targets = query_targets_to_burn_in(origin_rev)
print(f"\nFound {len(targets)} burn-in targets to generate\n")
for burn_in_name, original_target, test in targets:
print(f"Creating: {original_target} -> {burn_in_name}")
create_burn_in_target(original_target, burn_in_name, test)
@app.command()
def generate_tasks(origin_rev: str, outfile: Annotated[str, typer.Option()]):
os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
targets = query_targets_to_burn_in(origin_rev)
evg_conf = parse_evergreen_file("etc/evergreen.yml")
shrub_project = ShrubProject.empty()
results_tasks = []
for variant_name in evg_conf.variant_names:
variant = evg_conf.get_variant(variant_name)
if not (variant.is_required_variant() or variant.is_suggested_variant()):
continue
task = variant.get_task("resmoke_tests")
if task:
tags = variant.expansion("resmoke_tests_tag_filter").split(",")
targets_with_tag = []
for tag in tags:
targets_with_tag += get_targets_with_tag(tag)
burn_in_targets_to_run = [
target.burn_in_target
for target in targets
if target.original_target in targets_with_tag
]
if burn_in_targets_to_run:
burn_in_task = make_task(burn_in_targets_to_run, variant_name)
results_tasks.extend(
[make_results_task(target) for target in burn_in_targets_to_run]
)
build_variant = BuildVariant(name=variant_name)
build_variant.add_task_group(burn_in_task)
shrub_project.add_build_variant(build_variant)
# Patch in fields that not supported by shrub
project = shrub_project.as_dict()
project["tasks"] = project.get("tasks", [])
for variant in project.get("buildvariants", []):
for task in variant.get("tasks", []):
task["activate"] = False
for task in project["tasks"]:
task["exec_timeout_secs"] = 1800
project["tasks"].extend([task.as_dict() for task in results_tasks])
with open(outfile, "w") as f:
f.write(json.dumps(project, indent=4))
if __name__ == "__main__":
app()

4
buildscripts/ciconfig/evergreen.py
View File

@ -363,6 +363,10 @@ class Variant(object):
"""Return True if the variant is a required variant."""
return self.display_name.startswith("!")
def is_suggested_variant(self) -> bool:
"""Return True if the variant is a suggested variant."""
return self.display_name.startswith("*")
def get_task(self, task_name):
"""Return the task with the given name as an instance of VariantTask.

55
buildscripts/evergreen_activate_gen_tasks.py
View File

@ -2,6 +2,7 @@
"""Activate an evergreen task in the existing build."""
import os
import re
import sys
import click
@ -32,6 +33,7 @@ LOGGER = structlog.getLogger(__name__)
EVG_CONFIG_FILE = "./.evergreen.yml"
BURN_IN_TAGS = "burn_in_tags"
BURN_IN_TESTS = "burn_in_tests"
BAZEL_BURN_IN_TESTS = r"resmoke_tests_burn_in*"
BURN_IN_VARIANT_SUFFIX = "generated-by-burn-in-tags"
@ -69,30 +71,41 @@ def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None:
tasks_not_activated = []
if expansions.task == BURN_IN_TAGS:
version = evg_api.version_by_id(expansions.version_id)
burn_in_build_variants = [
variant
for variant in version.build_variants_map.keys()
if variant.endswith(BURN_IN_VARIANT_SUFFIX)
]
for build_variant in burn_in_build_variants:
for build_variant in version.build_variants_map.keys():
build_id = version.build_variants_map[build_variant]
task_list = evg_api.tasks_by_build(build_id)
for task in task_list:
if task.display_name == BURN_IN_TESTS:
LOGGER.info(
"Activating task", task_id=task.task_id, task_name=task.display_name
)
try:
evg_api.configure_task(task.task_id, activated=True)
except Exception:
LOGGER.error(
"Could not activate task",
task_id=task.task_id,
task_name=task.display_name,
exc_info=True,
if build_variant.endswith(BURN_IN_VARIANT_SUFFIX):
for task in task_list:
if task.display_name == BURN_IN_TESTS:
LOGGER.info(
"Activating task", task_id=task.task_id, task_name=task.display_name
)
tasks_not_activated.append(task.task_id)
try:
evg_api.configure_task(task.task_id, activated=True)
except Exception:
LOGGER.error(
"Could not activate task",
task_id=task.task_id,
task_name=task.display_name,
exc_info=True,
)
tasks_not_activated.append(task.task_id)
else:
for task in task_list:
if re.match(BAZEL_BURN_IN_TESTS, task.display_name):
LOGGER.info(
"Activating task", task_id=task.task_id, task_name=task.display_name
)
try:
evg_api.configure_task(task.task_id, activated=True)
except Exception:
LOGGER.error(
"Could not activate task",
task_id=task.task_id,
task_name=task.display_name,
exc_info=True,
)
tasks_not_activated.append(task.task_id)
else:
task_list = retry_call(

5
buildscripts/generate_result_tasks.py
View File

@ -28,8 +28,7 @@ RESMOKE_TEST_QUERY = 'attr(tags, "resmoke_suite_test", //...)'
app = typer.Typer(pretty_exceptions_show_locals=False)
def make_task(target: str) -> Task:
print(f"Generating task for {target}")
def make_results_task(target: str) -> Task:
commands = [
FunctionCall("fetch remote test results", {"test_label": target}),
]
@ -61,7 +60,7 @@ def main(outfile: Annotated[str, typer.Option()]):
test_targets = query_targets()
tasks = [make_task(target) for target in test_targets]
tasks = [make_results_task(target) for target in test_targets]
project = {"tasks": [task.as_dict() for task in tasks]}
with open(outfile, "w") as f:

2
buildscripts/resmokeconfig/suites/buildscripts_test.yml
View File

@ -6,7 +6,7 @@ selector:
- buildscripts/idl/tests/**/test_*.py
- buildscripts/bazel_rules_mongo/tests/test_*.py
exclude_files:
- buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py # Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783.
- buildscripts/tests/burn_in/test_burn_in_end2end.py # Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783.
# These tests are also @unittest.skip'ed. SERVER-48969 tracks re-enabling them.
- buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path.
- buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3.

47
buildscripts/tests/burn_in/BUILD.bazel Normal file
View File

@ -0,0 +1,47 @@
py_library(
name = "all_python_files",
srcs = glob(["*.py"]),
visibility = ["//visibility:public"],
)
py_test(
name = "test_burn_in",
srcs = [
"test_burn_in.py",
],
data = [
"//buildscripts/resmokeconfig:all_files",
"//buildscripts/resmokeconfig/loggers:all_files",
"//etc:burn_in_tests.yml",
],
deps = [
"//buildscripts:burn_in_tests",
"//buildscripts/resmokelib",
],
)
py_test(
name = "test_burn_in_end2end",
srcs = [
"test_burn_in_end2end.py",
],
data = [
"//buildscripts/resmokeconfig:all_files",
"//buildscripts/resmokeconfig/loggers:all_files",
"//etc:burn_in_tests.yml",
],
deps = [
"//buildscripts:burn_in_tests",
"//buildscripts/resmokelib",
],
)
py_test(
name = "test_bazel_burn_in",
srcs = [
"test_bazel_burn_in.py",
],
deps = [
"//buildscripts:bazel_burn_in",
],
)

0
buildscripts/tests/burn_in_tests_end2end/OWNERS.yml → buildscripts/tests/burn_in/OWNERS.yml
View File

1
buildscripts/tests/burn_in/__init__.py Normal file
View File

@ -0,0 +1 @@
"""Empty."""

228
buildscripts/tests/burn_in/test_bazel_burn_in.py Normal file
View File

@ -0,0 +1,228 @@
"""Unit tests for buildscripts/bazel_burn_in.py."""
import os
import unittest
from unittest.mock import mock_open, patch
import buildscripts.bazel_burn_in as under_test
NS = "buildscripts.bazel_burn_in"
def ns(relative_name):
"""Return a full name from a name relative to the test module's namespace."""
return NS + "." + relative_name
# Mock data fixtures
MOCK_RESMOKE_CONFIG = {"test_kind": "js_test", "selector": {"roots": ["jstests/**/*.js"]}}
MOCK_EXCLUSIONS = {"selector": {"js_test": {"exclude_suites": [], "exclude_tests": []}}}
MOCK_BUILDOZER_RULE = """resmoke_suite_test(
name = "core_config",
resmoke_args = ["--log=info"],
srcs = ["//jstests/core:all"],
shard_count = 4,
)"""
class TestParseBazelTarget(unittest.TestCase):
"""Tests for parse_bazel_target function."""
def test_parse_basic_target_with_colon(self):
"""Test parsing a basic target with colon and _config suffix."""
result = under_test.parse_bazel_target("//buildscripts/resmokeconfig:core_config")
self.assertEqual(
(os.path.join("buildscripts", "resmokeconfig", "BUILD.bazel"), "core"), result
)
def test_parse_target_without_colon(self):
"""Test parsing a target without colon."""
result = under_test.parse_bazel_target("//jstests/core")
self.assertEqual((os.path.join("jstests", "core", "BUILD.bazel"), "core"), result)
def test_parse_target_without_config_suffix(self):
"""Test parsing a target without _config suffix."""
result = under_test.parse_bazel_target("//buildscripts:test_target")
self.assertEqual((os.path.join("buildscripts", "BUILD.bazel"), "test_target"), result)
def test_parse_target_with_double_slash_prefix(self):
"""Test parsing a target with // prefix."""
result = under_test.parse_bazel_target("//path/to/package:target_name_config")
self.assertEqual(
(os.path.join("path", "to", "package", "BUILD.bazel"), "target_name"), result
)
def test_parse_target_without_double_slash(self):
"""Test parsing a target without // prefix."""
result = under_test.parse_bazel_target("buildscripts:target")
self.assertEqual((os.path.join("buildscripts", "BUILD.bazel"), "target"), result)
def test_parse_target_with_nested_path(self):
"""Test parsing a target with deeply nested path."""
result = under_test.parse_bazel_target("//a/b/c/d:target_config")
self.assertEqual((os.path.join("a", "b", "c", "d", "BUILD.bazel"), "target"), result)
def test_parse_target_edge_case_single_directory(self):
"""Test parsing a target with single directory."""
result = under_test.parse_bazel_target("//jstests:test_config")
self.assertEqual((os.path.join("jstests", "BUILD.bazel"), "test"), result)
def test_parse_target_with_underscores_in_name(self):
"""Test parsing a target with underscores in name."""
result = under_test.parse_bazel_target("//path:my_test_target_config")
self.assertEqual((os.path.join("path", "BUILD.bazel"), "my_test_target"), result)
def test_parse_target_config_in_middle_of_name(self):
"""Test that config in middle of name is NOT removed."""
result = under_test.parse_bazel_target("//path:config_test_target")
self.assertEqual((os.path.join("path", "BUILD.bazel"), "config_test_target"), result)
class TestCreateBurnInTarget(unittest.TestCase):
"""Unit tests for create_burn_in_target function."""
@patch(ns("buildozer.bd_set"))
@patch(ns("buildozer.bd_print"))
@patch("builtins.open", new_callable=mock_open)
@patch(ns("parse_bazel_target"))
def test_create_burn_in_target_basic(
self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set
):
"""Test basic burn-in target creation."""
# Setup
target_original = "//jstests:core_config"
target_burn_in = "//jstests:core_burn_in_find_js"
test = "jstests/core/find.js"
mock_parse.side_effect = [
("jstests/BUILD.bazel", "core"),
("jstests/BUILD.bazel", "core_burn_in_find_js"),
]
mock_rule = 'resmoke_suite_test(\n name = "core",\n resmoke_args = [],\n)'
mock_bd_print.side_effect = [mock_rule, "[]"]
# Execute
under_test.create_burn_in_target(target_original, target_burn_in, test)
# Assert
mock_parse.assert_any_call(target_original)
mock_parse.assert_any_call(target_burn_in)
mock_open_file.assert_called_once_with("jstests/BUILD.bazel", "a")
mock_bd_set.assert_any_call([target_burn_in], "srcs", "//jstests/core:find.js")
mock_bd_set.assert_any_call([target_burn_in], "shard_count", "1")
# Verify resmoke_args includes repeat parameters
resmoke_args_calls = [
call for call in mock_bd_set.call_args_list if call[0][1] == "resmoke_args"
]
self.assertEqual(len(resmoke_args_calls), 1)
resmoke_args_value = resmoke_args_calls[0][0][2]
self.assertIn("--repeatTestsMax=1000", resmoke_args_value)
self.assertIn("--repeatTestsMin=2", resmoke_args_value)
self.assertIn("--repeatTestsSecs=600.0", resmoke_args_value)
@patch(ns("buildozer.bd_set"))
@patch(ns("buildozer.bd_print"))
@patch("builtins.open", new_callable=mock_open)
@patch(ns("parse_bazel_target"))
def test_create_burn_in_target_with_existing_resmoke_args(
self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set
):
"""Test burn-in target creation preserves existing resmoke args."""
# Setup
target_original = "//jstests:core_config"
target_burn_in = "//jstests:core_burn_in_find_js"
test = "jstests/core/find.js"
mock_parse.side_effect = [
("jstests/BUILD.bazel", "core"),
("jstests/BUILD.bazel", "core_burn_in_find_js"),
]
mock_bd_print.side_effect = [
'resmoke_suite_test(name = "core")',
'["--log=debug" "--storageEngine=wiredTiger"]',
]
# Execute
under_test.create_burn_in_target(target_original, target_burn_in, test)
# Assert - verify existing args preserved and new args added
resmoke_args_calls = [
call for call in mock_bd_set.call_args_list if call[0][1] == "resmoke_args"
]
resmoke_args_value = resmoke_args_calls[0][0][2]
self.assertIn("--log=debug", resmoke_args_value)
self.assertIn("--storageEngine=wiredTiger", resmoke_args_value)
self.assertIn("--repeatTestsMax=1000", resmoke_args_value)
@patch(ns("buildozer.bd_set"))
@patch(ns("buildozer.bd_print"))
@patch("builtins.open", new_callable=mock_open)
@patch(ns("parse_bazel_target"))
def test_create_burn_in_target_with_missing_resmoke_args(
self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set
):
"""Test burn-in target handles missing resmoke_args."""
# Setup
target_original = "//jstests:core_config"
target_burn_in = "//jstests:core_burn_in_find_js"
test = "jstests/core/find.js"
mock_parse.side_effect = [
("jstests/BUILD.bazel", "core"),
("jstests/BUILD.bazel", "core_burn_in_find_js"),
]
mock_bd_print.side_effect = ['resmoke_suite_test(name = "core")', "(missing)"]
# Execute
under_test.create_burn_in_target(target_original, target_burn_in, test)
# Assert - verify (missing) is not in the args
resmoke_args_calls = [
call for call in mock_bd_set.call_args_list if call[0][1] == "resmoke_args"
]
resmoke_args_value = resmoke_args_calls[0][0][2]
self.assertNotIn("(missing)", resmoke_args_value)
self.assertIn("--repeatTestsMax=1000", resmoke_args_value)
@patch(ns("buildozer.bd_set"))
@patch(ns("buildozer.bd_print"))
@patch("builtins.open", new_callable=mock_open)
@patch(ns("parse_bazel_target"))
def test_create_burn_in_target_sets_correct_attributes(
self, mock_parse, mock_open_file, mock_bd_print, mock_bd_set
):
"""Test that bd_set is called with correct attributes."""
# Setup
target_original = "//jstests:core_config"
target_burn_in = "//jstests:core_burn_in_find_js"
test = "jstests/core/find.js"
mock_parse.side_effect = [
("jstests/BUILD.bazel", "core"),
("jstests/BUILD.bazel", "core_burn_in_find_js"),
]
mock_bd_print.side_effect = ['resmoke_suite_test(name = "core")', "[]"]
# Execute
under_test.create_burn_in_target(target_original, target_burn_in, test)
# Assert - bd_set called 3 times for srcs, shard_count, and resmoke_args
self.assertEqual(mock_bd_set.call_count, 3)
mock_bd_set.assert_any_call([target_burn_in], "srcs", "//jstests/core:find.js")
mock_bd_set.assert_any_call([target_burn_in], "shard_count", "1")
if __name__ == "__main__":
unittest.main()

6
buildscripts/tests/test_burn_in_tests.py → buildscripts/tests/burn_in/test_burn_in.py
View File

@ -9,9 +9,9 @@ import subprocess
import sys
import unittest
from io import StringIO
from unittest.mock import MagicMock, Mock, patch
import yaml
from mock import MagicMock, Mock, patch
import buildscripts.burn_in_tests as under_test
import buildscripts.resmokelib.parser as _parser
@ -514,3 +514,7 @@ class TestYamlBurnInExecutor(unittest.TestCase):
results = yaml.safe_load(yaml_raw)
self.assertEqual(n_tasks, len(results["discovered_tasks"]))
self.assertEqual(n_tests, len(results["discovered_tasks"][0]["suites"][0]["test_list"]))
if __name__ == "__main__":
unittest.main()

55
buildscripts/tests/burn_in/test_burn_in_end2end.py Normal file
View File

@ -0,0 +1,55 @@
import os
import subprocess
import sys
import unittest
import yaml
import buildscripts.burn_in_tests as under_test
class TestBurnInTestsEnd2End(unittest.TestCase):
@unittest.skip(
"Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783."
)
@classmethod
def setUpClass(cls):
subprocess.run(
[
sys.executable,
"buildscripts/burn_in_tests.py",
"generate-test-membership-map-file-for-ci",
]
)
@classmethod
def tearDownClass(cls):
if os.path.exists(under_test.BURN_IN_TEST_MEMBERSHIP_FILE):
os.remove(under_test.BURN_IN_TEST_MEMBERSHIP_FILE)
def test_valid_yaml_output(self):
process = subprocess.run(
[
sys.executable,
"buildscripts/burn_in_tests.py",
"run",
"--yaml",
],
text=True,
capture_output=True,
)
self.assertEqual(
0,
process.returncode,
process.stderr,
)
output = process.stdout
try:
yaml.safe_load(output)
except Exception:
self.fail(msg="burn_in_tests.py does not output valid yaml.")
if __name__ == "__main__":
unittest.main()

6
buildscripts/tests/burn_in_tests_end2end/BUILD.bazel
View File

@ -1,6 +0,0 @@
# TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets
py_library(
name = "all_python_files",
srcs = glob(["*.py"]),
visibility = ["//visibility:public"],
)

5
buildscripts/util/buildozer_utils.py
View File

@ -39,8 +39,9 @@ def bd_comment(labels: List[str], comment: str, attr: str = "", value: str = "")
_bd_command(f"comment {attr} {value} {comment}", labels)
def bd_print(labels: List[str], attrs: List[str]) -> None:
_bd_command(f'print {" ".join(attrs)}', labels)
def bd_print(labels: List[str], attrs: List[str]) -> str:
p = _bd_command(f'print {" ".join(attrs)}', labels)
return p.stdout
def bd_new_load(packages: List[str], path: str, rules: List[str]) -> None:

1
etc/BUILD.bazel
View File

@ -6,6 +6,7 @@ exports_files([
"lsan.suppressions",
"tsan.suppressions",
"extensions.yml",
"burn_in_tests.yml",
])
# This is a hack to work around the fact that the cc_library flag additional_compiler_inputs doesn't

2
etc/burn_in_tests.yml
View File

@ -1,7 +1,7 @@
# This file is used to exclude suites, tasks or tests from running in the burn_in_test task.
selector:
js_test:
# Exclude list of resmoke.py suite names.
# Exclude list of resmoke.py suite names or resmoke_suite_test target names.
exclude_suites:
# Requires an HTTP server to be running in the background.
- queryable_wt

25
etc/evergreen_yml_components/definitions.yml
View File

@ -1265,6 +1265,7 @@ functions:
build_variant: ${build_variant}
distro_id: ${distro_id}
execution: ${execution}
generate_burn_in_targets: ${generate_burn_in_targets}
otel_parent_id: ${otel_parent_id}
otel_trace_id: ${otel_trace_id}
project: ${project}
@ -1283,16 +1284,6 @@ functions:
- *f_expansions_write
- *execute_resmoke_tests_via_bazel_sh
"download build events json":
- command: s3.get
display_name: "download build events json"
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
bucket: mciuploads
remote_file: ${project}/${version_id}/${build_variant}/resmoke_tests/build_events.json
local_file: "build_events.json"
"fetch remote test results":
- command: subprocess.exec
params:
@ -1410,6 +1401,7 @@ functions:
optional: true
files:
- src/generated_resmoke_config/*.json
- src/generated_bazel_tasks.json
"generate version":
- *f_expansions_write
@ -1428,6 +1420,8 @@ functions:
- *validate_generate_tasks_config
"generate version burn in":
- *get_version_expansions
- *apply_version_expansions
- *f_expansions_write
- *configure_evergreen_api_credentials
- command: subprocess.exec
@ -1441,6 +1435,17 @@ functions:
- *validate_generate_tasks_config
- *upload_burn_in_generate_tasks_config
- *generate_resmoke_tasks_config
- command: s3.put
params:
optional: true
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: src/generated_bazel_tasks.json
remote_file: ${project}/${version_id}/${build_variant}/{task_name}/generated_bazel_tasks.json
bucket: mciuploads
permissions: private
visibility: signed
content_type: application/json
"initialize multiversion tasks":
- *f_expansions_write

11
etc/evergreen_yml_components/tasks/misc_tasks.yml
View File

@ -2136,6 +2136,9 @@ tasks:
- name: version_burn_in_gen
run_on: ubuntu2404-medium
tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
depends_on:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
- command: manifest.load
- func: "git get shallow project"
@ -2146,6 +2149,14 @@ tasks:
- func: "cleanup environment"
- func: "set up venv"
- func: "upload pip requirements"
- func: "get engflow creds"
- func: "build all resmoke configs"
vars:
targets: //...
bazel_args: >-
--build_tag_filters=resmoke_config
--noincompatible_enable_cc_toolchain_resolution
--repo_env=no_c++_toolchain=1
- func: "generate version burn in"
- name: version_gen

16
evergreen/bazel_evergreen_shutils.sh
View File

@ -351,3 +351,19 @@ bazel_evergreen_shutils::maybe_scale_test_timeout_and_append() {
bazel_args="${bazel_args:-} --test_timeout=${scaled}"
fi
}
# Queries all resmoke_config targets and outputs YAML key-value pairs.
# Usage: bazel_evergreen_shutils::query_resmoke_configs <bazel_binary> <flags> <output_file>
# example: bazel_evergreen_shutils::query_resmoke_configs "$BAZEL_BINARY" "${CONFIG_FLAGS}" "resmoke_suite_configs.yml"
# Outputs YAML entries like:
# //buildscripts/resmokeconfig:core_config: bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml
bazel_evergreen_shutils::query_resmoke_configs() {
local BAZEL_BINARY="$1"
local FLAGS="$2"
local OUTPUT_FILE="$3"
${BAZEL_BINARY} cquery ${FLAGS} 'kind(resmoke_config, //...)' \
--output=starlark \
--starlark:expr='": ".join([str(target.label).replace("@@","")] + [f.path for f in target.files.to_list()])' \
>"${OUTPUT_FILE}"
}

2
evergreen/generate_version_burn_in.sh
View File

@ -24,3 +24,5 @@ RUST_BACKTRACE=full PATH=$PATH:$HOME:/ ./mongo-task-generator \
--burn-in \
--burn-in-tests-command "python buildscripts/burn_in_tests.py run --origin-rev=$base_revision" \
$@
$python buildscripts/bazel_burn_in.py generate-tasks "$base_revision" --outfile=generated_bazel_tasks.json

12
evergreen/get_all_resmoke_suite_configs.sh
View File

@ -15,10 +15,8 @@ set -o verbose
source ./evergreen/bazel_evergreen_shutils.sh
BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
# Queries all resmoke_config targets: kind(resmoke_config, //...)
# and outputs YAML key-value pair created by the starlark expression for each target.
# str(target.label).replace('@@','') -> the target name, like //buildscripts/resmokeconfig:core_config
# f.path for f in target.files.to_list() -> the path to the config file, like bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml
${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \
--output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" >resmoke_suite_configs.yml
# Queries all resmoke_config targets and outputs YAML key-value pairs mapping targets to their config files.
bazel_evergreen_shutils::query_resmoke_configs \
"${BAZEL_BINARY}" \
"${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}" \
"resmoke_suite_configs.yml"

12
evergreen/resmoke_tests_execute_bazel.sh
View File

@ -53,12 +53,21 @@ for strategy in "${strategies[@]}"; do
done
ALL_FLAGS="${ci_flags} ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} ${patch_compile_flags:-}"
CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")"
echo "${ALL_FLAGS}" >.bazel_build_flags
# Save the invocation, intentionally excluding CI specific flags.
echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo "bazel test ${bazel_args} ${targets}" >>bazel-invocation.txt
if [ "${generate_burn_in_targets}" = "true" ]; then
echo "Generating burn-in test targets..."
base_revision="$(git merge-base ${revision} HEAD)"
${BAZEL_BINARY} build ${CONFIG_FLAGS} //... --build_tag_filters=resmoke_config
bazel_evergreen_shutils::query_resmoke_configs "${BAZEL_BINARY}" "${CONFIG_FLAGS}" "resmoke_suite_configs.yml"
${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:bazel_burn_in -- generate-targets "$base_revision" || echo "Failed to generate burn-in targets"
fi
set +o errexit
# Fetch then test with retries.
@ -92,12 +101,13 @@ if [[ "$RET" != "0" ]]; then
# The --config flag needs to stay consistent for the `bazel run` to avoid evicting the previous results.
# Strip out anything that isn't a --config flag that could interfere with the run command.
CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")"
eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_tests || true
fi
eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:append_result_tasks -- --outfile=generated_tasks.json
eval ${BAZEL_BINARY} shutdown # Explicitly shutdown the bazel server in case the Evergreen agent is tracking it for completion of this process.
# Return code 3 from `bazel test` indicates that the build was OK, but some tests failed or timed out.
# The test failures are reported in individual results tasks, so don't fail the task here.
if [[ "$RET" -eq 3 ]]; then

11
evergreen/spawnhost/download_test_binaries.py
View File

@ -16,8 +16,15 @@ def main():
task = evg_api.task_by_id(task_id)
tasks_in_variant = evg_api.tasks_by_build(task.build_id)
resmoke_tests_task = list(filter(lambda t: t.display_name == "resmoke_tests", tasks_in_variant))
assert len(resmoke_tests_task) == 1, "Could not find a unique resmoke_tests task"
if "_burn_in_" in task.display_name:
resmoke_tests_task = list(
filter(lambda t: t.display_name.startswith("resmoke_tests_burn_in"), tasks_in_variant)
)
else:
resmoke_tests_task = list(
filter(lambda t: t.display_name == "resmoke_tests", tasks_in_variant)
)
assert len(resmoke_tests_task) == 1, "Could not find a unique resmoke test task"
resmoke_tests_task = resmoke_tests_task[0]
output_dir = "/data/mci/artifacts-resmoke_tests"