diff --git a/buildscripts/bazel_burn_in.py b/buildscripts/bazel_burn_in.py index 17d069fa006..6e8ce7a3810 100644 --- a/buildscripts/bazel_burn_in.py +++ b/buildscripts/bazel_burn_in.py @@ -42,6 +42,7 @@ from buildscripts.burn_in_tests import ( SELECTOR_FILE, SUPPORTED_TEST_KINDS, LocalFileChangeDetector, + MockFileChangeDetector, ) from buildscripts.ciconfig.evergreen import parse_evergreen_file from buildscripts.generate_result_tasks import make_results_task @@ -141,8 +142,15 @@ def get_resmoke_configs(): return yaml.safe_load(f) -def query_targets_to_burn_in(origin_rev: str) -> set[BurnInTargetInfo]: - change_detector = LocalFileChangeDetector(origin_rev) +def query_targets_to_burn_in( + origin_rev: str, test_changed_files: str | None = None +) -> set[BurnInTargetInfo]: + # Use MockFileChangeDetector if test files are provided (for testing purposes) + if test_changed_files: + changed_files = {f.strip() for f in test_changed_files.split(",")} + change_detector = MockFileChangeDetector(changed_files) + else: + change_detector = LocalFileChangeDetector(origin_rev) tests_changed = change_detector.find_changed_tests([Repo(".")]) with open(SELECTOR_FILE, "r") as f: @@ -291,11 +299,17 @@ app = typer.Typer(pretty_exceptions_show_locals=False) @app.command() -def generate_targets(origin_rev: str): +def generate_targets( + origin_rev: str, + test_changed_files: Annotated[ + str | None, + typer.Option(hidden=True, help="Comma-separated list of changed files (for testing)"), + ] = None, +): """Generate burn-in test targets for changed test files.""" os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) - targets = query_targets_to_burn_in(origin_rev) + targets = query_targets_to_burn_in(origin_rev, test_changed_files) print(f"\nFound {len(targets)} burn-in targets to generate\n") for burn_in_name, original_target, test in targets: @@ -305,10 +319,17 @@ def generate_targets(origin_rev: str): @app.command() -def generate_tasks(origin_rev: str, outfile: Annotated[str, typer.Option()]): +def generate_tasks( + origin_rev: str, + outfile: Annotated[str, typer.Option()], + test_changed_files: Annotated[ + str | None, + typer.Option(hidden=True, help="Comma-separated list of changed files (for testing)"), + ] = None, +): os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) - targets = query_targets_to_burn_in(origin_rev) + targets = query_targets_to_burn_in(origin_rev, test_changed_files) evg_conf = parse_evergreen_file("etc/evergreen.yml") diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 704b38b8335..32f3fb975ef 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -538,6 +538,19 @@ class LocalFileChangeDetector(FileChangeDetector): return {} +class MockFileChangeDetector(FileChangeDetector): + """A change detector that returns a predefined list of changed files (for testing only).""" + + def __init__(self, changed_files: set[str]) -> None: + self.changed_files = changed_files + + def create_revision_map(self, repos: list[Repo]) -> RevisionMap: + return {} + + def find_changed_tests(self, repos: list[Repo]) -> set[str]: + return {os.path.normpath(path) for path in self.changed_files if is_file_a_test_file(path)} + + class BurnInExecutor(ABC): """An interface to execute discovered tests.""" @@ -748,6 +761,13 @@ def cli(): default=DEFAULT_EVG_PROJECT_FILE, help="Evergreen project config file", ) +@click.option( + "--test-changed-files", + "test_changed_files", + default=None, + hidden=True, + help="Comma-separated list of changed files to test (for testing purposes only).", +) @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED) def run( build_variant: str, @@ -761,6 +781,7 @@ def run( origin_rev: Optional[str], use_yaml: bool, evg_project_file: Optional[str], + test_changed_files: Optional[str], ) -> None: """ Run new or changed tests in repeated mode to validate their stability. @@ -794,6 +815,7 @@ def run( :param origin_rev: The revision that local changes will be compared against. :param use_yaml: Output discovered tasks in YAML. Tests will not be run. :param evg_project_file: Evergreen project config file. + :param test_changed_files: Comma-separated list of changed files (for testing only). """ _configure_logging(verbose) @@ -807,7 +829,13 @@ def run( repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] evg_conf = parse_evergreen_file(evg_project_file) - change_detector = LocalFileChangeDetector(origin_rev) + # Use MockFileChangeDetector if test files are provided (for testing purposes) + if test_changed_files: + changed_files = {f.strip() for f in test_changed_files.split(",")} + change_detector = MockFileChangeDetector(changed_files) + else: + change_detector = LocalFileChangeDetector(origin_rev) + executor = LocalBurnInExecutor(resmoke_args, repeat_config) if use_yaml: executor = YamlBurnInExecutor() diff --git a/buildscripts/resmokeconfig/suites/OWNERS.yml b/buildscripts/resmokeconfig/suites/OWNERS.yml index 44a989b0cce..12ad03c7771 100644 --- a/buildscripts/resmokeconfig/suites/OWNERS.yml +++ b/buildscripts/resmokeconfig/suites/OWNERS.yml @@ -45,6 +45,9 @@ filters: - "benchmarks_query.yml": approvers: - 10gen/query + - "buildscripts_test.yml": + approvers: + - 10gen/devprod-correctness - "bulk_write*": approvers: - 10gen/query-execution-write-exec diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml index 09967fb44ff..0bee6220576 100644 --- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml +++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml @@ -6,7 +6,6 @@ selector: - buildscripts/idl/tests/**/test_*.py - buildscripts/bazel_rules_mongo/tests/test_*.py exclude_files: - - buildscripts/tests/burn_in/test_burn_in_end2end.py # Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783. # These tests are also @unittest.skip'ed. SERVER-48969 tracks re-enabling them. - buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path. - buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3. diff --git a/buildscripts/tests/burn_in/test_bazel_burn_in_end2end.py b/buildscripts/tests/burn_in/test_bazel_burn_in_end2end.py new file mode 100644 index 00000000000..ca79080b056 --- /dev/null +++ b/buildscripts/tests/burn_in/test_bazel_burn_in_end2end.py @@ -0,0 +1,152 @@ +"""End-to-end tests for buildscripts/bazel_burn_in.py.""" + +import json +import os +import platform +import subprocess +import sys +import tempfile +import unittest + + +@unittest.skipUnless(platform.system() == "Linux", "Burn-in task generation only runs on Linux") +class TestBazelBurnInEnd2End(unittest.TestCase): + @classmethod + def setUpClass(cls): + print("\nBuilding resmoke configs with bazel...") + build_result = subprocess.run( + ["bazel", "build", "//...", "--build_tag_filters=resmoke_config", "--config=local"], + capture_output=True, + text=True, + ) + if build_result.returncode != 0: + raise RuntimeError( + f"Failed to build resmoke configs with bazel:\n" + f"stdout: {build_result.stdout}\n" + f"stderr: {build_result.stderr}" + ) + + print("Generating resmoke_suite_configs.yml...") + cquery_result = subprocess.run( + [ + "bazel", + "cquery", + "kind(resmoke_config, //jstests/suites/...)", # A subset of reality (//...), to speed up this test's runtime. + "--output=starlark", + "--starlark:expr", + "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])", + ], + capture_output=True, + text=True, + ) + + if cquery_result.returncode != 0: + raise RuntimeError( + f"Failed to query resmoke configs with bazel:\n" + f"stdout: {cquery_result.stdout}\n" + f"stderr: {cquery_result.stderr}" + ) + + with open("resmoke_suite_configs.yml", "w") as f: + f.write(cquery_result.stdout) + + @classmethod + def tearDownClass(cls): + """Clean up resmoke_suite_configs.yml file after tests complete.""" + if os.path.exists("resmoke_suite_configs.yml"): + os.remove("resmoke_suite_configs.yml") + + def test_generate_tasks(self): + mock_changed_files = "jstests/core/js/jssymbol.js" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + outfile = f.name + + try: + print("Generating tasks...") + process = subprocess.run( + [ + sys.executable, + "buildscripts/bazel_burn_in.py", + "generate-tasks", + "abc", # Is effectively ignored since --test-changed-files is used + "--outfile", + outfile, + "--test-changed-files", + mock_changed_files, + ], + text=True, + capture_output=True, + ) + + self.assertEqual( + 0, + process.returncode, + f"bazel_burn_in.py generate-tasks failed with stderr: {process.stderr}", + ) + + self.assertTrue(os.path.exists(outfile), "Output file should be created") + + with open(outfile, "r") as f: + try: + output_json = json.load(f) + self.assertIsNotNone(output_json, "Output should be valid JSON") + + self.assertIn( + "buildvariants", output_json, "JSON should contain 'buildvariants'" + ) + self.assertIn("tasks", output_json, "JSON should contain 'tasks'") + + buildvariants = output_json["buildvariants"] + tasks = output_json["tasks"] + + self.assertIsInstance(buildvariants, list, "buildvariants should be a list") + self.assertIsInstance(tasks, list, "tasks should be a list") + + # Property 1: tasks exist for the changed test (jssymbol.js) + found_burn_in_for_test = False + for task in tasks: + if "burn_in" in task["name"]: + commands = task.get("commands", []) + for cmd in commands: + if cmd.get("func") == "execute resmoke tests via bazel": + targets = cmd.get("vars", {}).get("targets", "") + if "jssymbol.js" in targets and "burn_in" in targets: + found_burn_in_for_test = True + break + if found_burn_in_for_test: + break + + self.assertTrue( + found_burn_in_for_test, + "Should have burn-in tasks for the changed test file (jssymbol.js)", + ) + + # Property 2: no duplicate tasks + task_names = [task["name"] for task in tasks] + unique_task_names = set(task_names) + self.assertEqual( + len(task_names), + len(unique_task_names), + f"Found duplicate task names: {[name for name in task_names if task_names.count(name) > 1]}", + ) + + # Property 3: no duplicate build variants + variant_names = [variant["name"] for variant in buildvariants] + unique_variant_names = set(variant_names) + self.assertEqual( + len(variant_names), + len(unique_variant_names), + f"Found duplicate variant names: {[name for name in variant_names if variant_names.count(name) > 1]}", + ) + + except json.JSONDecodeError as e: + self.fail(f"Output file does not contain valid JSON: {e}") + + finally: + if os.path.exists(outfile): + os.remove(outfile) + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/burn_in/test_burn_in_end2end.py b/buildscripts/tests/burn_in/test_burn_in_end2end.py index 4578338d8c1..58dc5879bdb 100644 --- a/buildscripts/tests/burn_in/test_burn_in_end2end.py +++ b/buildscripts/tests/burn_in/test_burn_in_end2end.py @@ -1,4 +1,5 @@ import os +import platform import subprocess import sys import unittest @@ -8,10 +9,8 @@ import yaml import buildscripts.burn_in_tests as under_test +@unittest.skipUnless(platform.system() == "Linux", "Burn-in task generation only runs on Linux") class TestBurnInTestsEnd2End(unittest.TestCase): - @unittest.skip( - "Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783." - ) @classmethod def setUpClass(cls): subprocess.run( @@ -19,7 +18,8 @@ class TestBurnInTestsEnd2End(unittest.TestCase): sys.executable, "buildscripts/burn_in_tests.py", "generate-test-membership-map-file-for-ci", - ] + ], + check=True, ) @classmethod @@ -27,28 +27,114 @@ class TestBurnInTestsEnd2End(unittest.TestCase): if os.path.exists(under_test.BURN_IN_TEST_MEMBERSHIP_FILE): os.remove(under_test.BURN_IN_TEST_MEMBERSHIP_FILE) - def test_valid_yaml_output(self): + def test_changed_files(self): + mock_changed_files = ( + "jstests/noPassthrough/shell/js/array.js,jstests/noPassthrough/shell/js/date.js" + ) process = subprocess.run( [ sys.executable, "buildscripts/burn_in_tests.py", "run", "--yaml", + "--test-changed-files", + mock_changed_files, ], text=True, capture_output=True, ) + self.assertEqual( 0, process.returncode, - process.stderr, + f"burn_in_tests.py failed with stderr: {process.stderr}", ) output = process.stdout try: - yaml.safe_load(output) - except Exception: - self.fail(msg="burn_in_tests.py does not output valid yaml.") + parsed_yaml = yaml.safe_load(output) + self.assertIsNotNone(parsed_yaml, "Output should be valid YAML") + + # Verify the structure of the YAML output + self.assertIn("discovered_tasks", parsed_yaml, "YAML should contain 'discovered_tasks'") + discovered_tasks = parsed_yaml["discovered_tasks"] + self.assertIsInstance(discovered_tasks, list, "discovered_tasks should be a list") + + # Verify we have at least one task discovered + self.assertGreater(len(discovered_tasks), 0, "Should discover at least one task") + + # Verify each task has the expected structure + for task in discovered_tasks: + self.assertIn("task_name", task, "Each task should have 'task_name'") + self.assertIn("suites", task, "Each task should have 'suites'") + self.assertIsInstance(task["suites"], list, "Task suites should be a list") + + # Verify each suite has the expected structure + for suite in task["suites"]: + self.assertIn("suite_name", suite, "Each suite should have 'suite_name'") + self.assertIn("test_list", suite, "Each suite should have 'test_list'") + self.assertIsInstance( + suite["test_list"], list, "Suite test_list should be a list" + ) + + # Verify changed test files appear in the results + all_tests = [] + for task in discovered_tasks: + for suite in task["suites"]: + all_tests.extend(suite["test_list"]) + expected_tests = [ + "jstests/noPassthrough/shell/js/array.js", + "jstests/noPassthrough/shell/js/date.js", + ] + for expected_test in expected_tests: + self.assertIn( + expected_test, + all_tests, + f"Expected test {expected_test} should be in discovered tests", + ) + + except Exception as e: + self.fail(f"burn_in_tests.py does not output valid yaml: {e}\nOutput: {output}") + + def test_non_test_files(self): + mock_changed_files = "src/mongo/shell/mongo_main.cpp" + + process = subprocess.run( + [ + sys.executable, + "buildscripts/burn_in_tests.py", + "run", + "--yaml", + "--test-changed-files", + mock_changed_files, + ], + text=True, + capture_output=True, + ) + + self.assertEqual( + 0, + process.returncode, + f"burn_in_tests.py failed with stderr: {process.stderr}", + ) + + output = process.stdout + try: + parsed_yaml = yaml.safe_load(output) + self.assertIsNotNone(parsed_yaml, "Output should be valid YAML") + + self.assertIn("discovered_tasks", parsed_yaml, "YAML should contain 'discovered_tasks'") + discovered_tasks = parsed_yaml["discovered_tasks"] + self.assertIsInstance(discovered_tasks, list, "discovered_tasks should be a list") + + self.assertEqual( + len(discovered_tasks), + 0, + "Should have no discovered tasks for non-test files", + ) + + except Exception as e: + self.fail(f"burn_in_tests.py does not output valid yaml: {e}\nOutput: {output}") if __name__ == "__main__":