SERVER-108783 Add tests for end-to-end burn-in task generation (#48771)

GitOrigin-RevId: c909d9b9c691d244fab7ecfe7d6e047c4fa18506

buildscripts/bazel_burn_in.py

@@ -42,6 +42,7 @@ from buildscripts.burn_in_tests import (
     SELECTOR_FILE,
     SUPPORTED_TEST_KINDS,
     LocalFileChangeDetector,
+    MockFileChangeDetector,
 )
 from buildscripts.ciconfig.evergreen import parse_evergreen_file
 from buildscripts.generate_result_tasks import make_results_task
@@ -141,8 +142,15 @@ def get_resmoke_configs():
         return yaml.safe_load(f)
 
 
-def query_targets_to_burn_in(origin_rev: str) -> set[BurnInTargetInfo]:
-    change_detector = LocalFileChangeDetector(origin_rev)
+def query_targets_to_burn_in(
+    origin_rev: str, test_changed_files: str | None = None
+) -> set[BurnInTargetInfo]:
+    # Use MockFileChangeDetector if test files are provided (for testing purposes)
+    if test_changed_files:
+        changed_files = {f.strip() for f in test_changed_files.split(",")}
+        change_detector = MockFileChangeDetector(changed_files)
+    else:
+        change_detector = LocalFileChangeDetector(origin_rev)
     tests_changed = change_detector.find_changed_tests([Repo(".")])
 
     with open(SELECTOR_FILE, "r") as f:
@@ -291,11 +299,17 @@ app = typer.Typer(pretty_exceptions_show_locals=False)
 
 
 @app.command()
-def generate_targets(origin_rev: str):
+def generate_targets(
+    origin_rev: str,
+    test_changed_files: Annotated[
+        str | None,
+        typer.Option(hidden=True, help="Comma-separated list of changed files (for testing)"),
+    ] = None,
+):
     """Generate burn-in test targets for changed test files."""
     os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
 
-    targets = query_targets_to_burn_in(origin_rev)
+    targets = query_targets_to_burn_in(origin_rev, test_changed_files)
     print(f"\nFound {len(targets)} burn-in targets to generate\n")
 
     for burn_in_name, original_target, test in targets:
@@ -305,10 +319,17 @@ def generate_targets(origin_rev: str):
 
 
 @app.command()
-def generate_tasks(origin_rev: str, outfile: Annotated[str, typer.Option()]):
+def generate_tasks(
+    origin_rev: str,
+    outfile: Annotated[str, typer.Option()],
+    test_changed_files: Annotated[
+        str | None,
+        typer.Option(hidden=True, help="Comma-separated list of changed files (for testing)"),
+    ] = None,
+):
     os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
 
-    targets = query_targets_to_burn_in(origin_rev)
+    targets = query_targets_to_burn_in(origin_rev, test_changed_files)
 
     evg_conf = parse_evergreen_file("etc/evergreen.yml")
 

buildscripts/burn_in_tests.py

@@ -538,6 +538,19 @@ class LocalFileChangeDetector(FileChangeDetector):
         return {}
 
 
+class MockFileChangeDetector(FileChangeDetector):
+    """A change detector that returns a predefined list of changed files (for testing only)."""
+
+    def __init__(self, changed_files: set[str]) -> None:
+        self.changed_files = changed_files
+
+    def create_revision_map(self, repos: list[Repo]) -> RevisionMap:
+        return {}
+
+    def find_changed_tests(self, repos: list[Repo]) -> set[str]:
+        return {os.path.normpath(path) for path in self.changed_files if is_file_a_test_file(path)}
+
+
 class BurnInExecutor(ABC):
     """An interface to execute discovered tests."""
 
@@ -748,6 +761,13 @@ def cli():
     default=DEFAULT_EVG_PROJECT_FILE,
     help="Evergreen project config file",
 )
+@click.option(
+    "--test-changed-files",
+    "test_changed_files",
+    default=None,
+    hidden=True,
+    help="Comma-separated list of changed files to test (for testing purposes only).",
+)
 @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED)
 def run(
     build_variant: str,
@@ -761,6 +781,7 @@ def run(
     origin_rev: Optional[str],
     use_yaml: bool,
     evg_project_file: Optional[str],
+    test_changed_files: Optional[str],
 ) -> None:
     """
     Run new or changed tests in repeated mode to validate their stability.
@@ -794,6 +815,7 @@ def run(
     :param origin_rev: The revision that local changes will be compared against.
     :param use_yaml: Output discovered tasks in YAML. Tests will not be run.
     :param evg_project_file: Evergreen project config file.
+    :param test_changed_files: Comma-separated list of changed files (for testing only).
     """
     _configure_logging(verbose)
 
@@ -807,7 +829,13 @@ def run(
     repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
     evg_conf = parse_evergreen_file(evg_project_file)
 
-    change_detector = LocalFileChangeDetector(origin_rev)
+    # Use MockFileChangeDetector if test files are provided (for testing purposes)
+    if test_changed_files:
+        changed_files = {f.strip() for f in test_changed_files.split(",")}
+        change_detector = MockFileChangeDetector(changed_files)
+    else:
+        change_detector = LocalFileChangeDetector(origin_rev)
+
     executor = LocalBurnInExecutor(resmoke_args, repeat_config)
     if use_yaml:
         executor = YamlBurnInExecutor()

buildscripts/resmokeconfig/suites/OWNERS.yml

@@ -45,6 +45,9 @@ filters:
   - "benchmarks_query.yml":
     approvers:
       - 10gen/query
+  - "buildscripts_test.yml":
+    approvers:
+      - 10gen/devprod-correctness
   - "bulk_write*":
     approvers:
       - 10gen/query-execution-write-exec

buildscripts/resmokeconfig/suites/buildscripts_test.yml

@@ -6,7 +6,6 @@ selector:
     - buildscripts/idl/tests/**/test_*.py
     - buildscripts/bazel_rules_mongo/tests/test_*.py
   exclude_files:
-    - buildscripts/tests/burn_in/test_burn_in_end2end.py # Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783.
     # These tests are also @unittest.skip'ed. SERVER-48969 tracks re-enabling them.
     - buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path.
     - buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3.

buildscripts/tests/burn_in/test_bazel_burn_in_end2end.py

@@ -0,0 +1,152 @@
+"""End-to-end tests for buildscripts/bazel_burn_in.py."""
+
+import json
+import os
+import platform
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+@unittest.skipUnless(platform.system() == "Linux", "Burn-in task generation only runs on Linux")
+class TestBazelBurnInEnd2End(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        print("\nBuilding resmoke configs with bazel...")
+        build_result = subprocess.run(
+            ["bazel", "build", "//...", "--build_tag_filters=resmoke_config", "--config=local"],
+            capture_output=True,
+            text=True,
+        )
+        if build_result.returncode != 0:
+            raise RuntimeError(
+                f"Failed to build resmoke configs with bazel:\n"
+                f"stdout: {build_result.stdout}\n"
+                f"stderr: {build_result.stderr}"
+            )
+
+        print("Generating resmoke_suite_configs.yml...")
+        cquery_result = subprocess.run(
+            [
+                "bazel",
+                "cquery",
+                "kind(resmoke_config, //jstests/suites/...)",  # A subset of reality (//...), to speed up this test's runtime.
+                "--output=starlark",
+                "--starlark:expr",
+                "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])",
+            ],
+            capture_output=True,
+            text=True,
+        )
+
+        if cquery_result.returncode != 0:
+            raise RuntimeError(
+                f"Failed to query resmoke configs with bazel:\n"
+                f"stdout: {cquery_result.stdout}\n"
+                f"stderr: {cquery_result.stderr}"
+            )
+
+        with open("resmoke_suite_configs.yml", "w") as f:
+            f.write(cquery_result.stdout)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Clean up resmoke_suite_configs.yml file after tests complete."""
+        if os.path.exists("resmoke_suite_configs.yml"):
+            os.remove("resmoke_suite_configs.yml")
+
+    def test_generate_tasks(self):
+        mock_changed_files = "jstests/core/js/jssymbol.js"
+
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            outfile = f.name
+
+        try:
+            print("Generating tasks...")
+            process = subprocess.run(
+                [
+                    sys.executable,
+                    "buildscripts/bazel_burn_in.py",
+                    "generate-tasks",
+                    "abc",  # Is effectively ignored since --test-changed-files is used
+                    "--outfile",
+                    outfile,
+                    "--test-changed-files",
+                    mock_changed_files,
+                ],
+                text=True,
+                capture_output=True,
+            )
+
+            self.assertEqual(
+                0,
+                process.returncode,
+                f"bazel_burn_in.py generate-tasks failed with stderr: {process.stderr}",
+            )
+
+            self.assertTrue(os.path.exists(outfile), "Output file should be created")
+
+            with open(outfile, "r") as f:
+                try:
+                    output_json = json.load(f)
+                    self.assertIsNotNone(output_json, "Output should be valid JSON")
+
+                    self.assertIn(
+                        "buildvariants", output_json, "JSON should contain 'buildvariants'"
+                    )
+                    self.assertIn("tasks", output_json, "JSON should contain 'tasks'")
+
+                    buildvariants = output_json["buildvariants"]
+                    tasks = output_json["tasks"]
+
+                    self.assertIsInstance(buildvariants, list, "buildvariants should be a list")
+                    self.assertIsInstance(tasks, list, "tasks should be a list")
+
+                    # Property 1: tasks exist for the changed test (jssymbol.js)
+                    found_burn_in_for_test = False
+                    for task in tasks:
+                        if "burn_in" in task["name"]:
+                            commands = task.get("commands", [])
+                            for cmd in commands:
+                                if cmd.get("func") == "execute resmoke tests via bazel":
+                                    targets = cmd.get("vars", {}).get("targets", "")
+                                    if "jssymbol.js" in targets and "burn_in" in targets:
+                                        found_burn_in_for_test = True
+                                        break
+                        if found_burn_in_for_test:
+                            break
+
+                    self.assertTrue(
+                        found_burn_in_for_test,
+                        "Should have burn-in tasks for the changed test file (jssymbol.js)",
+                    )
+
+                    # Property 2: no duplicate tasks
+                    task_names = [task["name"] for task in tasks]
+                    unique_task_names = set(task_names)
+                    self.assertEqual(
+                        len(task_names),
+                        len(unique_task_names),
+                        f"Found duplicate task names: {[name for name in task_names if task_names.count(name) > 1]}",
+                    )
+
+                    # Property 3: no duplicate build variants
+                    variant_names = [variant["name"] for variant in buildvariants]
+                    unique_variant_names = set(variant_names)
+                    self.assertEqual(
+                        len(variant_names),
+                        len(unique_variant_names),
+                        f"Found duplicate variant names: {[name for name in variant_names if variant_names.count(name) > 1]}",
+                    )
+
+                except json.JSONDecodeError as e:
+                    self.fail(f"Output file does not contain valid JSON: {e}")
+
+        finally:
+            if os.path.exists(outfile):
+                os.remove(outfile)
+
+
+if __name__ == "__main__":
+    unittest.main()

buildscripts/tests/burn_in/test_burn_in_end2end.py

@@ -1,4 +1,5 @@
 import os
+import platform
 import subprocess
 import sys
 import unittest
@@ -8,10 +9,8 @@ import yaml
 import buildscripts.burn_in_tests as under_test
 
 
+@unittest.skipUnless(platform.system() == "Linux", "Burn-in task generation only runs on Linux")
 class TestBurnInTestsEnd2End(unittest.TestCase):
-    @unittest.skip(
-        "Disabled since this test has behavior dependent on currently modified jstests. Re-enable with SERVER-108783."
-    )
     @classmethod
     def setUpClass(cls):
         subprocess.run(
@@ -19,7 +18,8 @@ class TestBurnInTestsEnd2End(unittest.TestCase):
                 sys.executable,
                 "buildscripts/burn_in_tests.py",
                 "generate-test-membership-map-file-for-ci",
-            ]
+            ],
+            check=True,
         )
 
     @classmethod
@@ -27,28 +27,114 @@ class TestBurnInTestsEnd2End(unittest.TestCase):
         if os.path.exists(under_test.BURN_IN_TEST_MEMBERSHIP_FILE):
             os.remove(under_test.BURN_IN_TEST_MEMBERSHIP_FILE)
 
-    def test_valid_yaml_output(self):
+    def test_changed_files(self):
+        mock_changed_files = (
+            "jstests/noPassthrough/shell/js/array.js,jstests/noPassthrough/shell/js/date.js"
+        )
         process = subprocess.run(
             [
                 sys.executable,
                 "buildscripts/burn_in_tests.py",
                 "run",
                 "--yaml",
+                "--test-changed-files",
+                mock_changed_files,
             ],
             text=True,
             capture_output=True,
         )
+
         self.assertEqual(
             0,
             process.returncode,
-            process.stderr,
+            f"burn_in_tests.py failed with stderr: {process.stderr}",
         )
 
         output = process.stdout
         try:
-            yaml.safe_load(output)
-        except Exception:
-            self.fail(msg="burn_in_tests.py does not output valid yaml.")
+            parsed_yaml = yaml.safe_load(output)
+            self.assertIsNotNone(parsed_yaml, "Output should be valid YAML")
+
+            # Verify the structure of the YAML output
+            self.assertIn("discovered_tasks", parsed_yaml, "YAML should contain 'discovered_tasks'")
+            discovered_tasks = parsed_yaml["discovered_tasks"]
+            self.assertIsInstance(discovered_tasks, list, "discovered_tasks should be a list")
+
+            # Verify we have at least one task discovered
+            self.assertGreater(len(discovered_tasks), 0, "Should discover at least one task")
+
+            # Verify each task has the expected structure
+            for task in discovered_tasks:
+                self.assertIn("task_name", task, "Each task should have 'task_name'")
+                self.assertIn("suites", task, "Each task should have 'suites'")
+                self.assertIsInstance(task["suites"], list, "Task suites should be a list")
+
+                # Verify each suite has the expected structure
+                for suite in task["suites"]:
+                    self.assertIn("suite_name", suite, "Each suite should have 'suite_name'")
+                    self.assertIn("test_list", suite, "Each suite should have 'test_list'")
+                    self.assertIsInstance(
+                        suite["test_list"], list, "Suite test_list should be a list"
+                    )
+
+            # Verify changed test files appear in the results
+            all_tests = []
+            for task in discovered_tasks:
+                for suite in task["suites"]:
+                    all_tests.extend(suite["test_list"])
+            expected_tests = [
+                "jstests/noPassthrough/shell/js/array.js",
+                "jstests/noPassthrough/shell/js/date.js",
+            ]
+            for expected_test in expected_tests:
+                self.assertIn(
+                    expected_test,
+                    all_tests,
+                    f"Expected test {expected_test} should be in discovered tests",
+                )
+
+        except Exception as e:
+            self.fail(f"burn_in_tests.py does not output valid yaml: {e}\nOutput: {output}")
+
+    def test_non_test_files(self):
+        mock_changed_files = "src/mongo/shell/mongo_main.cpp"
+
+        process = subprocess.run(
+            [
+                sys.executable,
+                "buildscripts/burn_in_tests.py",
+                "run",
+                "--yaml",
+                "--test-changed-files",
+                mock_changed_files,
+            ],
+            text=True,
+            capture_output=True,
+        )
+
+        self.assertEqual(
+            0,
+            process.returncode,
+            f"burn_in_tests.py failed with stderr: {process.stderr}",
+        )
+
+        output = process.stdout
+        try:
+            parsed_yaml = yaml.safe_load(output)
+            self.assertIsNotNone(parsed_yaml, "Output should be valid YAML")
+
+            self.assertIn("discovered_tasks", parsed_yaml, "YAML should contain 'discovered_tasks'")
+            discovered_tasks = parsed_yaml["discovered_tasks"]
+            self.assertIsInstance(discovered_tasks, list, "discovered_tasks should be a list")
+
+            self.assertEqual(
+                len(discovered_tasks),
+                0,
+                "Should have no discovered tasks for non-test files",
+            )
+
+        except Exception as e:
+            self.fail(f"burn_in_tests.py does not output valid yaml: {e}\nOutput: {output}")
 
 
 if __name__ == "__main__":