From fc6c183e2d249147a245c34a536022a4c92f033a Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 29 Apr 2026 13:35:19 -0500 Subject: [PATCH] SERVER-122303 Move copybara syncs to master (#50337) GitOrigin-RevId: d2045870869e65f9c864106f9351cb3e3a0db823 --- .gitignore | 1 + OWNERS.yml | 6 +- bazel/wrapper_hook/lint.py | 179 +- bazel/wrapper_hook/lint_test.py | 256 +- .../codeowners/parsers/owners_v1.py | 7 +- buildscripts/sync_repo_with_copybara.py | 907 ----- buildscripts/tests/BUILD.bazel | 17 +- .../tests/test_codeowners_auto_approver.py | 93 + .../tests/test_sync_repo_with_copybara.py | 3123 ++++++++++++++++- docs/branching/README.md | 11 +- docs/owners/owners_format.md | 8 +- etc/BUILD.bazel | 1 + etc/evergreen.yml | 2 + 13 files changed, 3498 insertions(+), 1113 deletions(-) delete mode 100644 buildscripts/sync_repo_with_copybara.py create mode 100644 buildscripts/tests/test_codeowners_auto_approver.py diff --git a/.gitignore b/.gitignore index baa0a3f1e37..eeb0959f1e1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ /.tmp !/.tmp/_placeholder_ venv +/buildscripts/copybara/copybara_path_rules.bara.sky *~ *.test_iwyu.h diff --git a/OWNERS.yml b/OWNERS.yml index 2d1a6384bdd..259b8ed671b 100644 --- a/OWNERS.yml +++ b/OWNERS.yml @@ -8,7 +8,7 @@ filters: - 10gen/server-root-ownership metadata: emeritus_approvers: - - visemet # TODO: add back to approvers once project work is finished (Ignore linting) + - visemet # TODO SERVER-122669: add back to approvers once project work is finished - "/BUILD.bazel": approvers: - 10gen/devprod-build @@ -61,10 +61,6 @@ filters: - "BUILD.bazel": approvers: - 10gen/devprod-build - - "copy.bara.sky": - approvers: - - IamXander - - smcclure15 - "eslint.config.mjs": approvers: - 10gen/devprod-test-infrastructure diff --git a/bazel/wrapper_hook/lint.py b/bazel/wrapper_hook/lint.py index 62401ca7628..38bef2892f1 100644 --- a/bazel/wrapper_hook/lint.py +++ b/bazel/wrapper_hook/lint.py @@ -314,12 +314,25 @@ class LintRunner: else: print(f"All {type_name} files have BUILD.bazel targets!") - def run_bazel(self, target: str, args: list = []): + def run_bazel(self, target: str, args: list | None = None) -> bool: + args = args or [] p = subprocess.run([self.bazel_bin, "run", target] + (["--"] + args if args else [])) if p.returncode != 0: self.fail = True if not self.keep_going: raise LinterFail("Linter failed") + return False + return True + + def check_copybara_generated_evergreen(self, *, fix: bool, dry_run: bool) -> None: + print("Checking generated Copybara Evergreen yaml...") + if fix and not dry_run: + if self.run_bazel("//buildscripts/copybara:generate_evergreen"): + print("Generated Copybara Evergreen yaml has been updated") + return + + if self.run_bazel("//buildscripts/copybara:generate_evergreen", ["--check"]): + print("Generated Copybara Evergreen yaml is up to date") def refresh_module_lockfile( self, @@ -533,6 +546,125 @@ def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> return list(file_set) +def _get_existing_python_files(files_to_lint: list[str]) -> list[str]: + """Return Python files that still exist in the working tree.""" + return [str(file) for file in files_to_lint if file.endswith(".py") and os.path.exists(file)] + + +def _source_label_to_workspace_path(label: str) -> str | None: + """Return the workspace-relative path represented by a Bazel source-file label.""" + local_repository_prefixes = { + "@bazel_rules_mongo": "buildscripts/bazel_rules_mongo", + "@@bazel_rules_mongo": "buildscripts/bazel_rules_mongo", + } + + if label.startswith("//"): + label_body = label[2:] + local_root = "" + elif label.startswith("@"): + repository, _, label_body = label.partition("//") + local_root = local_repository_prefixes.get(repository) + if local_root is None: + return None + else: + return None + + package, separator, target = label_body.partition(":") + if not separator: + return None + + return os.path.normpath(os.path.join(local_root, package, target)) + + +def _get_rules_lint_source_labels_for_changed_files( + files_to_lint: list[str], + files_with_targets: list[str], +) -> list[str]: + """Return Bazel source-file labels for changed files supported by rules_lint.""" + path_to_label = {} + for label in files_with_targets: + workspace_path = _source_label_to_workspace_path(label) + if workspace_path is not None: + path_to_label[workspace_path] = label + + rules_lint_labels = [] + seen = set() + for file in files_to_lint: + if not file.endswith((".py", ".js", ".mjs")): + continue + + workspace_path = os.path.normpath(file).removeprefix(f".{os.sep}") + label = path_to_label.get(workspace_path) + if label is None or label in seen: + continue + + rules_lint_labels.append(label) + seen.add(label) + + return rules_lint_labels + + +def _get_rules_lint_targets_for_source_labels( + bazel_bin: str, + source_labels: list[str], +) -> list[str]: + """Return Bazel rule targets that directly own the given source-file labels.""" + owner_targets = [] + seen = set() + + for source_label in source_labels: + query = f'kind(".* rule", same_pkg_direct_rdeps({source_label}))' + result = subprocess.run( + [bazel_bin, "query", query, "--output=label"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + print(f"Failed to query rules_lint owner target for {source_label}:") + if result.stderr.strip(): + print(result.stderr.strip()) + raise LinterFail(f"Failed to query rules_lint owner target for {source_label}") + + for target in result.stdout.splitlines(): + target = target.strip() + if not target or target in seen: + continue + owner_targets.append(target) + seen.add(target) + + return owner_targets + + +def _get_rules_lint_targets_for_changed_files( + bazel_bin: str, + files_to_lint: list[str], + files_with_targets: list[str], +) -> list[str]: + """Return Bazel rule targets that own changed files supported by rules_lint.""" + return _get_rules_lint_targets_for_source_labels( + bazel_bin, + _get_rules_lint_source_labels_for_changed_files(files_to_lint, files_with_targets), + ) + + +def _should_check_copybara_generated_evergreen(lint_all: bool, files_to_lint: list[str]) -> bool: + """Return whether lint should check the generated Copybara Evergreen YAML.""" + if lint_all: + return True + + copybara_prefixes = ( + "buildscripts/copybara/", + "etc/evergreen_yml_components/copybara/", + ) + copybara_files = { + "etc/evergreen.yml", + } + return any( + file in copybara_files or file.startswith(copybara_prefixes) for file in files_to_lint + ) + + def get_parsed_args(args): parser = argparse.ArgumentParser() parser.add_argument( @@ -666,17 +798,21 @@ def run_rules_lint(bazel_bin: str, args: list[str]): ): lr.run_bazel("//buildscripts:errorcodes", ["--quiet"]) + existing_python_files = _get_existing_python_files(files_to_lint) if lint_all: lr.run_bazel("//buildscripts:pyrightlint", ["lint-all"]) - elif any(file.endswith(".py") for file in files_to_lint): - lr.run_bazel( - "//buildscripts:pyrightlint", - ["lints"] + [str(file) for file in files_to_lint if file.endswith(".py")], - ) + elif existing_python_files: + lr.run_bazel("//buildscripts:pyrightlint", ["lints"] + existing_python_files) if lint_all or "poetry.lock" in files_to_lint or "pyproject.toml" in files_to_lint: lr.run_bazel("//buildscripts:poetry_lock_check") + if _should_check_copybara_generated_evergreen(lint_all, files_to_lint): + lr.check_copybara_generated_evergreen( + fix=parsed_args.fix, + dry_run=parsed_args.dry_run, + ) + if lint_all or any(file.endswith(".yml") for file in files_to_lint): print("Linting evergreen yaml...") lr.run_bazel( @@ -714,9 +850,22 @@ def run_rules_lint(bazel_bin: str, args: list[str]): if lr.fail: raise LinterFail("Linter(s) failed") - # Default to linting everything in rules_lint if no path was passed in. + # Default to linting changed files in rules_lint if no path was passed in. if len([arg for arg in args if not arg.startswith("--")]) == 0: - args = ["//..."] + args + rules_lint_targets = _get_rules_lint_targets_for_changed_files( + bazel_bin, + files_to_lint, + files_with_targets, + ) + if not rules_lint_targets: + print("No changed files with rules_lint owner targets; skipping rules_lint.") + return + + print( + f"No explicit rules_lint target provided; running rules_lint on " + f"{len(rules_lint_targets)} owner target(s)." + ) + args = rules_lint_targets + args fix = "" buildevents_fd, buildevents_path = tempfile.mkstemp() @@ -804,17 +953,25 @@ def run_rules_lint(bazel_bin: str, args: list[str]): [bazel_bin, "build"] + fix_args, check=True, stdout=sys.stdout, stderr=sys.stderr ) + applied_patch_contents: set[str] = set() for patch in _jq_files(".patch", fix_buildevents_path): if "coverage.dat" in patch or not os.path.exists(patch) or not os.path.getsize(patch): continue + patch_contents = pathlib.Path(patch).read_text(encoding="utf-8") + if patch_contents in applied_patch_contents: + continue + applied_patch_contents.add(patch_contents) + if fix == "print": print(f"From {patch}:") - with open(patch, "r", encoding="utf-8") as f: - print(f.read()) + print(patch_contents) print() elif fix == "patch": subprocess.run( - ["patch", "-p1"], check=True, stdin=open(patch, "r", encoding="utf-8") + ["patch", "-p1"], + check=True, + input=patch_contents, + text=True, ) else: print(f"ERROR: unknown fix type {fix}", file=sys.stderr) diff --git a/bazel/wrapper_hook/lint_test.py b/bazel/wrapper_hook/lint_test.py index 734b13a8de2..eaf7d1515d3 100644 --- a/bazel/wrapper_hook/lint_test.py +++ b/bazel/wrapper_hook/lint_test.py @@ -161,12 +161,11 @@ class RunRulesLintTest(unittest.TestCase): self._patches = [ mock.patch.object(lint.platform, "system", return_value="Linux"), mock.patch.object(lint, "create_build_files_in_new_js_dirs"), - mock.patch.object(lint, "list_files_with_targets", return_value=[]), + mock.patch.object(lint, "list_files_with_targets", return_value=["//:foo.py"]), mock.patch.object(lint.LintRunner, "refresh_module_lockfile"), mock.patch.object(lint.LintRunner, "list_files_without_targets"), mock.patch.object(lint.LintRunner, "run_bazel"), mock.patch.object(lint, "_git_distance", return_value=0), - mock.patch.object(lint, "_get_files_changed_since_fork_point", return_value=[]), ] for p in self._patches: p.start() @@ -180,8 +179,9 @@ class RunRulesLintTest(unittest.TestCase): extra_args: list[str], *, check_report: str | None = None, - fix_patch: str | None = None, - ) -> tuple[list[list[str]], bool, lint.LinterFail | None]: + fix_patch: str | list[str] | None = None, + changed_files: list[str] | None = None, + ) -> tuple[list[list[str]], int, lint.LinterFail | None]: """ Invoke run_rules_lint with the preamble mocked out. @@ -190,25 +190,45 @@ class RunRulesLintTest(unittest.TestCase): fix_patch: content written into the .patch file that the fix pass "finds". None means nothing to fix. - Returns (bazel_build_calls, patch_was_applied, raised_exception). + Returns (bazel_build_calls, patch_apply_count, raised_exception). """ with tempfile.TemporaryDirectory() as tmpdir: tmpdir_path = pathlib.Path(tmpdir) check_report_path = str(tmpdir_path / "check.out") - fix_patch_path = str(tmpdir_path / "fix.patch") check_events_path = str(tmpdir_path / "check_events") fix_events_path = str(tmpdir_path / "fix_events") + fix_patch_paths: list[str] = [] + if changed_files is None: + changed_files = ["foo.py"] if check_report is not None: pathlib.Path(check_report_path).write_text(check_report, encoding="utf-8") if fix_patch is not None: - pathlib.Path(fix_patch_path).write_text(fix_patch, encoding="utf-8") + fix_patches = [fix_patch] if isinstance(fix_patch, str) else fix_patch + for index, fix_patch_contents in enumerate(fix_patches): + fix_patch_path = str(tmpdir_path / f"fix_{index}.patch") + pathlib.Path(fix_patch_path).write_text( + fix_patch_contents, + encoding="utf-8", + ) + fix_patch_paths.append(fix_patch_path) bazel_build_calls: list[list[str]] = [] - patch_applied = [False] + patch_apply_count = [0] def fake_run(args, **kwargs): args = list(args) + if args[:2] == ["bazel", "query"]: + self.assertEqual( + args, + [ + "bazel", + "query", + 'kind(".* rule", same_pkg_direct_rdeps(//:foo.py))', + "--output=label", + ], + ) + return subprocess.CompletedProcess(args, 0, stdout="//:foo_lib\n") if args[:2] == ["bazel", "build"]: bazel_build_calls.append(args) return subprocess.CompletedProcess(args, 0) @@ -218,7 +238,7 @@ class RunRulesLintTest(unittest.TestCase): ext = args[3] events_path = args[-1] if ext == ".patch" and events_path == fix_events_path and fix_patch is not None: - stdout = fix_patch_path + stdout = "\n".join(fix_patch_paths) elif ( ext == ".out" and events_path == check_events_path @@ -229,7 +249,7 @@ class RunRulesLintTest(unittest.TestCase): stdout = "" return subprocess.CompletedProcess(args, 0, stdout=stdout) if args[0] == "patch": - patch_applied[0] = True + patch_apply_count[0] += 1 if "stdin" in kwargs: kwargs["stdin"].close() return subprocess.CompletedProcess(args, 0) @@ -247,6 +267,11 @@ class RunRulesLintTest(unittest.TestCase): raised: lint.LinterFail | None = None with ( + mock.patch.object( + lint, + "_get_files_changed_since_fork_point", + return_value=changed_files, + ), mock.patch.object(lint.subprocess, "run", side_effect=fake_run), mock.patch.object(lint.tempfile, "mkstemp", side_effect=fake_mkstemp), mock.patch.object(lint.os, "close"), @@ -257,7 +282,7 @@ class RunRulesLintTest(unittest.TestCase): except lint.LinterFail as e: raised = e - return bazel_build_calls, patch_applied[0], raised + return bazel_build_calls, patch_apply_count[0], raised def test_check_only_no_violations_runs_single_build_and_passes(self): builds, patched, exc = self._run([]) @@ -276,7 +301,7 @@ class RunRulesLintTest(unittest.TestCase): def test_fix_with_only_fixable_violations_applies_patch_and_passes(self): patch_content = "--- a/foo.py\n+++ b/foo.py\n@@ -1 +1 @@\n-import os,sys\n+import os\n" - builds, patched, exc = self._run(["--fix"], fix_patch=patch_content) + builds, patched, exc = self._run(["--fix", "foo.py"], fix_patch=patch_content) self.assertIsNone(exc) self.assertEqual(len(builds), 2) # First build is the fix pass — must carry the fix flags. @@ -285,11 +310,12 @@ class RunRulesLintTest(unittest.TestCase): # Second build is the check pass — must not carry fix flags. self.assertNotIn("--@aspect_rules_lint//lint:fix", builds[1]) self.assertTrue(patched) + self.assertNotIn("//...", builds[0]) def test_fix_with_unfixable_violations_remaining_applies_patch_and_fails(self): patch_content = "--- a/foo.py\n+++ b/foo.py\n@@ -1 +1 @@\n-import os,sys\n+import os\n" builds, patched, exc = self._run( - ["--fix"], + ["--fix", "foo.py"], fix_patch=patch_content, check_report="F841 local variable `result` is assigned to but never used", ) @@ -301,7 +327,8 @@ class RunRulesLintTest(unittest.TestCase): def test_fix_with_only_unfixable_violations_runs_two_builds_and_fails(self): builds, patched, exc = self._run( - ["--fix"], check_report="F841 local variable `result` is assigned to but never used" + ["--fix", "foo.py"], + check_report="F841 local variable `result` is assigned to but never used", ) self.assertIsInstance(exc, lint.LinterFail) self.assertEqual(len(builds), 2) @@ -309,11 +336,210 @@ class RunRulesLintTest(unittest.TestCase): def test_dry_run_prints_patches_without_applying_them(self): patch_content = "--- a/foo.py\n+++ b/foo.py\n@@ -1 +1 @@\n-import os,sys\n+import os\n" - builds, patched, exc = self._run(["--fix", "--dry-run"], fix_patch=patch_content) + builds, patched, exc = self._run( + ["--fix", "--dry-run", "foo.py"], + fix_patch=patch_content, + ) self.assertIsNone(exc) self.assertEqual(len(builds), 2) self.assertFalse(patched) # patch -p1 must NOT be called in dry-run mode + def test_fix_skips_duplicate_patch_contents(self): + patch_content = "--- a/foo.py\n+++ b/foo.py\n@@ -1 +1 @@\n-import os,sys\n+import os\n" + builds, patched, exc = self._run( + ["--fix", "foo.py"], + fix_patch=[patch_content, patch_content], + ) + self.assertIsNone(exc) + self.assertEqual(len(builds), 2) + self.assertEqual(patched, 1) + + def test_no_target_fix_defaults_to_changed_rules_lint_files(self): + builds, patched, exc = self._run(["--fix"]) + self.assertIsNone(exc) + self.assertEqual(len(builds), 2) + self.assertIn("//:foo_lib", builds[0]) + self.assertIn("//:foo_lib", builds[1]) + self.assertNotIn("//:foo.py", builds[0]) + self.assertNotIn("//:foo.py", builds[1]) + self.assertNotIn("//...", builds[0]) + self.assertNotIn("//...", builds[1]) + self.assertEqual(patched, 0) + + +class ExistingPythonFilesTest(unittest.TestCase): + def test_filters_deleted_python_paths(self): + files_to_lint = [ + "buildscripts/sync_repo_with_copybara.py", + "buildscripts/copybara/sync_repo_with_copybara.py", + "docs/branching/README.md", + ] + + with mock.patch.object( + lint.os.path, + "exists", + side_effect=lambda path: path == "buildscripts/copybara/sync_repo_with_copybara.py", + ): + self.assertEqual( + lint._get_existing_python_files(files_to_lint), + ["buildscripts/copybara/sync_repo_with_copybara.py"], + ) + + def test_maps_main_repo_source_label_to_workspace_path(self): + self.assertEqual( + lint._source_label_to_workspace_path("//buildscripts/copybara:generate_evergreen.py"), + "buildscripts/copybara/generate_evergreen.py", + ) + + def test_maps_local_repository_source_label_to_workspace_path(self): + self.assertEqual( + lint._source_label_to_workspace_path("@bazel_rules_mongo//codeowners:parsers/foo.py"), + "buildscripts/bazel_rules_mongo/codeowners/parsers/foo.py", + ) + + def test_get_rules_lint_source_labels_for_changed_files(self): + self.assertEqual( + lint._get_rules_lint_source_labels_for_changed_files( + [ + "buildscripts/copybara/generate_evergreen.py", + "buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py", + "etc/evergreen.yml", + ], + [ + "//buildscripts/copybara:generate_evergreen.py", + "@bazel_rules_mongo//codeowners:parsers/owners_v1.py", + "//etc:evergreen.yml", + ], + ), + [ + "//buildscripts/copybara:generate_evergreen.py", + "@bazel_rules_mongo//codeowners:parsers/owners_v1.py", + ], + ) + + def test_maps_canonical_local_repository_source_label_to_workspace_path(self): + self.assertEqual( + lint._source_label_to_workspace_path("@@bazel_rules_mongo//codeowners:parsers/foo.py"), + "buildscripts/bazel_rules_mongo/codeowners/parsers/foo.py", + ) + + def test_get_rules_lint_targets_for_source_labels_queries_owner_rules(self): + def fake_run(args, **kwargs): + self.assertEqual( + args, + [ + "bazel", + "query", + 'kind(".* rule", same_pkg_direct_rdeps(//buildscripts/copybara:generate_evergreen.py))', + "--output=label", + ], + ) + self.assertTrue(kwargs["capture_output"]) + self.assertTrue(kwargs["text"]) + self.assertFalse(kwargs["check"]) + return subprocess.CompletedProcess( + args, + 0, + stdout=( + "//buildscripts/copybara:generate_evergreen\n" + "//buildscripts/copybara:generate_evergreen_test\n" + ), + ) + + with mock.patch.object(lint.subprocess, "run", side_effect=fake_run): + self.assertEqual( + lint._get_rules_lint_targets_for_source_labels( + "bazel", + ["//buildscripts/copybara:generate_evergreen.py"], + ), + [ + "//buildscripts/copybara:generate_evergreen", + "//buildscripts/copybara:generate_evergreen_test", + ], + ) + + def test_get_rules_lint_targets_for_changed_files_returns_owner_rules(self): + with mock.patch.object( + lint, + "_get_rules_lint_targets_for_source_labels", + return_value=["//buildscripts/copybara:generate_evergreen"], + ) as mock_get_targets: + self.assertEqual( + lint._get_rules_lint_targets_for_changed_files( + "bazel", + ["buildscripts/copybara/generate_evergreen.py"], + ["//buildscripts/copybara:generate_evergreen.py"], + ), + ["//buildscripts/copybara:generate_evergreen"], + ) + + mock_get_targets.assert_called_once_with( + "bazel", + ["//buildscripts/copybara:generate_evergreen.py"], + ) + + +class CopybaraGeneratedEvergreenCheckTest(unittest.TestCase): + def test_runs_for_lint_all(self): + self.assertTrue(lint._should_check_copybara_generated_evergreen(True, [])) + + def test_runs_for_copybara_config_change(self): + self.assertTrue( + lint._should_check_copybara_generated_evergreen( + False, + ["buildscripts/copybara/v8_2.sky"], + ) + ) + + def test_runs_for_generated_copybara_yaml_change(self): + self.assertTrue( + lint._should_check_copybara_generated_evergreen( + False, + ["etc/evergreen_yml_components/copybara/copybara_gen.yml"], + ) + ) + + def test_skips_unrelated_files(self): + self.assertFalse( + lint._should_check_copybara_generated_evergreen( + False, + ["src/mongo/db/query/query.cpp"], + ) + ) + + def test_check_mode_runs_generated_yaml_check(self): + runner = lint.LintRunner(keep_going=False, bazel_bin="bazel") + + with mock.patch.object(runner, "run_bazel", return_value=True) as mock_run_bazel: + with contextlib.redirect_stdout(io.StringIO()): + runner.check_copybara_generated_evergreen(fix=False, dry_run=False) + + mock_run_bazel.assert_called_once_with( + "//buildscripts/copybara:generate_evergreen", + ["--check"], + ) + + def test_fix_mode_runs_generated_yaml_writer(self): + runner = lint.LintRunner(keep_going=False, bazel_bin="bazel") + + with mock.patch.object(runner, "run_bazel", return_value=True) as mock_run_bazel: + with contextlib.redirect_stdout(io.StringIO()): + runner.check_copybara_generated_evergreen(fix=True, dry_run=False) + + mock_run_bazel.assert_called_once_with("//buildscripts/copybara:generate_evergreen") + + def test_fix_dry_run_keeps_generated_yaml_check_only(self): + runner = lint.LintRunner(keep_going=False, bazel_bin="bazel") + + with mock.patch.object(runner, "run_bazel", return_value=True) as mock_run_bazel: + with contextlib.redirect_stdout(io.StringIO()): + runner.check_copybara_generated_evergreen(fix=True, dry_run=True) + + mock_run_bazel.assert_called_once_with( + "//buildscripts/copybara:generate_evergreen", + ["--check"], + ) + if __name__ == "__main__": unittest.main() diff --git a/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py b/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py index 1f3e6ef3c54..0f4d1a4d4bb 100644 --- a/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py +++ b/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py @@ -2,18 +2,21 @@ import glob import os import pathlib from functools import cache +from typing import Any import yaml # Parser for OWNERS.yml files version 1.0.0 class OwnersParserV1: - def parse(self, directory: str, owners_file_path: str, contents: dict[str, any]) -> list[str]: + def parse(self, directory: str, owners_file_path: str, contents: dict[str, Any]) -> list[str]: lines = [] no_parent_owners = False + no_auto_approver = False if "options" in contents: options = contents["options"] no_parent_owners = "no_parent_owners" in options and options["no_parent_owners"] + no_auto_approver = "no_auto_approver" in options and options["no_auto_approver"] if no_parent_owners: # Specfying no owners will ensure that no file in this directory has an owner unless it @@ -63,7 +66,7 @@ class OwnersParserV1: else: process_owner(approver) # Add the auto revert bot - if self.should_add_auto_approver(): + if self.should_add_auto_approver() and not no_auto_approver: process_owner("svc-auto-approve-bot") lines.append(self.get_owner_line(directory, pattern, owners)) diff --git a/buildscripts/sync_repo_with_copybara.py b/buildscripts/sync_repo_with_copybara.py deleted file mode 100644 index 13bcfbde253..00000000000 --- a/buildscripts/sync_repo_with_copybara.py +++ /dev/null @@ -1,907 +0,0 @@ -"""Module for syncing a repo with Copybara and setting up configurations.""" - -from __future__ import annotations - -import argparse -import fileinput -import os -import re -import shutil -import subprocess -import sys -from datetime import datetime -from pathlib import Path -from typing import NamedTuple, Optional - -from github import GithubIntegration - -from buildscripts.util.read_config import read_config_file -from evergreen.api import RetryingEvergreenApi - -# this will be populated by the github jwt tokens (1 hour lifetimes) -REDACTED_STRINGS = [] - -# This is the list of file globs to check for -# after the dryrun has created the destination output tree -EXCLUDED_PATTERNS = [ - "src/mongo/db/modules/", - "buildscripts/modules/", - ".github/workflows/", - "src/third_party/private/", - "sbom.private.json", - ".agents/", - ".cursor/", - ".claude/", - "AGENTS.md", - "CLAUDE.md", - ".github/CODEOWNERS", - "monguard/", - "etc/evergreen_yml_components/", -] -ACCEPTABLE_ERROR_MESSAGES = [ - # Indicates the two repositories are identical. - "No new changes to import for resolved ref", - # Indicates differences exist but no changes affect the destination (for example: exclusion rules). - "Iterative workflow produced no changes in the destination for resolved ref", - # Indicates commits have already been synced over with another copybara task. - "Updates were rejected because the remote contains work that you do", -] -PROD_PINNED_REF_VARIABLE = "prodRefForPinnedSourceCommit" - -# Commit hash of Copybara to use (v20251110) -COPYBARA_COMMIT_HASH = "3f050c9e08b84aeda98875bf1b02a3288d351333" - - -class CopybaraRepoConfig(NamedTuple): - """Copybara source and destination repo sync configuration.""" - - git_url: Optional[str] = None - repo_name: Optional[str] = None - branch: Optional[str] = None - - -class CopybaraConfig(NamedTuple): - """Copybara sync configuration.""" - - source: Optional[CopybaraRepoConfig] = None - destination: Optional[CopybaraRepoConfig] = None - - @classmethod - def empty(cls) -> CopybaraConfig: - return cls( - source=None, - destination=None, - ) - - @classmethod - def from_copybara_sky_file(cls, workflow: str, branch: str, file_path: str) -> CopybaraConfig: - with open(file_path, "r") as file: - content = file.read() - # Drop inline comments so key/value regexes do not match commented-out config. - content = re.sub(r"#.*", "", content) - - # Capture the URL string assigned to sourceUrl (inside double quotes). - source_url_match = re.search(r'sourceUrl = "(.+?)"', content) - if source_url_match is None: - return cls.empty() - - if workflow == "prod": - destination_url_match = re.search(r'prodUrl = "(.+?)"', content) - if destination_url_match is None: - return cls.empty() - else: - destination_url_match = re.search(r'testUrl = "(.+?)"', content) - if destination_url_match is None: - return cls.empty() - - # Extract "owner/repo" from a git remote URL, e.g. ".../10gen/mongo.git". - repo_name_regex = re.compile(r"([^:/]+/[^:/]+)\.git") - - source_git_url = source_url_match.group(1) - source_repo_name_match = repo_name_regex.search(source_git_url) - if source_repo_name_match is None: - return cls.empty() - - destination_git_url = destination_url_match.group(1) - destination_repo_name_match = repo_name_regex.search(destination_git_url) - if destination_repo_name_match is None: - return cls.empty() - - return cls( - source=CopybaraRepoConfig( - git_url=source_git_url, - repo_name=source_repo_name_match.group(1), - branch=branch, - ), - destination=CopybaraRepoConfig( - git_url=destination_git_url, - repo_name=destination_repo_name_match.group(1), - branch=branch, - ), - ) - - def is_complete(self) -> bool: - return self.source is not None and self.destination is not None - - -def run_command(command): - redacted_command = redact_secrets(command) - print(redacted_command) - try: - process = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Merge stderr into stdout - text=True, - bufsize=1, - ) - - output_lines = [] - for line in process.stdout: - safe_line = redact_secrets(line) - print(safe_line, end="") - output_lines.append(safe_line) - - full_output = "".join(output_lines) - process.wait() - - if process.returncode != 0: - # Attach output so except block can read it - raise subprocess.CalledProcessError( - process.returncode, redacted_command, output=full_output - ) - - return full_output - - except subprocess.CalledProcessError: - # Let main handle it - raise - - -def redact_secrets(text: str) -> str: - """Mask known token values and token-like credentials in GitHub URLs.""" - # First redact exact known values (runtime-generated app tokens). - for secret in filter(None, REDACTED_STRINGS): - text = text.replace(secret, "") - - # Then redact any tokenized GitHub URL credential to catch unknown/ambient secrets - # (for example Evergreen-generated credentials not in REDACTED_STRINGS). - return re.sub( - r"(https://x-access-token:)[^@\s]+(@github\.com)", - r"\1\2", - text, - ) - - -def create_mongodb_bot_gitconfig(): - """Create the mongodb-bot.gitconfig file with the desired content.""" - - content = """ - [user] - name = MongoDB Bot - email = mongo-bot@mongodb.com - """ - - gitconfig_path = os.path.expanduser("~/mongodb-bot.gitconfig") - - with open(gitconfig_path, "w") as file: - file.write(content) - - print("mongodb-bot.gitconfig file created.") - - -def get_installation_access_token( - app_id: int, private_key: str, installation_id: int -) -> Optional[str]: # noqa: D407,D413 - """ - Obtain an installation access token using JWT. - - Args: - - app_id (int): The application ID for GitHub App. - - private_key (str): The private key associated with the GitHub App. - - installation_id (int): The installation ID of the GitHub App for a particular account. - - Returns - - Optional[str]: The installation access token. Returns `None` if there's an error obtaining the token. - - """ - integration = GithubIntegration(app_id, private_key) - auth = integration.get_access_token(installation_id) - - if auth: - return auth.token - else: - print("Error obtaining installation token") - return None - - -def send_failure_message_to_slack(expansions, error_message): - """ - Send a failure message to a specific Slack channel when the Copybara task fails. - - :param expansions: Dictionary containing various expansion data. - """ - truncated_error_message = error_message[0:200] - task_id = expansions.get("task_id", None) - error_msg = "\n".join( - [ - "Evergreen task '* Copybara Sync Between Repos' failed", - "See troubleshooting doc .", - f"See task: .", - f"Error message: {truncated_error_message}" - + ("... (truncated)" if len(error_message) > 200 else ""), - ] - ) - - evg_api = RetryingEvergreenApi.get_api(config_file=".evergreen.yml") - evg_api.send_slack_message( - target="#devprod-build-automation", - msg=error_msg, - ) - - -def check_destination_branch_exists(copybara_config: CopybaraConfig) -> bool: - """ - Check if a specific branch exists in the destination git repository. - - Args: - - copybara_config (CopybaraConfig): Copybara configuration. - - Returns - - bool: `True` if the branch exists in the destination repository, `False` otherwise. - """ - - command = ( - f"git ls-remote {copybara_config.destination.git_url} {copybara_config.destination.branch}" - ) - output = run_command(command) - return copybara_config.destination.branch in output - - -def find_matching_commit(dir_source_repo: str, dir_destination_repo: str) -> Optional[str]: - """ - Finds a matching commit in the destination repository based on the commit hash from the source repository. - - Args: - - dir_source_repo: The directory of the source repository. - - dir_destination_repo: The directory of the destination repository. - - Returns - The hash of the matching commit if found; otherwise, prints a message and returns None. - """ - - # Navigate to the source repository - os.chdir(dir_source_repo) - - # Find the latest commit hash. - source_hash = run_command('git log --pretty=format:"%H" -1') - - # Attempt to find a matching commit in the destination repository. - commit = run_command( - f'git --git-dir={dir_destination_repo}/.git log -1 --pretty=format:"%H" --grep "GitOrigin-RevId: {source_hash}"' - ) - - first_commit = run_command("git rev-list --max-parents=0 HEAD") - - # Loop until a matching commit is found or the first commit is reached. - while len(commit.splitlines()) != 1: - current_commit = run_command('git log --pretty=format:"%H" -1') - - if current_commit.strip() == first_commit.strip(): - print( - "No matching commit found, and have reverted to the first commit of the repository." - ) - return None - - # Revert to the previous commit in the source repository and try again. - run_command("git checkout HEAD~1") - source_hash = run_command('git log --pretty=format:"%H" -1') - - # Attempt to find a matching commit again in the destination repository. - commit = run_command( - f'git --git-dir={dir_destination_repo}/.git log -1 --pretty=format:"%H" --grep "GitOrigin-RevId: {source_hash}"' - ) - return commit - - -def has_only_destination_repo_remote(repo_name: str): - """ - Check if the current directory's Git repository only contains the destination repository remote URL. - - Returns - bool: True if the repository only contains the destination repository remote URL, False otherwise. - """ - git_config_path = os.path.join(".git", "config") - with open(git_config_path, "r") as f: - config_content = f.read() - - # Define a regular expression pattern to match the '{owner}/{repo}.git' - url_pattern = r"url\s*=\s*(.*?\.git\s*)" - matches = re.findall(url_pattern, config_content) - - if len(matches) == 1 and matches[0].strip().endswith(f"{repo_name}.git"): - return True - print( - f"The current directory's Git repository contains not only the '{repo_name}.git' remote URL." - ) - return False - - -def push_branch_to_destination_repo( - destination_repo_dir: str, copybara_config: CopybaraConfig, branching_off_commit: str -): - """ - Pushes a new branch to the remote repository after ensuring it branches off the public repository. - - Args: - destination_repo_dir (str): Path to the cloned destination repository. - copybara_config (CopybaraConfig): Copybara configuration. - branching_off_commit (str): The commit hash of the matching commit in the destination repository. - - Raises - Exception: If the new branch is not branching off the destination repository. - """ - - os.chdir(destination_repo_dir) - - # Check the current repo has only destination repository remote. - if not has_only_destination_repo_remote(copybara_config.destination.repo_name): - raise Exception(f"{destination_repo_dir} git repo has not only the destination repo remote") - - # Confirm the top commit is matching the found commit before pushing - new_branch_top_commit = run_command('git log --pretty=format:"%H" -1') - if not new_branch_top_commit == branching_off_commit: - raise Exception( - "The new branch top commit does not match the branching_off_commit. Aborting push." - ) - - # Confirming whether the commit exists in the destination repository to ensure - # we are not pushing anything that isn't already in the destination repository. - # run_command will raise an exception if the commit is not found in the destination branch. - run_command(f"git branch -r --contains {new_branch_top_commit}") - - # Push the new branch to the destination repository - run_command( - f"git push {copybara_config.destination.git_url} {copybara_config.destination.branch}" - ) - - -def handle_failure(expansions, error_message, output_logs): - if not has_acceptable_copybara_message(output_logs): - send_failure_message_to_slack(expansions, error_message) - - -def create_branch_from_matching_commit(copybara_config: CopybaraConfig) -> None: - """ - Create a new branch in the copybara destination repository based on a matching commit found in - source repository and destination repository. - - Args: - copybara_config (CopybaraConfig): Copybara configuration. - """ - - # Save original directory - original_dir = os.getcwd() - - try: - # Create a unique directory based on the current timestamp. - working_dir = os.path.join( - original_dir, "make_branch_attempt_" + datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - ) - os.makedirs(working_dir, exist_ok=True) - os.chdir(working_dir) - - # Clone the specified branch of the source repository and master of destination repository - cloned_source_repo_dir = os.path.join(working_dir, "source-repo") - cloned_destination_repo_dir = os.path.join(working_dir, "destination-repo") - - run_command( - f"git clone -b {copybara_config.source.branch}" - f" {copybara_config.source.git_url} {cloned_source_repo_dir}" - ) - run_command( - f"git clone {copybara_config.destination.git_url} {cloned_destination_repo_dir}" - ) - - # Find matching commits to branching off - commit = find_matching_commit(cloned_source_repo_dir, cloned_destination_repo_dir) - if commit is not None: - # Delete the cloned_source_repo_dir folder - shutil.rmtree(cloned_source_repo_dir) - if os.path.exists(cloned_source_repo_dir): - raise Exception(cloned_source_repo_dir + ": did not get removed") - - # Once a matching commit is found, create a new branch based on it. - os.chdir(cloned_destination_repo_dir) - run_command(f"git checkout -b {copybara_config.destination.branch} {commit}") - - # Push the new branch to the remote repository - push_branch_to_destination_repo(cloned_destination_repo_dir, copybara_config, commit) - else: - print( - f"Could not find matching commits between {copybara_config.destination.repo_name}/master" - f" and {copybara_config.source.repo_name}/{copybara_config.source.branch} to branching off" - ) - sys.exit(1) - except Exception as err: - print(f"An error occurred when creating destination branch: {err}") - raise - finally: - # Change back to the original directory - os.chdir(original_dir) - - -def is_current_repo_origin(expected_repo: str) -> bool: - """Check if the current repo's origin matches 'owner/repo'.""" - try: - url = run_command("git config --get remote.origin.url").strip() - except subprocess.CalledProcessError: - return False - # Accept SSH/HTTPS-style remotes and capture the trailing "owner/repo" before ".git". - m = re.search(r"([^/:]+/[^/:]+)\.git$", url) - return bool(m and m.group(1) == expected_repo) - - -def sky_file_has_version_id(config_file: str, version_id: str) -> bool: - contents = Path(config_file).read_text() - return str(version_id) in contents - - -def branch_exists_remote(remote_url: str, branch_name: str) -> bool: - """Return True if branch exists on the remote.""" - try: - output = run_command(f"git ls-remote --heads {remote_url} {branch_name}") - return bool(output.strip()) - except subprocess.CalledProcessError: - return False - - -def canonicalize_excluded_pattern(pattern: str) -> str: - """ - Canonicalize exclusion patterns for parity checks and local matching. - - Supported forms: - - Root-relative exact file path, e.g. "AGENTS.md" - - Root-relative directory subtree, e.g. "monguard/" or "monguard/**" - """ - normalized = pattern.strip() - if not normalized: - print("ERROR: Found empty exclusion pattern.") - sys.exit(1) - - is_directory_pattern = False - if normalized.endswith("/**"): - normalized = normalized.removesuffix("/**") - is_directory_pattern = True - - if normalized.endswith("/"): - normalized = normalized.rstrip("/") - is_directory_pattern = True - - if not normalized: - print(f"ERROR: Invalid exclusion pattern '{pattern}'.") - sys.exit(1) - - # Keep local dry-run semantics explicit and aligned with what this script can evaluate. - if any(char in normalized for char in ["*", "?", "[", "]", "{", "}"]): - print( - "ERROR: Unsupported exclusion pattern " - f"'{pattern}'. Only exact paths and directory subtrees are supported." - ) - sys.exit(1) - - return f"{normalized}/" if is_directory_pattern else normalized - - -def get_checkout_relative_path(file_path: Path, preview_dir: Path) -> str: - """Return a POSIX-style path rooted at the checkout directory if present.""" - posix_parts = file_path.parts - if "checkout" in posix_parts: - checkout_index = posix_parts.index("checkout") - return Path(*posix_parts[checkout_index + 1 :]).as_posix() - return file_path.relative_to(preview_dir).as_posix() - - -def matches_excluded_pattern(path_in_checkout: str, pattern: str) -> bool: - """ - Match exclusions with checkout-root anchoring. - - Directory patterns (ending in "/") match only that directory subtree at repo root. - File patterns (without trailing slash) match exact relative paths. - """ - canonical_pattern = canonicalize_excluded_pattern(pattern) - if canonical_pattern.endswith("/"): - prefix = canonical_pattern.rstrip("/") - return path_in_checkout == prefix or path_in_checkout.startswith(prefix + "/") - return path_in_checkout == canonical_pattern - - -def extract_sky_excluded_patterns(config_file: str) -> set[str]: - contents = Path(config_file).read_text() - # Remove single-line comments so commented-out exclude entries are ignored. - contents = re.sub(r"#.*", "", contents) - - # Find the origin_files = glob(..., exclude=[...]) block and capture only the list body. - # DOTALL allows this to match when the glob call spans multiple lines. - origin_files_match = re.search( - r"origin_files\s*=\s*glob\((?:.|\n)*?exclude\s*=\s*\[(.*?)\]\s*\)", - contents, - flags=re.DOTALL, - ) - if origin_files_match is None: - print(f"ERROR: Could not locate origin_files exclude list in {config_file}") - sys.exit(1) - - excludes = set() - exclude_list = origin_files_match.group(1) - # Each exclude entry is a quoted string; capture the contents between quotes. - for pattern_match in re.finditer(r'"([^"]+)"', exclude_list): - excludes.add(pattern_match.group(1)) - return excludes - - -def has_acceptable_copybara_message(output_logs: Optional[str]) -> bool: - return bool( - output_logs - and any( - acceptable_message in output_logs for acceptable_message in ACCEPTABLE_ERROR_MESSAGES - ) - ) - - -def check_script_exclusions_match_sky(config_file: str): - sky_excluded = { - canonicalize_excluded_pattern(pattern) - for pattern in extract_sky_excluded_patterns(config_file) - } - script_excluded = {canonicalize_excluded_pattern(pattern) for pattern in EXCLUDED_PATTERNS} - missing = sorted(script_excluded - sky_excluded) - extra = sorted(sky_excluded - script_excluded) - if missing or extra: - if missing: - print(f"ERROR: Missing required exclusions in {config_file}: " + ", ".join(missing)) - if extra: - print(f"ERROR: Unexpected exclusions in {config_file}: " + ", ".join(extra)) - sys.exit(1) - - -def pin_prod_workflow_ref_to_commit(config_file: str, source_commit_sha: str): - contents = Path(config_file).read_text() - # Match exactly one assignment line for PROD_PINNED_REF_VARIABLE. - # Group 1 preserves leading indentation so formatting stays unchanged on replacement. - variable_pattern = rf"^(\s*){re.escape(PROD_PINNED_REF_VARIABLE)}\s*=\s*\"[^\"]*\"\s*$" - if not re.search(variable_pattern, contents, flags=re.MULTILINE): - print( - f"ERROR: Could not pin prod workflow ref in {config_file}. " - f'Expected to find variable assignment for "{PROD_PINNED_REF_VARIABLE}".' - ) - sys.exit(1) - updated_contents = re.sub( - variable_pattern, - lambda m: f'{m.group(1)}{PROD_PINNED_REF_VARIABLE} = "{source_commit_sha}"', - contents, - flags=re.MULTILINE, - ) - Path(config_file).write_text(updated_contents) - - -def get_prod_pinned_source_ref(config_file: str) -> str: - contents = Path(config_file).read_text() - variable_pattern = rf'^\s*{re.escape(PROD_PINNED_REF_VARIABLE)}\s*=\s*"([^"]+)"\s*(?:#.*)?$' - match = re.search(variable_pattern, contents, flags=re.MULTILINE) - if match is None: - print( - f"ERROR: Could not read pinned prod source ref from {config_file}. " - f'Expected to find variable assignment for "{PROD_PINNED_REF_VARIABLE}".' - ) - sys.exit(1) - pinned_ref = match.group(1).strip() - if not pinned_ref: - print( - f"ERROR: {PROD_PINNED_REF_VARIABLE} in {config_file} is empty. " - "Expected a branch name such as master or v8.0." - ) - sys.exit(1) - return pinned_ref - - -def get_prod_copybara_config_from_master(current_dir: str) -> str: - source_config_file = os.path.join(current_dir, "copy.bara.sky") - source_ref = get_prod_pinned_source_ref(source_config_file) - run_command(f"git fetch origin {source_ref}") - source_commit_sha = run_command(f"git rev-parse origin/{source_ref}").strip() - config_file = os.path.join(current_dir, "tmp_copybara_config_from_master.sky") - sky_contents = run_command(f"git --no-pager show {source_commit_sha}:copy.bara.sky") - Path(config_file).write_text(sky_contents) - pin_prod_workflow_ref_to_commit(config_file, source_commit_sha) - return config_file - - -def delete_remote_branch(remote_url: str, branch_name: str): - """Delete branch from remote if it exists.""" - if branch_exists_remote(remote_url, branch_name): - print(f"Deleting remote branch {branch_name} from {remote_url}") - run_command(f"git push {remote_url} --delete {branch_name}") - - -def push_test_branches(copybara_config, expansions): - """Push test branch with Evergreen patch changes to source, and clean revision to destination.""" - # Safety checks - if copybara_config.source.branch != copybara_config.destination.branch: - print( - f"ERROR: test branches must match: source={copybara_config.source.branch} dest={copybara_config.destination.branch}" - ) - sys.exit(1) - if not copybara_config.source.branch.startswith( - "copybara_test_branch" - ) or not copybara_config.destination.branch.startswith("copybara_test_branch"): - print(f"ERROR: can not push non copybara test branch: {copybara_config.source.branch}") - sys.exit(1) - if not is_current_repo_origin("10gen/mongo"): - print("Refusing to push copybara_test_branch to non 10gen/mongo repo") - sys.exit(1) - - # First, delete stale remote branches if present - delete_remote_branch(copybara_config.source.git_url, copybara_config.source.branch) - delete_remote_branch(copybara_config.destination.git_url, copybara_config.destination.branch) - - # --- Push patched branch to DEST repo (local base Evergreen state) --- - run_command(f"git remote add dest_repo {copybara_config.destination.git_url}") - run_command(f"git checkout -B {copybara_config.destination.branch}") - run_command(f"git push dest_repo {copybara_config.destination.branch}") - - # --- Push patched branch to SOURCE repo (local patched Evergreen state) --- - run_command(f'git commit -am "Evergreen patch for version_id {expansions["version_id"]}"') - run_command(f"git remote add source_repo {copybara_config.source.git_url}") - run_command(f"git push source_repo {copybara_config.source.branch}") - - -def main(): - global REDACTED_STRINGS - """Clone the Copybara repo, build its Docker image, and set up and run migrations.""" - parser = argparse.ArgumentParser() - - parser.add_argument( - "--expansions-file", - "-e", - default="../expansions.yml", - help="Location of expansions file generated by evergreen.", - ) - - parser.add_argument( - "--workflow", - default="test", - choices=["prod", "test"], - help="The copybara workflow to use (test is a dryrun)", - ) - - args = parser.parse_args() - - # Check if the copybara directory already exists - if os.path.exists("copybara"): - print("Copybara directory already exists.") - else: - run_command("git clone https://github.com/10gen/copybara.git") - - # Checkout the specific commit of Copybara we want to use - run_command(f"cd copybara && git checkout {COPYBARA_COMMIT_HASH}") - - # Navigate to the Copybara directory and build the Copybara Docker image - run_command("cd copybara && docker build --rm -t copybara_container .") - - # Read configurations - expansions = read_config_file(args.expansions_file) - - token_mongodb_mongo = get_installation_access_token( - expansions["app_id_copybara_syncer_after_fix"], - expansions["private_key_copybara_syncer"], - expansions["installation_id_copybara_syncer"], - ) - token_10gen_mongo = get_installation_access_token( - expansions["app_id_copybara_syncer_10gen"], - expansions["private_key_copybara_syncer_10gen"], - expansions["installation_id_copybara_syncer_10gen"], - ) - - REDACTED_STRINGS += [token_mongodb_mongo, token_10gen_mongo] - - tokens_map = { - "https://github.com/mongodb/mongo.git": token_mongodb_mongo, - "https://github.com/10gen/mongo.git": token_10gen_mongo, - "https://github.com/10gen/mongo-copybara.git": token_10gen_mongo, - } - - # Create the mongodb-bot.gitconfig file as necessary. - create_mongodb_bot_gitconfig() - - current_dir = os.getcwd() - - if args.workflow == "test": - test_args = ["--init-history", f"--last-rev={expansions['revision']}"] - branch = f"copybara_test_branch_{expansions['version_id']}" - test_branch_str = 'testBranch = "copybara_test_branch"' - config_file = f"{current_dir}/copy.bara.sky" - elif args.workflow == "prod": - if expansions["is_patch"] == "true": - print("ERROR: prod workflow should not be run in patch builds!") - sys.exit(1) - test_args = [] - branch = "master" - config_file = get_prod_copybara_config_from_master(current_dir) - else: - raise Exception(f"invalid workflow {args.workflow}") - - # Overwrite repo urls in copybara config in-place - with fileinput.FileInput(config_file, inplace=True) as file: - for line in file: - token = None - - # Replace GitHub URL with token-authenticated URL - for repo, value in tokens_map.items(): - if repo in line: - token = value - break # no need to check other repos - - if token: - print( - line.replace( - "https://github.com", - f"https://x-access-token:{token}@github.com", - ), - end="", - ) - - # Update testBranch in .sky file if running test workflow - elif args.workflow == "test" and test_branch_str in line: - print( - line.replace( - test_branch_str, - test_branch_str[:-1] + f"_{expansions['version_id']}\"\n", - ), - end="", - ) - - else: - print(line, end="") - - if args.workflow == "test": - if not sky_file_has_version_id(config_file, expansions["version_id"]): - print( - f"Copybara test branch in {config_file} does not contain version_id {expansions['version_id']}" - ) - sys.exit(1) - - copybara_config = CopybaraConfig.from_copybara_sky_file(args.workflow, branch, config_file) - - if args.workflow == "test": - push_test_branches(copybara_config, expansions) - - # Create destination branch if it does not exist - if not copybara_config.is_complete(): - print("ERROR!!!") - print( - f"ERROR!!! Source or destination configuration could not be parsed from the {config_file}." - ) - print("ERROR!!!") - sys.exit(1) - else: - if args.workflow == "prod": - if not check_destination_branch_exists(copybara_config): - create_branch_from_matching_commit(copybara_config) - print( - f"New branch named '{copybara_config.destination.branch}' has been created" - f" for the '{copybara_config.destination.repo_name}' repo" - ) - else: - print( - f"The branch named '{copybara_config.destination.branch}' already exists" - f" in the '{copybara_config.destination.repo_name}' repo." - ) - - os.makedirs("tmp_copybara") - - docker_cmd = [ - "docker", - "run", - "--rm", - "-v", - f"{os.path.expanduser('~/.ssh')}:/root/.ssh", - "-v", - f"{os.path.expanduser('~/mongodb-bot.gitconfig')}:/root/.gitconfig", - "-v", - f"{config_file}:/usr/src/app/copy.bara.sky", - "-v", - f"{os.getcwd()}/tmp_copybara:/tmp/copybara-preview", - "copybara_container", - "migrate", - "/usr/src/app/copy.bara.sky", - args.workflow, - "-v", - "--output-root=/tmp/copybara-preview", - ] - - try: - run_command(" ".join(docker_cmd + ["--dry-run"] + test_args)) - - found_forbidden = False - preview_dir = Path("tmp_copybara") - check_script_exclusions_match_sky(config_file) - - for file_path in preview_dir.rglob("*"): - if file_path.is_file(): - path_in_checkout = get_checkout_relative_path(file_path, preview_dir) - for pattern in EXCLUDED_PATTERNS: - if matches_excluded_pattern(path_in_checkout, pattern): - print(f"ERROR: Found excluded path: {file_path}") - found_forbidden = True - - if found_forbidden: - sys.exit(1) - except subprocess.CalledProcessError as err: - if has_acceptable_copybara_message(err.output): - print("Copybara dry-run reported an acceptable no-op result. Skipping sync.") - return - if args.workflow == "prod": - error_message = f"Copybara failed with error: {err.returncode}" - handle_failure(expansions, error_message, err.output) - raise - - # Write newly generated tokens to the config file to make sure - # the token isn't expired by the time the dry-run finishes - token_mongodb_mongo = get_installation_access_token( - expansions["app_id_copybara_syncer_after_fix"], - expansions["private_key_copybara_syncer"], - expansions["installation_id_copybara_syncer"], - ) - token_10gen_mongo = get_installation_access_token( - expansions["app_id_copybara_syncer_10gen"], - expansions["private_key_copybara_syncer_10gen"], - expansions["installation_id_copybara_syncer_10gen"], - ) - - REDACTED_STRINGS += [token_mongodb_mongo, token_10gen_mongo] - - tokens_map = { - "mongodb/mongo.git": token_mongodb_mongo, - "10gen/mongo.git": token_10gen_mongo, - "10gen/mongo-copybara.git": token_10gen_mongo, - } - - with fileinput.FileInput(config_file, inplace=True) as file: - for line in file: - token = None - - for repo, value in tokens_map.items(): - if repo in line: - token = value - break - - if token: - print( - # Replace any existing GitHub token in the URL while preserving the rest of the line. - re.sub( - r"https://x-access-token:.*@github.com", - f"https://x-access-token:{token}@github.com", - line, - ), - end="", - ) - else: - print(line, end="") - - # dry run successful, time to push - try: - run_command(" ".join(docker_cmd + test_args)) - except subprocess.CalledProcessError as err: - if has_acceptable_copybara_message(err.output): - print("Copybara migrate reported an acceptable no-op result.") - return - if args.workflow == "prod": - error_message = f"Copybara failed with error: {err.returncode}" - handle_failure(expansions, error_message, err.output) - raise - - -if __name__ == "__main__": - main() diff --git a/buildscripts/tests/BUILD.bazel b/buildscripts/tests/BUILD.bazel index d65b84b28c6..efa42a5041b 100644 --- a/buildscripts/tests/BUILD.bazel +++ b/buildscripts/tests/BUILD.bazel @@ -1,6 +1,6 @@ load("@poetry//:dependencies.bzl", "dependency") load("//bazel:mongo_script_rules.bzl", "mongo_toolchain_py_cxx_test") -load("@rules_python//python:defs.bzl", "py_library") +load("@rules_python//python:defs.bzl", "py_library", "py_test") mongo_toolchain_py_cxx_test( name = "test_clang_tidy", @@ -19,6 +19,21 @@ mongo_toolchain_py_cxx_test( ], ) +py_test( + name = "test_sync_repo_with_copybara", + srcs = [ + "test_sync_repo_with_copybara.py", + ], + data = [ + "//buildscripts/copybara:copybara_config_files", + "//etc:evergreen_yml_components/copybara/copybara_gen.yml", + ], + visibility = ["//visibility:public"], + deps = [ + "//buildscripts/copybara", + ], +) + # TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets py_library( name = "all_python_files", diff --git a/buildscripts/tests/test_codeowners_auto_approver.py b/buildscripts/tests/test_codeowners_auto_approver.py new file mode 100644 index 00000000000..f3554d519ca --- /dev/null +++ b/buildscripts/tests/test_codeowners_auto_approver.py @@ -0,0 +1,93 @@ +"""Tests for OWNERS auto-approver generation behavior.""" + +import importlib.util +import os +import sys +import types +import unittest +from copy import deepcopy +from pathlib import Path +from unittest.mock import patch + +CODEOWNERS_ROOT = Path(__file__).resolve().parents[1] / "bazel_rules_mongo" / "codeowners" +PARSERS_ROOT = CODEOWNERS_ROOT / "parsers" + + +def _load_module(module_name: str, module_path: Path): + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load module spec for {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +codeowners_package = types.ModuleType("codeowners") +parsers_package = types.ModuleType("codeowners.parsers") +sys.modules.setdefault("codeowners", codeowners_package) +sys.modules.setdefault("codeowners.parsers", parsers_package) + +owners_v1_module = _load_module("codeowners.parsers.owners_v1", PARSERS_ROOT / "owners_v1.py") +sys.modules["codeowners.parsers.owners_v1"] = owners_v1_module +OwnersParserV1 = owners_v1_module.OwnersParserV1 + +owners_v2_module = _load_module("codeowners.parsers.owners_v2", PARSERS_ROOT / "owners_v2.py") +sys.modules["codeowners.parsers.owners_v2"] = owners_v2_module +OwnersParserV2 = owners_v2_module.OwnersParserV2 + + +class _TestOwnersParserV1(OwnersParserV1): + def test_pattern(self, pattern: str) -> bool: + return True + + +class _TestOwnersParserV2(OwnersParserV2): + def test_pattern(self, pattern: str) -> bool: + return True + + +class TestCodeownersAutoApprover(unittest.TestCase): + def setUp(self) -> None: + self.contents = { + "filters": [ + { + "*": None, + "approvers": ["example-user"], + } + ] + } + + def test_adds_auto_approver_without_opt_out(self) -> None: + with patch.dict(os.environ, {"ADD_AUTO_APPROVE_USER": "true"}, clear=False): + for parser_cls in (_TestOwnersParserV1, _TestOwnersParserV2): + with self.subTest(parser=parser_cls.__name__): + parser = parser_cls() + result = parser.parse( + "buildscripts/copybara", + "buildscripts/copybara/OWNERS.yml", + deepcopy(self.contents), + ) + self.assertEqual(len(result), 1) + self.assertIn("@example-user", result[0]) + self.assertIn("@svc-auto-approve-bot", result[0]) + + def test_skips_auto_approver_when_owners_file_opts_out(self) -> None: + contents = deepcopy(self.contents) + contents["options"] = {"no_auto_approver": True} + + with patch.dict(os.environ, {"ADD_AUTO_APPROVE_USER": "true"}, clear=False): + for parser_cls in (_TestOwnersParserV1, _TestOwnersParserV2): + with self.subTest(parser=parser_cls.__name__): + parser = parser_cls() + result = parser.parse( + "buildscripts/copybara", + "buildscripts/copybara/OWNERS.yml", + deepcopy(contents), + ) + self.assertEqual(len(result), 1) + self.assertIn("@example-user", result[0]) + self.assertNotIn("@svc-auto-approve-bot", result[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/test_sync_repo_with_copybara.py b/buildscripts/tests/test_sync_repo_with_copybara.py index 2184a80fed1..19fa9db49d9 100644 --- a/buildscripts/tests/test_sync_repo_with_copybara.py +++ b/buildscripts/tests/test_sync_repo_with_copybara.py @@ -1,11 +1,99 @@ +import io +import json import os +import shutil +import subprocess import sys import tempfile +import textwrap import traceback import unittest -from unittest.mock import patch +from collections.abc import Sequence +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import MagicMock, call, patch -from buildscripts import sync_repo_with_copybara +from buildscripts.copybara import generate_evergreen, sync_repo_with_copybara +from buildscripts.copybara.path_rules import render_copybara_path_rules_module_from_files + +DEFAULT_COMMON_EXCLUDED_PATTERNS = [ + ".agents/**", + ".claude/**", + ".cursor/**", + ".github/CODEOWNERS", + ".github/workflows/**", + "AGENTS.md", + "CLAUDE.md", + "buildscripts/copybara/**", + "buildscripts/modules/**", + "etc/evergreen_yml_components/**", + "monguard/**", + "sbom.private.json", + "src/mongo/db/modules/**", + "src/third_party/private/**", +] + +DEFAULT_TEST_COPYBARA_PATH_RULES_INCLUDES = ("**",) +DEFAULT_TEST_COPYBARA_PATH_RULES_EXCLUDES = tuple(DEFAULT_COMMON_EXCLUDED_PATTERNS) + +REPO_COPYBARA_TEMPLATE_PATH = ( + Path(__file__).resolve().parents[2] + / "buildscripts" + / "copybara" + / "copybara_path_rules.bara.sky.template" +) + + +def get_repo_base_copybara_config_path(root: Path) -> Path: + return root / sync_repo_with_copybara.COPYBARA_BASE_CONFIG_PATH + + +def get_repo_copybara_path_rules_path(root: Path) -> Path: + return root / sync_repo_with_copybara.COPYBARA_PATH_RULES_PATH + + +def get_repo_copybara_path_rules_template_path(root: Path) -> Path: + return root / sync_repo_with_copybara.COPYBARA_PATH_RULES_TEMPLATE_PATH + + +def get_repo_copybara_path_rules_module_path(root: Path) -> Path: + return root / sync_repo_with_copybara.COPYBARA_PATH_RULES_MODULE_PATH + + +def write_copybara_path_rules( + path: Path, + *, + common_includes: Sequence[str], + common_excludes: Sequence[str], +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "common_files_to_include": list(common_includes), + "common_files_to_exclude": list(common_excludes), + }, + indent=2, + ) + + "\n" + ) + + +def write_copybara_path_rules_module(path: Path, path_rules_path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + render_copybara_path_rules_module_from_files( + REPO_COPYBARA_TEMPLATE_PATH, + path_rules_path, + ) + ) + + +def write_copybara_path_rules_template(path: Path, template_text: str | None = None) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if template_text is None: + template_text = REPO_COPYBARA_TEMPLATE_PATH.read_text() + path.write_text(template_text) @unittest.skipIf( @@ -47,6 +135,7 @@ class TestBranchFunctions(unittest.TestCase): sync_repo_with_copybara.run_command("git init") sync_repo_with_copybara.run_command('git config --local user.email "test@example.com"') sync_repo_with_copybara.run_command('git config --local user.name "Test User"') + sync_repo_with_copybara.run_command("git config --local commit.gpgsign false") # Used to store commit hashes commit_hashes = [] for i in range(num_commits): @@ -143,6 +232,47 @@ class TestBranchFunctions(unittest.TestCase): result = self.mock_search(test_name, 2, 0) self.assertIsNone(result, f"{test_name}: SUCCESS!") + def test_prefers_newest_destination_commit_for_duplicate_origin_rev_id(self): + with tempfile.TemporaryDirectory() as tmpdir: + source_dir = os.path.join(tmpdir, "source") + destination_dir = os.path.join(tmpdir, "destination") + os.mkdir(source_dir) + os.mkdir(destination_dir) + + private_hashes = TestBranchFunctions.create_mock_repo_commits(source_dir, 1) + public_hashes = TestBranchFunctions.create_mock_repo_commits( + destination_dir, + 2, + [private_hashes[0], private_hashes[0]], + ) + + result = sync_repo_with_copybara.find_matching_commit(source_dir, destination_dir) + self.assertEqual(result, public_hashes[-1]) + + def test_find_matching_commit_pair_returns_source_and_destination(self): + with tempfile.TemporaryDirectory() as tmpdir: + source_dir = os.path.join(tmpdir, "source") + destination_dir = os.path.join(tmpdir, "destination") + os.mkdir(source_dir) + os.mkdir(destination_dir) + + private_hashes = TestBranchFunctions.create_mock_repo_commits(source_dir, 3) + public_hashes = TestBranchFunctions.create_mock_repo_commits( + destination_dir, + 2, + [private_hashes[0], private_hashes[1]], + ) + + result = sync_repo_with_copybara.find_matching_commit_pair(source_dir, destination_dir) + + self.assertEqual( + result, + sync_repo_with_copybara.MatchingCommit( + source_commit=private_hashes[1], + destination_commit=public_hashes[-1], + ), + ) + def test_branch_exists(self): """Perform a test to check that the branch exists in a repository.""" test_name = "branch_exists_test" @@ -184,6 +314,76 @@ class TestBranchFunctions(unittest.TestCase): result = sync_repo_with_copybara.check_destination_branch_exists(copybara_config) self.assertFalse(result, f"{test_name}: SUCCESS!") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_branch_exists_remote_requires_exact_branch_match(self, mock_run_command): + mock_run_command.return_value = "\n".join( + [ + "08995ea824ba2492ba1e496a2fb58e80ea2d22c3\trefs/heads/markbenvenuto/master", + "b4dbdfd07f20ec4e0f4873bff4059073c9da62c4\trefs/heads/sql/master", + ] + ) + + self.assertFalse( + sync_repo_with_copybara.branch_exists_remote("https://example.com/source.git", "master") + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_get_remote_branch_head_prefers_exact_branch_match(self, mock_run_command): + mock_run_command.return_value = "\n".join( + [ + "08995ea824ba2492ba1e496a2fb58e80ea2d22c3\trefs/heads/markbenvenuto/master", + "78efcf74c13378efe35e4e49fdf7cf0c9206af56\trefs/heads/master", + "b4dbdfd07f20ec4e0f4873bff4059073c9da62c4\trefs/heads/sql/master", + ] + ) + + self.assertEqual( + sync_repo_with_copybara.get_remote_branch_head( + "https://example.com/source.git", "master" + ), + "78efcf74c13378efe35e4e49fdf7cf0c9206af56", + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_check_destination_branch_exists_requires_exact_branch_match(self, mock_run_command): + mock_run_command.return_value = "\n".join( + [ + "08995ea824ba2492ba1e496a2fb58e80ea2d22c3\trefs/heads/markbenvenuto/master", + "b4dbdfd07f20ec4e0f4873bff4059073c9da62c4\trefs/heads/sql/master", + ] + ) + + self.assertFalse( + sync_repo_with_copybara.check_destination_branch_exists( + sync_repo_with_copybara.CopybaraConfig( + destination=sync_repo_with_copybara.CopybaraRepoConfig( + git_url="https://example.com/destination.git", + branch="master", + ) + ) + ) + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_check_destination_branch_exists_accepts_exact_branch_match(self, mock_run_command): + mock_run_command.return_value = "\n".join( + [ + "78efcf74c13378efe35e4e49fdf7cf0c9206af56\trefs/heads/master", + "b4dbdfd07f20ec4e0f4873bff4059073c9da62c4\trefs/heads/sql/master", + ] + ) + + self.assertTrue( + sync_repo_with_copybara.check_destination_branch_exists( + sync_repo_with_copybara.CopybaraConfig( + destination=sync_repo_with_copybara.CopybaraRepoConfig( + git_url="https://example.com/destination.git", + branch="master", + ) + ) + ) + ) + def test_only_mongodb_mongo_repo(self): """Perform a test that the repository is only the MongoDB official repository.""" test_name = "only_mongodb_mongo_repo_test" @@ -298,203 +498,2798 @@ class TestBranchFunctions(unittest.TestCase): self.fail(f"{test_name}: FAIL!") +class TestReleaseTagHelpers(unittest.TestCase): + def test_parse_release_tag_request_maps_public_branch(self): + self.assertEqual( + sync_repo_with_copybara.parse_release_tag_request("r8.2.7"), + sync_repo_with_copybara.ReleaseTagRequest( + release_tag="r8.2.7", + public_branch="v8.2.7", + ), + ) + + def test_parse_release_tag_request_preserves_suffix_in_public_branch(self): + self.assertEqual( + sync_repo_with_copybara.parse_release_tag_request("r8.2.7-hotfix"), + sync_repo_with_copybara.ReleaseTagRequest( + release_tag="r8.2.7-hotfix", + public_branch="v8.2.7-hotfix", + ), + ) + + def test_parse_release_tag_request_rejects_invalid_format(self): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.parse_release_tag_request("r8.2") + + def test_prepared_copybara_workflow_name_prefers_release_tag(self): + self.assertEqual( + sync_repo_with_copybara.get_prepared_copybara_workflow_name( + "prod", "v8.2.7-hotfix", "r8.2.7-hotfix" + ), + "prod_r8.2.7-hotfix", + ) + + def test_prepared_copybara_workflow_name_uses_branch_without_release_tag(self): + self.assertEqual( + sync_repo_with_copybara.get_prepared_copybara_workflow_name("prod", "v8.2"), + "prod_v8.2", + ) + + def test_parse_remote_tag_commit_prefers_exact_peeled_match(self): + output = "\n".join( + [ + "1111111111111111111111111111111111111111\trefs/tags/r8.2.70", + "2222222222222222222222222222222222222222\trefs/tags/r8.2.7", + "3333333333333333333333333333333333333333\trefs/tags/r8.2.7^{}", + ] + ) + + self.assertEqual( + sync_repo_with_copybara.parse_remote_tag_commit(output, "r8.2.7"), + "3333333333333333333333333333333333333333", + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_tag_exists_remote_requires_exact_tag_match(self, mock_run_command): + mock_run_command.return_value = ( + "1111111111111111111111111111111111111111\trefs/tags/r8.2.70\n" + "2222222222222222222222222222222222222222\trefs/tags/r8.2.7-hotfix" + ) + + self.assertFalse( + sync_repo_with_copybara.tag_exists_remote("https://example.com/public.git", "r8.2.7") + ) + + def test_resolve_requested_release_tag_branches_creates_synthetic_fragment(self): + with tempfile.TemporaryDirectory() as tmpdir: + branch_to_fragment = {"master": Path("master.sky")} + + requested_branches, release_requests = ( + sync_repo_with_copybara.resolve_requested_release_tag_branches( + requested_branches="master, r8.2.7", + branch_to_fragment=branch_to_fragment, + bundle_dir=Path(tmpdir), + ) + ) + + self.assertEqual(requested_branches, "master,v8.2.7") + self.assertEqual( + release_requests, + { + "v8.2.7": sync_repo_with_copybara.ReleaseTagRequest( + release_tag="r8.2.7", + public_branch="v8.2.7", + ) + }, + ) + synthetic_fragment = branch_to_fragment["v8.2.7"] + self.assertTrue(synthetic_fragment.is_file()) + self.assertIn('sync_tag("r8.2.7")', synthetic_fragment.read_text()) + + def test_resolve_requested_release_tag_branches_preserves_suffix(self): + with tempfile.TemporaryDirectory() as tmpdir: + branch_to_fragment = {"master": Path("master.sky")} + + requested_branches, release_requests = ( + sync_repo_with_copybara.resolve_requested_release_tag_branches( + requested_branches="r8.2.7-hotfix", + branch_to_fragment=branch_to_fragment, + bundle_dir=Path(tmpdir), + ) + ) + + self.assertEqual(requested_branches, "v8.2.7-hotfix") + self.assertEqual( + release_requests, + { + "v8.2.7-hotfix": sync_repo_with_copybara.ReleaseTagRequest( + release_tag="r8.2.7-hotfix", + public_branch="v8.2.7-hotfix", + ) + }, + ) + self.assertIn( + 'sync_tag("r8.2.7-hotfix")', + branch_to_fragment["v8.2.7-hotfix"].read_text(), + ) + + def test_extract_release_tags_from_fragment_reads_sync_tag(self): + with tempfile.TemporaryDirectory() as tmpdir: + fragment_path = Path(tmpdir) / "release_tag.sky" + fragment_path.write_text('sync_tag("r8.2.7-hotfix")\n') + + self.assertEqual( + sync_repo_with_copybara.extract_release_tags_from_fragment(fragment_path), + ["r8.2.7-hotfix"], + ) + + def test_extract_branches_from_fragment_rejects_release_tag_as_branch(self): + with tempfile.TemporaryDirectory() as tmpdir: + fragment_path = Path(tmpdir) / "bad_branch.sky" + fragment_path.write_text('sync_branch("r8.2.7")\n') + + with self.assertRaises(SystemExit): + sync_repo_with_copybara.extract_branches_from_fragment(fragment_path) + + def test_extract_release_tags_from_fragment_rejects_branch_as_release_tag(self): + with tempfile.TemporaryDirectory() as tmpdir: + fragment_path = Path(tmpdir) / "bad_tag.sky" + fragment_path.write_text('sync_tag("v8.2")\n') + + with self.assertRaises(SystemExit): + sync_repo_with_copybara.extract_release_tags_from_fragment(fragment_path) + + +def write_base_copybara_config( + path: Path, + common_patterns: list[str] | None = None, + common_includes: list[str] | None = None, + source_url: str = sync_repo_with_copybara.SOURCE_REPO_URL, + prod_url: str = sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL, + test_url: str = sync_repo_with_copybara.TEST_REPO_URL, +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if common_patterns is None: + common_patterns = list(DEFAULT_COMMON_EXCLUDED_PATTERNS) + if common_includes is None: + common_includes = ["**"] + + include_entries = "\n".join(f' "{pattern}",' for pattern in common_includes) + exclude_entries = "\n".join(f' "{pattern}",' for pattern in common_patterns) + path.write_text( + f'source_url = "{source_url}"\n' + f'prod_url = "{prod_url}"\n' + f'test_url = "{test_url}"\n' + f'test_branch_prefix = "{sync_repo_with_copybara.DEFAULT_TEST_BRANCH_PREFIX}"\n' + f'test_workflow_base_branch = ""\n' + f'test_workflow_source_branch = ""\n' + f"source_refs = {{}}\n" + f"\n" + f"common_files_to_include = [\n" + f"{include_entries}\n" + f"]\n" + f"\n" + f"common_files_to_exclude = [\n" + f"{exclude_entries}\n" + f"]\n" + f"\n" + f"def make_workflow(\n" + f" workflow_name,\n" + f" destination_url,\n" + f" source_ref,\n" + f" destination_ref,\n" + f"):\n" + f" pass\n" + f"\n" + f"def sync_branch(branch_name):\n" + f" source_ref = source_refs.get(branch_name, branch_name)\n" + f' make_workflow("prod_" + branch_name, prod_url, source_ref, branch_name)\n' + f" make_workflow(\n" + f' "test_" + branch_name,\n' + f" test_url,\n" + f" source_ref,\n" + f' test_branch_prefix + "_" + branch_name,\n' + f" )\n" + ) + + class TestSkyExclusionChecks(unittest.TestCase): def test_extract_sky_excluded_patterns(self): with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - """ - origin_files = glob(["**"], exclude = [ - "src/mongo/db/modules/**", - "buildscripts/modules/**", - ".github/workflows/**", - "src/third_party/private/**", - "sbom.private.json", - ".agents/**", - ".cursor/**", - ".claude/**", - "AGENTS.md", - "CLAUDE.md", - ".github/CODEOWNERS", - "monguard/**", - "etc/evergreen_yml_components/**", - ]) - """ - ) - patterns = sync_repo_with_copybara.extract_sky_excluded_patterns(sky_path) + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + + patterns = sync_repo_with_copybara.extract_sky_excluded_patterns(str(sky_path)) + self.assertIn("src/mongo/db/modules/**", patterns) self.assertIn("AGENTS.md", patterns) - def test_check_script_exclusions_match_sky_passes(self): + def test_extract_sky_excluded_patterns_prefers_adjacent_path_rules(self): with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - """ - origin_files = glob(["**"], exclude = [ - "src/mongo/db/modules/**", - "buildscripts/modules/**", - ".github/workflows/**", - "src/third_party/private/**", - "sbom.private.json", - ".agents/**", - ".cursor/**", - ".claude/**", - "AGENTS.md", - "CLAUDE.md", - ".github/CODEOWNERS", - "monguard/**", - "etc/evergreen_yml_components/**", - ]) - """ - ) - sync_repo_with_copybara.check_script_exclusions_match_sky(sky_path) - - def test_check_script_exclusions_match_sky_fails(self): - with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - """ - origin_files = glob(["**"], exclude = [ - "src/mongo/db/modules/**", - "buildscripts/modules/**", - ]) - """ - ) - with self.assertRaises(SystemExit): - sync_repo_with_copybara.check_script_exclusions_match_sky(sky_path) - - def test_pin_prod_workflow_ref_to_commit(self): - with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - f'{sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "master"\n' - 'make_workflow("prod", prodUrl, prodRefForPinnedSourceCommit, "master")\n' - ) - sync_repo_with_copybara.pin_prod_workflow_ref_to_commit(sky_path, "abc123") - with open(sky_path, "r") as f: - rewritten = f.read() - self.assertIn( - f'{sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "abc123"', rewritten + sky_path = Path(tmpdir) / "copy.bara.sky" + sky_path.write_text('common_files_to_exclude = ["ignored/**"]\n') + write_copybara_path_rules( + sky_path.with_name("copybara_path_rules.json"), + common_includes=DEFAULT_TEST_COPYBARA_PATH_RULES_INCLUDES, + common_excludes=["src/authoritative/**"], ) - def test_pin_prod_workflow_ref_to_commit_reuses_existing_variable(self): + patterns = sync_repo_with_copybara.extract_sky_excluded_patterns(str(sky_path)) + + self.assertEqual(patterns, {"src/authoritative/**"}) + + def test_get_preview_excluded_patterns_includes_common_exclusions(self): with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - f""" - {sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "oldsha" - make_workflow("prod", prodUrl, {sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE}, "master") - """ - ) - sync_repo_with_copybara.pin_prod_workflow_ref_to_commit(sky_path, "newsha") - with open(sky_path, "r") as f: - rewritten = f.read() - self.assertEqual(rewritten.count(sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE), 2) - self.assertIn( - f'{sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "newsha"', - rewritten, + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_patterns=DEFAULT_COMMON_EXCLUDED_PATTERNS + ["docs/private-notes/**"], + ) + sky_path.write_text(sky_path.read_text() + '\nsync_branch("master")\n') + + patterns = sync_repo_with_copybara.get_preview_excluded_patterns( + str(sky_path), "master" ) - def test_pin_prod_workflow_ref_to_commit_fails_if_variable_missing(self): - with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write('make_workflow("prod", prodUrl, "master", "master")') - with self.assertRaises(SystemExit): - sync_repo_with_copybara.pin_prod_workflow_ref_to_commit(sky_path, "abc123") + self.assertIn("docs/private-notes/**", patterns) + self.assertIn("AGENTS.md", patterns) - def test_get_prod_pinned_source_ref(self): + def test_get_preview_excluded_patterns_includes_branch_specific_additions(self): with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - f'{sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "v8.0"\n' - 'make_workflow("prod", prodUrl, prodRefForPinnedSourceCommit, "master")\n' - ) + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + sky_path.write_text( + sky_path.read_text() + + '\nrelease_files_to_exclude = [\n "docs/private-notes/**",\n]\n' + + 'sync_branch("v8.2", release_files_to_exclude)\n' + ) - pinned_ref = sync_repo_with_copybara.get_prod_pinned_source_ref(sky_path) - self.assertEqual(pinned_ref, "v8.0") + patterns = sync_repo_with_copybara.get_preview_excluded_patterns(str(sky_path), "v8.2") - def test_get_prod_pinned_source_ref_fails_if_variable_missing(self): + self.assertIn("docs/private-notes/**", patterns) + self.assertIn("AGENTS.md", patterns) + + def test_get_preview_excluded_patterns_deduplicates_common_and_branch_exclusions(self): with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write('make_workflow("prod", prodUrl, "master", "master")') + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_patterns=["AGENTS.md", "internal/"], + ) + sky_path.write_text( + sky_path.read_text() + + '\nrelease_files_to_exclude = [\n "internal/",\n "private/**",\n]\n' + + 'sync_branch("v8.2", release_files_to_exclude)\n' + ) + + patterns = sync_repo_with_copybara.get_preview_excluded_patterns(str(sky_path), "v8.2") + + self.assertEqual(patterns, ["AGENTS.md", "internal/", "private/**"]) + + def test_extract_branch_public_patterns_defaults_to_all_files(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + sky_path.write_text(sky_path.read_text() + '\nsync_branch("master")\n') + + patterns = sync_repo_with_copybara.extract_branch_public_patterns( + str(sky_path), "master" + ) + + self.assertEqual(patterns, {"**"}) + + def test_extract_branch_public_patterns_from_common_list(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_includes=["README.md", "docs/**"], + ) + sky_path.write_text(sky_path.read_text() + '\nsync_branch("master")\n') + + patterns = sync_repo_with_copybara.extract_branch_public_patterns( + str(sky_path), "master" + ) + + self.assertEqual(patterns, {"README.md", "docs/**"}) + + def test_extract_branch_public_patterns_supports_sync_tag(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_includes=["README.md", "docs/**"], + ) + sky_path.write_text(sky_path.read_text() + '\nsync_tag("r8.2.7")\n') + + patterns = sync_repo_with_copybara.extract_branch_public_patterns( + str(sky_path), "v8.2.7" + ) + + self.assertEqual(patterns, {"README.md", "docs/**"}) + + def test_extract_branch_public_patterns_ignores_branch_exclusions_only(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_patterns=["AGENTS.md", "internal/**"], + common_includes=["src/", "buildscripts/", "jstests/**"], + ) + sky_path.write_text( + sky_path.read_text() + + '\nrelease_exclusions = [\n "private/**",\n "secrets/**",\n]\n' + + 'sync_branch("v8.2", release_exclusions)\n' + ) + + patterns = sync_repo_with_copybara.extract_branch_public_patterns(str(sky_path), "v8.2") + + self.assertEqual(patterns, {"src/", "buildscripts/", "jstests/**"}) + + def test_extract_branch_public_patterns_prefers_adjacent_path_rules(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + sky_path.write_text( + 'common_files_to_include = ["ignored/**"]\n' + '\nsync_branch("master")\n' + ) + write_copybara_path_rules( + sky_path.with_name("copybara_path_rules.json"), + common_excludes=DEFAULT_TEST_COPYBARA_PATH_RULES_EXCLUDES, + common_includes=["README.md", "docs/**"], + ) + + patterns = sync_repo_with_copybara.extract_branch_public_patterns( + str(sky_path), "master" + ) + + self.assertEqual(patterns, {"README.md", "docs/**"}) + + def test_extract_branch_public_patterns_from_named_list(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + sky_path.write_text( + sky_path.read_text() + + '\nmaster_public_files = [\n "README.md",\n "docs/**",\n]\n' + + 'sync_branch("master", [], master_public_files)\n' + ) + + patterns = sync_repo_with_copybara.extract_branch_public_patterns( + str(sky_path), "master" + ) + + self.assertEqual(patterns, {"README.md", "docs/**"}) + + def test_check_branch_top_level_paths_are_labeled_passes(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_includes=["README.md", "docs/**"], + ) + sky_path.write_text(sky_path.read_text() + '\nsync_branch("master")\n') + + sync_repo_with_copybara.check_branch_top_level_paths_are_labeled( + str(sky_path), + "master", + {"README.md", "docs", "monguard", "sbom.private.json"}, + ) + + def test_check_branch_top_level_paths_are_labeled_supports_sync_tag(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_includes=["README.md", "docs/**"], + ) + sky_path.write_text(sky_path.read_text() + '\nsync_tag("r8.2.7")\n') + + sync_repo_with_copybara.check_branch_top_level_paths_are_labeled( + str(sky_path), + "v8.2.7", + {"README.md", "docs", "monguard", "sbom.private.json"}, + ) + + def test_check_branch_top_level_paths_are_labeled_fails_for_unlabeled_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + sky_path, + common_includes=["README.md", "docs/**"], + ) + sky_path.write_text(sky_path.read_text() + '\nsync_branch("master")\n') with self.assertRaises(SystemExit): - sync_repo_with_copybara.get_prod_pinned_source_ref(sky_path) - - @patch("buildscripts.sync_repo_with_copybara.pin_prod_workflow_ref_to_commit") - @patch("buildscripts.sync_repo_with_copybara.run_command") - def test_get_prod_copybara_config_uses_ref_from_sky_file(self, mock_run_command, mock_pin_ref): - with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - f'{sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "v8.0"\n' - 'make_workflow("prod", prodUrl, prodRefForPinnedSourceCommit, "master")\n' + sync_repo_with_copybara.check_branch_top_level_paths_are_labeled( + str(sky_path), + "master", + {"README.md", "docs", "src"}, ) - mock_run_command.side_effect = [ - "", - "deadbeef123\n", - 'sourceUrl = "https://github.com/10gen/mongo.git"\n', - ] - config_file = sync_repo_with_copybara.get_prod_copybara_config_from_master(tmpdir) +class TestCopybaraConfigHelpers(unittest.TestCase): + def test_build_copybara_config_uses_test_branch_prefix(self): + config = sync_repo_with_copybara.build_copybara_config( + workflow="test", + branch="v8.2", + source_ref="deadbeef123", + test_branch_prefix="copybara_test_branch_patch123", + ) - self.assertEqual( - config_file, os.path.join(tmpdir, "tmp_copybara_config_from_master.sky") - ) - self.assertEqual(mock_run_command.call_args_list[0].args[0], "git fetch origin v8.0") - self.assertEqual( - mock_run_command.call_args_list[1].args[0], "git rev-parse origin/v8.0" - ) - self.assertEqual( - mock_run_command.call_args_list[2].args[0], - "git --no-pager show deadbeef123:copy.bara.sky", - ) - mock_pin_ref.assert_called_once_with(config_file, "deadbeef123") + self.assertEqual(config.source.branch, "v8.2") + self.assertEqual(config.source.ref, "deadbeef123") + self.assertEqual(config.destination.branch, "copybara_test_branch_patch123_v8.2") + self.assertEqual(config.destination.repo_name, sync_repo_with_copybara.TEST_REPO_NAME) - @patch("buildscripts.sync_repo_with_copybara.run_command") - def test_get_prod_copybara_config_keeps_destination_on_original_prod_ref( - self, mock_run_command + @patch("buildscripts.copybara.sync_repo_with_copybara.get_installation_access_token") + def test_get_copybara_tokens_uses_master_sync_public_app_expansion( + self, mock_get_installation_access_token ): - with tempfile.TemporaryDirectory() as tmpdir: - sky_path = os.path.join(tmpdir, "copy.bara.sky") - with open(sky_path, "w") as f: - f.write( - f'{sync_repo_with_copybara.PROD_PINNED_REF_VARIABLE} = "master"\n' - 'make_workflow("prod", prodUrl, prodRefForPinnedSourceCommit, "master")\n' - ) + expansions = { + "app_id_copybara_syncer_master_sync": "101", + "private_key_copybara_syncer": "public-private-key", + "installation_id_copybara_syncer": "202", + "app_id_copybara_syncer_10gen": "303", + "private_key_copybara_syncer_10gen": "private-private-key", + "installation_id_copybara_syncer_10gen": "404", + } + mock_get_installation_access_token.side_effect = ["public-token", "private-token"] - mock_run_command.side_effect = [ - "", - "deadbeef123\n", - ( - 'prodRefForPinnedSourceCommit = "master"\n' - 'make_workflow("prod", prodUrl, prodRefForPinnedSourceCommit, "master")\n' + tokens = sync_repo_with_copybara.get_copybara_tokens(expansions) + + self.assertEqual( + mock_get_installation_access_token.call_args_list, + [ + call(101, "public-private-key", 202), + call(303, "private-private-key", 404), + ], + ) + self.assertEqual( + tokens, + { + sync_repo_with_copybara.SOURCE_REPO_URL: "private-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "public-token", + sync_repo_with_copybara.TEST_REPO_URL: "private-token", + }, + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_list_copybara_fragment_paths_excludes_base_config(self, mock_run_command): + mock_run_command.return_value = "\n".join( + [ + sync_repo_with_copybara.COPYBARA_BASE_CONFIG_PATH.as_posix(), + sync_repo_with_copybara.COPYBARA_PATH_RULES_MODULE_PATH.as_posix(), + "buildscripts/copybara/master.sky", + "buildscripts/copybara/v8_2.sky", + ] + ) + + fragment_paths = sync_repo_with_copybara.list_copybara_fragment_paths( + Path("/tmp/fetch"), + sync_repo_with_copybara.COPYBARA_CONFIG_FETCH_REF, + ) + + self.assertEqual( + fragment_paths, + ["buildscripts/copybara/master.sky", "buildscripts/copybara/v8_2.sky"], + ) + + def test_discover_copybara_branches_reads_fragments(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + write_base_copybara_config(get_repo_base_copybara_config_path(root)) + write_copybara_path_rules( + get_repo_copybara_path_rules_path(root), + common_includes=DEFAULT_TEST_COPYBARA_PATH_RULES_INCLUDES, + common_excludes=DEFAULT_TEST_COPYBARA_PATH_RULES_EXCLUDES, + ) + write_copybara_path_rules_module( + get_repo_copybara_path_rules_module_path(root), + get_repo_copybara_path_rules_path(root), + ) + fragment_dir = root / "buildscripts" / "copybara" + fragment_dir.mkdir(parents=True, exist_ok=True) + (fragment_dir / "master.sky").write_text('sync_branch("master")\n') + (fragment_dir / "v8_2.sky").write_text( + 'sync_branch("v8.2")\n' 'sync_branch("v8.2.6-hotfix")\n' + ) + (fragment_dir / "v8_2_7_tag.sky").write_text('sync_tag("r8.2.7")\n') + + branch_to_fragment = sync_repo_with_copybara.discover_copybara_branches(tmpdir) + + self.assertEqual(branch_to_fragment["master"], fragment_dir / "master.sky") + self.assertEqual(branch_to_fragment["v8.2"], fragment_dir / "v8_2.sky") + self.assertEqual(branch_to_fragment["v8.2.6-hotfix"], fragment_dir / "v8_2.sky") + self.assertNotIn("v8.2.7", branch_to_fragment) + + def test_resolve_requested_branches_preserves_user_order(self): + branch_to_fragment = { + "master": Path("master.sky"), + "v8.0": Path("v8_0.sky"), + "v8.2": Path("v8_2.sky"), + } + + selected = sync_repo_with_copybara.resolve_requested_branches( + "v8.2, master, v8.2", branch_to_fragment + ) + + self.assertEqual(selected, ["v8.2", "master"]) + + def test_prepare_branch_sync_for_test_workflow_generates_branch_specific_config(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + base_config_path = get_repo_base_copybara_config_path(root) + write_base_copybara_config(base_config_path) + path_rules_path = get_repo_copybara_path_rules_path(root) + write_copybara_path_rules( + path_rules_path, + common_includes=DEFAULT_TEST_COPYBARA_PATH_RULES_INCLUDES, + common_excludes=DEFAULT_TEST_COPYBARA_PATH_RULES_EXCLUDES, + ) + path_rules_module_path = get_repo_copybara_path_rules_module_path(root) + write_copybara_path_rules_module(path_rules_module_path, path_rules_path) + fragment_dir = root / "buildscripts" / "copybara" + fragment_dir.mkdir(parents=True, exist_ok=True) + fragment_path = fragment_dir / "v8_2.sky" + fragment_path.write_text('sync_branch("v8.2")\n') + config_bundle = sync_repo_with_copybara.CopybaraConfigBundle( + config_sha="configsha123", + bundle_dir=root, + base_config_path=base_config_path, + path_rules_path=path_rules_path, + path_rules_module_path=path_rules_module_path, + branch_to_fragment={"v8.2": fragment_path}, + ) + test_baseline = sync_repo_with_copybara.TestWorkflowBaseline( + source_last_rev="feedface123", + destination_base_revision="publicdeadbeef456", + public_branch="v8.2", + ) + + stdout = io.StringIO() + with redirect_stdout(stdout): + prepared = sync_repo_with_copybara.prepare_branch_sync( + current_dir=tmpdir, + workflow="test", + branch="v8.2", + source_ref="deadbeef123", + config_bundle=config_bundle, + fragment_path=fragment_path, + tokens_map={ + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + }, + test_branch_prefix="copybara_test_branch_patch123", + test_baseline=test_baseline, + ) + log_output = stdout.getvalue() + + self.assertTrue(prepared.config_file.with_name("copybara_path_rules.json").is_file()) + self.assertTrue( + prepared.config_file.with_name("copybara_path_rules.bara.sky").is_file() + ) + + generated_config = prepared.config_file.read_text() + expected_test_branch = "copybara_test_branch_patch123_v8.2" + + self.assertIn('sync_branch("v8.2")', generated_config) + self.assertIn('test_branch_prefix = "copybara_test_branch_patch123"', generated_config) + self.assertIn(f'source_refs = {{"v8.2": "{expected_test_branch}"}}', generated_config) + self.assertEqual(prepared.source_ref, expected_test_branch) + self.assertEqual(prepared.config_sha, "configsha123") + self.assertEqual(prepared.workflow_name, "test_v8.2") + self.assertEqual(prepared.copybara_config.source.branch, expected_test_branch) + self.assertEqual(prepared.copybara_config.source.ref, expected_test_branch) + self.assertEqual(prepared.copybara_config.destination.branch, expected_test_branch) + self.assertEqual(prepared.last_rev, "feedface123") + self.assertEqual(prepared.test_baseline, test_baseline) + self.assertEqual( + prepared.dry_run_args, + ("--init-history", "--last-rev=feedface123"), + ) + self.assertIn(f"{prepared.config_file.parent}:/usr/src/app", prepared.docker_command) + self.assertNotIn( + f"{prepared.config_file}:/usr/src/app/copy.bara.sky", prepared.docker_command + ) + self.assertIn("[v8.2] BEGIN generated Copybara config:", log_output) + self.assertIn("[v8.2] BEGIN generated Copybara path rules module:", log_output) + self.assertIn( + 'source_url = "https://x-access-token:@github.com/10gen/mongo.git"', + log_output, + ) + self.assertIn("common_files_to_exclude = [", log_output) + self.assertIn('"src/mongo/db/modules/**"', log_output) + self.assertNotIn("source-token", log_output) + self.assertNotIn("prod-token", log_output) + self.assertNotIn("test-token", log_output) + + @patch("buildscripts.copybara.sync_repo_with_copybara.check_destination_branch_exists") + def test_prepare_branch_sync_for_prod_workflow_uses_configured_prod_destination( + self, mock_check_destination_branch_exists + ): + mock_check_destination_branch_exists.return_value = True + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + base_config_path = get_repo_base_copybara_config_path(root) + write_base_copybara_config( + base_config_path, + prod_url=sync_repo_with_copybara.TEST_REPO_URL, + ) + path_rules_path = get_repo_copybara_path_rules_path(root) + write_copybara_path_rules( + path_rules_path, + common_includes=DEFAULT_TEST_COPYBARA_PATH_RULES_INCLUDES, + common_excludes=DEFAULT_TEST_COPYBARA_PATH_RULES_EXCLUDES, + ) + path_rules_module_path = get_repo_copybara_path_rules_module_path(root) + write_copybara_path_rules_module(path_rules_module_path, path_rules_path) + fragment_dir = root / "buildscripts" / "copybara" + fragment_dir.mkdir(parents=True, exist_ok=True) + fragment_path = fragment_dir / "v8_2.sky" + fragment_path.write_text('sync_branch("v8.2")\n') + config_bundle = sync_repo_with_copybara.CopybaraConfigBundle( + config_sha="configsha123", + bundle_dir=root, + base_config_path=base_config_path, + path_rules_path=path_rules_path, + path_rules_module_path=path_rules_module_path, + branch_to_fragment={"v8.2": fragment_path}, + ) + + prepared = sync_repo_with_copybara.prepare_branch_sync( + current_dir=tmpdir, + workflow="prod", + branch="v8.2", + source_ref="deadbeef123", + config_bundle=config_bundle, + fragment_path=fragment_path, + tokens_map={ + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + }, + test_branch_prefix="copybara_test_branch_patch123", + ) + + self.assertEqual( + prepared.copybara_config.destination.git_url, + "https://x-access-token:test-token@github.com/10gen/mongo-copybara.git", + ) + self.assertEqual( + prepared.copybara_config.destination.repo_name, + sync_repo_with_copybara.TEST_REPO_NAME, + ) + mock_check_destination_branch_exists.assert_called_once_with(prepared.copybara_config) + + +class TestCopybaraConfigAndTestWorkflowHelpers(unittest.TestCase): + def test_get_test_workflow_base_branch_override_reads_sky_variable(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + sky_path.write_text( + sky_path.read_text().replace( + 'test_workflow_base_branch = ""', + 'test_workflow_base_branch = " v8.2 "', + ) + ) + + self.assertEqual( + sync_repo_with_copybara.get_test_workflow_base_branch_override(sky_path), + "v8.2", + ) + + def test_get_test_workflow_base_branch_override_defaults_to_none(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + + self.assertIsNone( + sync_repo_with_copybara.get_test_workflow_base_branch_override(sky_path) + ) + + def test_resolve_test_workflow_requested_branches_prefers_base_branch_override(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + sky_path.write_text( + sky_path.read_text().replace( + 'test_workflow_base_branch = ""', + 'test_workflow_base_branch = "v8.2"', + ) + ) + + requested, override = sync_repo_with_copybara.resolve_test_workflow_requested_branches( + "master", + sky_path, + ) + + self.assertEqual(requested, "v8.2") + self.assertEqual(override, "v8.2") + + def test_resolve_test_workflow_requested_branches_defaults_to_requested_branches(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + + requested, override = sync_repo_with_copybara.resolve_test_workflow_requested_branches( + "master", + sky_path, + ) + + self.assertEqual(requested, "master") + self.assertIsNone(override) + + def test_get_test_workflow_source_branch_override_reads_sky_variable(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + sky_path.write_text( + sky_path.read_text().replace( + 'test_workflow_source_branch = ""', + 'test_workflow_source_branch = " daniel.moody/8.3_test_branch "', + ) + ) + + self.assertEqual( + sync_repo_with_copybara.get_test_workflow_source_branch_override(sky_path), + "daniel.moody/8.3_test_branch", + ) + + def test_get_test_workflow_source_branch_override_defaults_to_none(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + + self.assertIsNone( + sync_repo_with_copybara.get_test_workflow_source_branch_override(sky_path) + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_list_untracked_paths_skips_untracked_directories(self, mock_run_command): + with tempfile.TemporaryDirectory() as tmpdir: + repo_dir = Path(tmpdir) + (repo_dir / "new.txt").write_text("new") + (repo_dir / "copybara").mkdir() + mock_run_command.return_value = "new.txt\0copybara\0" + + untracked_paths = sync_repo_with_copybara.list_untracked_paths(repo_dir) + + self.assertEqual(untracked_paths, [Path("new.txt")]) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_list_untracked_paths_keeps_symlinked_directories(self, mock_run_command): + if sys.platform == "win32": + self.skipTest("symlink permissions vary on Windows") + + with tempfile.TemporaryDirectory() as tmpdir: + repo_dir = Path(tmpdir) + (repo_dir / "target_dir").mkdir() + os.symlink("target_dir", repo_dir / "linked_dir") + mock_run_command.return_value = "linked_dir\0" + + untracked_paths = sync_repo_with_copybara.list_untracked_paths(repo_dir) + + self.assertEqual(untracked_paths, [Path("linked_dir")]) + + def test_filter_untracked_paths_for_test_source_skips_task_generated_paths(self): + filtered_paths = sync_repo_with_copybara.filter_untracked_paths_for_test_source( + [ + Path(".evergreen.yml"), + Path("tmp_copybara/config_bundle/copy.bara.sky"), + Path("new.txt"), + Path("buildscripts/new_copybara_fragment.sky"), + ] + ) + + self.assertEqual( + filtered_paths, + [ + Path("new.txt"), + Path("buildscripts/new_copybara_fragment.sky"), + ], + ) + + def test_copy_paths_into_repo_skips_directories(self): + with ( + tempfile.TemporaryDirectory() as source_tmpdir, + tempfile.TemporaryDirectory() as dest_tmpdir, + ): + source_dir = Path(source_tmpdir) + destination_dir = Path(dest_tmpdir) + + (source_dir / "tracked.txt").write_text("tracked") + (source_dir / "copybara").mkdir() + (source_dir / "copybara" / "README.md").write_text("nested repo contents") + + sync_repo_with_copybara.copy_paths_into_repo( + source_dir, + destination_dir, + [Path("copybara"), Path("tracked.txt")], + ) + + self.assertFalse((destination_dir / "copybara").exists()) + self.assertEqual((destination_dir / "tracked.txt").read_text(), "tracked") + + def test_copy_paths_into_repo_preserves_symlinks(self): + if sys.platform == "win32": + self.skipTest("symlink permissions vary on Windows") + + with ( + tempfile.TemporaryDirectory() as source_tmpdir, + tempfile.TemporaryDirectory() as dest_tmpdir, + ): + source_dir = Path(source_tmpdir) + destination_dir = Path(dest_tmpdir) + + (source_dir / "target.txt").write_text("target") + os.symlink("target.txt", source_dir / "linked.txt") + (destination_dir / "linked.txt").write_text("stale") + + sync_repo_with_copybara.copy_paths_into_repo( + source_dir, + destination_dir, + [Path("linked.txt")], + ) + + self.assertTrue((destination_dir / "linked.txt").is_symlink()) + self.assertEqual(os.readlink(destination_dir / "linked.txt"), "target.txt") + + def test_remove_paths_from_repo_removes_symlinks(self): + if sys.platform == "win32": + self.skipTest("symlink permissions vary on Windows") + + with tempfile.TemporaryDirectory() as tmpdir: + repo_dir = Path(tmpdir) + (repo_dir / "target.txt").write_text("target") + os.symlink("target.txt", repo_dir / "linked.txt") + + sync_repo_with_copybara.remove_paths_from_repo(repo_dir, [Path("linked.txt")]) + + self.assertFalse((repo_dir / "linked.txt").exists()) + self.assertFalse((repo_dir / "linked.txt").is_symlink()) + + @patch("buildscripts.copybara.sync_repo_with_copybara.tempfile.mkdtemp") + @patch("buildscripts.copybara.sync_repo_with_copybara.find_matching_commit_pair") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + def test_resolve_test_workflow_baseline_uses_public_branch_when_present( + self, + mock_branch_exists_remote, + mock_run_command, + mock_find_matching_commit_pair, + mock_mkdtemp, + ): + mock_branch_exists_remote.return_value = True + public_baseline_dir = Path("/tmp/public-baseline") + mock_mkdtemp.return_value = str(public_baseline_dir) + mock_find_matching_commit_pair.return_value = sync_repo_with_copybara.MatchingCommit( + source_commit="private123", + destination_commit="public456", + ) + + baseline = sync_repo_with_copybara.resolve_test_workflow_baseline( + current_dir="/repo", + branch="v8.2", + patch_base_revision="deadbeef123", + public_repo_url="https://example.com/public.git", + ) + + self.assertEqual( + baseline, + sync_repo_with_copybara.TestWorkflowBaseline( + source_last_rev="private123", + destination_base_revision="public456", + public_branch="v8.2", + ), + ) + mock_run_command.assert_called_once_with( + "git clone --filter=blob:none --no-checkout --single-branch -b v8.2 " + f"https://example.com/public.git {sync_repo_with_copybara.shell_quote(public_baseline_dir)}" + ) + mock_find_matching_commit_pair.assert_called_once_with( + "/repo", + str(public_baseline_dir), + source_ref="deadbeef123", + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.tempfile.mkdtemp") + @patch("buildscripts.copybara.sync_repo_with_copybara.find_matching_commit_pair") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + def test_resolve_test_workflow_baseline_uses_source_branch_override( + self, + mock_branch_exists_remote, + mock_run_command, + mock_find_matching_commit_pair, + mock_mkdtemp, + ): + mock_branch_exists_remote.return_value = True + public_baseline_dir = Path("/tmp/public-baseline") + private_source_dir = Path("/tmp/private-source") + mock_mkdtemp.side_effect = [str(public_baseline_dir), str(private_source_dir)] + mock_find_matching_commit_pair.return_value = sync_repo_with_copybara.MatchingCommit( + source_commit="private123", + destination_commit="public456", + ) + + baseline = sync_repo_with_copybara.resolve_test_workflow_baseline( + current_dir="/repo", + branch="v8.2", + patch_base_revision="deadbeef123", + public_repo_url="https://example.com/public.git", + source_repo_url="https://example.com/source.git", + test_source_branch="daniel.moody/8.3_test_branch", + ) + + self.assertEqual( + baseline, + sync_repo_with_copybara.TestWorkflowBaseline( + source_last_rev="private123", + destination_base_revision="public456", + public_branch="v8.2", + ), + ) + self.assertEqual( + mock_run_command.call_args_list[0].args[0], + "git clone --filter=blob:none --no-checkout --single-branch -b " + "daniel.moody/8.3_test_branch https://example.com/source.git " + f"{sync_repo_with_copybara.shell_quote(private_source_dir)}", + ) + self.assertEqual( + mock_run_command.call_args_list[1].args[0], + "git clone --filter=blob:none --no-checkout --single-branch -b v8.2 " + f"https://example.com/public.git {sync_repo_with_copybara.shell_quote(public_baseline_dir)}", + ) + mock_find_matching_commit_pair.assert_called_once_with( + str(private_source_dir), + str(public_baseline_dir), + source_ref="HEAD", + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.tempfile.mkdtemp") + @patch("buildscripts.copybara.sync_repo_with_copybara.find_matching_commit_pair") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + def test_resolve_test_workflow_baseline_falls_back_to_public_default_branch( + self, + mock_branch_exists_remote, + mock_run_command, + mock_find_matching_commit_pair, + mock_mkdtemp, + ): + mock_branch_exists_remote.return_value = False + public_baseline_dir = Path("/tmp/public-baseline") + mock_mkdtemp.return_value = str(public_baseline_dir) + mock_find_matching_commit_pair.return_value = sync_repo_with_copybara.MatchingCommit( + source_commit="private123", + destination_commit="public456", + ) + + baseline = sync_repo_with_copybara.resolve_test_workflow_baseline( + current_dir="/repo", + branch="v8.2-hotfix", + patch_base_revision="deadbeef123", + public_repo_url="https://example.com/public.git", + ) + + self.assertEqual(baseline.public_branch, sync_repo_with_copybara.COPYBARA_CONFIG_REF) + mock_run_command.assert_called_once_with( + "git clone --filter=blob:none --no-checkout --single-branch -b master " + f"https://example.com/public.git {sync_repo_with_copybara.shell_quote(public_baseline_dir)}" + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.remove_paths_from_repo") + @patch("buildscripts.copybara.sync_repo_with_copybara.copy_paths_into_repo") + @patch("buildscripts.copybara.sync_repo_with_copybara.list_untracked_paths") + @patch("buildscripts.copybara.sync_repo_with_copybara.list_changed_paths_between_refs") + @patch("buildscripts.copybara.sync_repo_with_copybara.tempfile.mkdtemp") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_create_patched_test_source_repo_rebuilds_from_base_revision( + self, + mock_run_command, + mock_mkdtemp, + mock_list_changed_paths_between_refs, + mock_list_untracked_paths, + mock_copy_paths_into_repo, + mock_remove_paths_from_repo, + ): + patched_source_dir = Path("/tmp/patched-source") + mock_mkdtemp.return_value = str(patched_source_dir) + mock_list_changed_paths_between_refs.return_value = ( + [Path("tracked.txt")], + [Path("deleted.txt")], + ) + mock_list_untracked_paths.return_value = [ + Path(".evergreen.yml"), + Path("tmp_copybara/config_bundle/copy.bara.sky"), + Path("new.txt"), + ] + mock_run_command.side_effect = ["", "", "", "M tracked.txt\n", ""] + + result = sync_repo_with_copybara.create_patched_test_source_repo( + "/repo", + "deadbeef123", + "patch123", + ) + + self.assertEqual(result, patched_source_dir) + mock_list_changed_paths_between_refs.assert_called_once_with("/repo", "deadbeef123") + mock_list_untracked_paths.assert_called_once_with("/repo") + mock_copy_paths_into_repo.assert_called_once_with( + "/repo", + patched_source_dir, + [Path("new.txt"), Path("tracked.txt")], + ) + mock_remove_paths_from_repo.assert_called_once_with( + patched_source_dir, + [Path("deleted.txt")], + ) + self.assertEqual( + mock_run_command.call_args_list[0].args[0], + "git clone --shared --no-checkout /repo " + f"{sync_repo_with_copybara.shell_quote(patched_source_dir)}", + ) + self.assertEqual( + mock_run_command.call_args_list[1].args[0], + "git -C " + f"{sync_repo_with_copybara.shell_quote(patched_source_dir)} " + "checkout --detach deadbeef123", + ) + for command_call in mock_run_command.call_args_list: + self.assertNotIn(" apply ", command_call.args[0]) + + @patch("buildscripts.copybara.sync_repo_with_copybara.tempfile.mkdtemp") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_create_test_source_repo_from_branch_clones_existing_branch( + self, + mock_run_command, + mock_mkdtemp, + ): + source_branch_dir = Path("/tmp/source-branch") + mock_mkdtemp.return_value = str(source_branch_dir) + + result = sync_repo_with_copybara.create_test_source_repo_from_branch( + "https://example.com/source.git", + "daniel.moody/8.3_test_branch", + ) + + self.assertEqual(result, source_branch_dir) + mock_run_command.assert_called_once_with( + "git clone --filter=blob:none --no-checkout --single-branch -b " + "daniel.moody/8.3_test_branch https://example.com/source.git " + f"{sync_repo_with_copybara.shell_quote(source_branch_dir)}" + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + @patch("buildscripts.copybara.sync_repo_with_copybara.push_test_destination_branch") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + @patch("buildscripts.copybara.sync_repo_with_copybara.create_patched_test_source_repo") + @patch("buildscripts.copybara.sync_repo_with_copybara.is_current_repo_origin") + def test_push_test_branches_pushes_clean_destination_and_patched_source( + self, + mock_is_current_repo_origin, + mock_create_patched_test_source_repo, + mock_branch_exists_remote, + mock_push_test_destination_branch, + mock_run_command, + ): + mock_is_current_repo_origin.return_value = True + patched_source_dir = Path("/tmp/patched-source") + mock_create_patched_test_source_repo.return_value = patched_source_dir + mock_branch_exists_remote.return_value = True + + test_branch = "copybara_test_branch_patch123_v8.2" + source_url = "https://example.com/source.git" + destination_url = "https://example.com/destination.git" + sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2", + source_ref=test_branch, + config_sha="local", + workflow_name="test_v8.2", + config_file=Path("/tmp/copy.bara.sky"), + preview_dir=Path("/tmp/preview"), + docker_command=("echo",), + copybara_config=sync_repo_with_copybara.CopybaraConfig( + source=sync_repo_with_copybara.CopybaraRepoConfig( + git_url=source_url, + repo_name=sync_repo_with_copybara.SOURCE_REPO_NAME, + branch=test_branch, + ref=test_branch, + ), + destination=sync_repo_with_copybara.CopybaraRepoConfig( + git_url=destination_url, + repo_name=sync_repo_with_copybara.TEST_REPO_NAME, + branch=test_branch, + ), + ), + last_rev="privatebase123", + test_baseline=sync_repo_with_copybara.TestWorkflowBaseline( + source_last_rev="privatebase123", + destination_base_revision="publicbase456", + public_branch="v8.2", + ), + ) + + sync_repo_with_copybara.push_test_branches( + "/repo", + [sync], + "patch123", + patch_base_revision="deadbeef123", + public_repo_url="https://example.com/public.git", + source_repo_url=source_url, + ) + + mock_create_patched_test_source_repo.assert_called_once_with( + "/repo", + "deadbeef123", + "patch123", + ) + mock_push_test_destination_branch.assert_called_once_with( + public_repo_url="https://example.com/public.git", + public_branch="v8.2", + destination_url=destination_url, + destination_branch=test_branch, + destination_base_revision="publicbase456", + ) + mock_run_command.assert_has_calls( + [ + call( + "git push " + f"{sync_repo_with_copybara.shell_quote(source_url)} --delete " + f"{sync_repo_with_copybara.shell_quote(test_branch)}" + ), + call( + "git push " + f"{sync_repo_with_copybara.shell_quote(destination_url)} --delete " + f"{sync_repo_with_copybara.shell_quote(test_branch)}" + ), + call( + f"git -C {sync_repo_with_copybara.shell_quote(patched_source_dir)} push " + f"{sync_repo_with_copybara.shell_quote(source_url)} " + f"{sync_repo_with_copybara.shell_quote(f'HEAD:refs/heads/{test_branch}')}" ), ] + ) - config_file = sync_repo_with_copybara.get_prod_copybara_config_from_master(tmpdir) + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + @patch("buildscripts.copybara.sync_repo_with_copybara.push_test_destination_branch") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + @patch("buildscripts.copybara.sync_repo_with_copybara.create_test_source_repo_from_branch") + @patch("buildscripts.copybara.sync_repo_with_copybara.create_patched_test_source_repo") + @patch("buildscripts.copybara.sync_repo_with_copybara.is_current_repo_origin") + def test_push_test_branches_uses_source_branch_override( + self, + mock_is_current_repo_origin, + mock_create_patched_test_source_repo, + mock_create_test_source_repo_from_branch, + mock_branch_exists_remote, + mock_push_test_destination_branch, + mock_run_command, + ): + mock_is_current_repo_origin.return_value = True + source_branch_dir = Path("/tmp/source-branch") + mock_create_test_source_repo_from_branch.return_value = source_branch_dir + mock_branch_exists_remote.return_value = True - with open(config_file, "r") as f: - generated_config = f.read() + test_branch = "copybara_test_branch_patch123_v8.2" + source_url = "https://example.com/source.git" + destination_url = "https://example.com/destination.git" + sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2", + source_ref=test_branch, + config_sha="local", + workflow_name="test_v8.2", + config_file=Path("/tmp/copy.bara.sky"), + preview_dir=Path("/tmp/preview"), + docker_command=("echo",), + copybara_config=sync_repo_with_copybara.CopybaraConfig( + source=sync_repo_with_copybara.CopybaraRepoConfig( + git_url=source_url, + repo_name=sync_repo_with_copybara.SOURCE_REPO_NAME, + branch=test_branch, + ref=test_branch, + ), + destination=sync_repo_with_copybara.CopybaraRepoConfig( + git_url=destination_url, + repo_name=sync_repo_with_copybara.TEST_REPO_NAME, + branch=test_branch, + ), + ), + last_rev="privatebase123", + test_baseline=sync_repo_with_copybara.TestWorkflowBaseline( + source_last_rev="privatebase123", + destination_base_revision="publicbase456", + public_branch="v8.2", + ), + ) + sync_repo_with_copybara.push_test_branches( + "/repo", + [sync], + "patch123", + patch_base_revision="deadbeef123", + public_repo_url="https://example.com/public.git", + source_repo_url=source_url, + test_source_branch="daniel.moody/8.3_test_branch", + ) + + mock_create_patched_test_source_repo.assert_not_called() + mock_create_test_source_repo_from_branch.assert_called_once_with( + source_url, + "daniel.moody/8.3_test_branch", + ) + mock_push_test_destination_branch.assert_called_once() + self.assertEqual( + mock_run_command.call_args_list[-1].args[0], + f"git -C {sync_repo_with_copybara.shell_quote(source_branch_dir)} push " + f"{sync_repo_with_copybara.shell_quote(source_url)} " + f"{sync_repo_with_copybara.shell_quote(f'HEAD:refs/heads/{test_branch}')}", + ) + + +class TestMainWorkflow(unittest.TestCase): + @patch( + "buildscripts.copybara.sync_repo_with_copybara.ensure_generated_copybara_evergreen_is_current" + ) + @patch("buildscripts.copybara.sync_repo_with_copybara.handle_failure") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_branch_migrate") + @patch("buildscripts.copybara.sync_repo_with_copybara.rewrite_copybara_config") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_branch_dry_run") + @patch("buildscripts.copybara.sync_repo_with_copybara.push_test_branches") + @patch("buildscripts.copybara.sync_repo_with_copybara.prepare_branch_sync") + @patch("buildscripts.copybara.sync_repo_with_copybara.resolve_test_workflow_baseline") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_test_workflow_source_branch_override") + @patch("buildscripts.copybara.sync_repo_with_copybara.resolve_test_workflow_requested_branches") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_local_copybara_config_bundle") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_test_workflow_base_revision") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_copybara_tokens") + @patch("buildscripts.copybara.sync_repo_with_copybara.create_mongodb_bot_gitconfig") + @patch("buildscripts.copybara.sync_repo_with_copybara.ensure_copybara_checkout_and_image") + @patch("buildscripts.copybara.sync_repo_with_copybara.read_config_file") + @patch("buildscripts.copybara.sync_repo_with_copybara.os.getcwd") + def test_main_runs_test_migration_after_successful_dry_run( + self, + mock_getcwd, + mock_read_config_file, + mock_ensure_copybara_checkout_and_image, + mock_create_mongodb_bot_gitconfig, + mock_get_copybara_tokens, + mock_get_test_workflow_base_revision, + mock_get_local_copybara_config_bundle, + mock_resolve_test_workflow_requested_branches, + mock_get_test_workflow_source_branch_override, + mock_resolve_test_workflow_baseline, + mock_prepare_branch_sync, + mock_push_test_branches, + mock_run_branch_dry_run, + mock_rewrite_copybara_config, + mock_run_branch_migrate, + mock_handle_failure, + mock_ensure_generated_copybara_evergreen_is_current, + ): + mock_read_config_file.return_value = { + "project": sync_repo_with_copybara.EXPECTED_EVERGREEN_PROJECT, + "version_id": "patch123", + } + mock_getcwd.return_value = "/repo" + tokens = { + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + } + mock_get_copybara_tokens.side_effect = [tokens, tokens] + mock_get_test_workflow_base_revision.return_value = "base123" + mock_resolve_test_workflow_requested_branches.return_value = ("v8.2", "v8.2") + mock_get_test_workflow_source_branch_override.return_value = "daniel.moody/v8.2_test_branch" + test_baseline = sync_repo_with_copybara.TestWorkflowBaseline( + source_last_rev="private123", + destination_base_revision="public456", + public_branch="v8.2", + ) + mock_resolve_test_workflow_baseline.return_value = test_baseline + + with tempfile.TemporaryDirectory() as tmpdir: + base_config_path = Path(tmpdir) / "copy.bara.sky" + fragment_path = Path(tmpdir) / "v8_2.sky" + write_base_copybara_config(base_config_path) + fragment_path.write_text('sync_branch("v8.2")\n') + mock_get_local_copybara_config_bundle.return_value = ( + sync_repo_with_copybara.CopybaraConfigBundle( + config_sha="local", + bundle_dir=Path(tmpdir), + base_config_path=base_config_path, + path_rules_path=Path(tmpdir) / "copybara_path_rules.json", + path_rules_module_path=Path(tmpdir) / "copybara_path_rules.bara.sky", + branch_to_fragment={"v8.2": fragment_path}, + ) + ) + prepared_sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2", + source_ref="copybara_test_branch_patch123_v8.2", + config_sha="local", + workflow_name="test_v8.2", + config_file=Path(tmpdir) / "generated.sky", + preview_dir=Path(tmpdir) / "preview", + docker_command=("echo",), + copybara_config=sync_repo_with_copybara.CopybaraConfig( + source=sync_repo_with_copybara.CopybaraRepoConfig( + git_url="https://example.com/source.git", + repo_name=sync_repo_with_copybara.SOURCE_REPO_NAME, + branch="copybara_test_branch_patch123_v8.2", + ref="copybara_test_branch_patch123_v8.2", + ), + destination=sync_repo_with_copybara.CopybaraRepoConfig( + git_url="https://example.com/destination.git", + repo_name=sync_repo_with_copybara.TEST_REPO_NAME, + branch="copybara_test_branch_patch123_v8.2", + ), + ), + last_rev="private123", + test_baseline=test_baseline, + ) + mock_prepare_branch_sync.return_value = prepared_sync + mock_run_branch_dry_run.return_value = sync_repo_with_copybara.BranchDryRunResult( + branch="v8.2" + ) + + argv = ["buildscripts/copybara/sync_repo_with_copybara.py", "--workflow=test"] + with patch.object(sys, "argv", argv): + sync_repo_with_copybara.main() + + mock_prepare_branch_sync.assert_called_once_with( + current_dir="/repo", + workflow="test", + branch="v8.2", + source_ref="private123", + config_bundle=mock_get_local_copybara_config_bundle.return_value, + fragment_path=fragment_path, + tokens_map=tokens, + test_branch_prefix="copybara_test_branch_patch123", + test_baseline=test_baseline, + expansions=mock_read_config_file.return_value, + release_tag=None, + release_source_commit=None, + ) + mock_push_test_branches.assert_called_once() + mock_ensure_generated_copybara_evergreen_is_current.assert_called_once_with(Path(tmpdir)) + mock_run_branch_dry_run.assert_called_once_with(prepared_sync) + mock_rewrite_copybara_config.assert_called_once_with(prepared_sync.config_file, tokens) + mock_run_branch_migrate.assert_called_once_with(prepared_sync) + mock_handle_failure.assert_not_called() + + @patch( + "buildscripts.copybara.sync_repo_with_copybara.ensure_generated_copybara_evergreen_is_current" + ) + @patch("buildscripts.copybara.sync_repo_with_copybara.publish_release_tag") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_branch_migrate") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_branch_dry_run") + @patch("buildscripts.copybara.sync_repo_with_copybara.prepare_branch_sync") + @patch("buildscripts.copybara.sync_repo_with_copybara.activate_new_hotfix_tasks") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_remote_tag_commit") + @patch("buildscripts.copybara.sync_repo_with_copybara.tag_exists_remote") + @patch("buildscripts.copybara.sync_repo_with_copybara.fetch_remote_copybara_config_bundle") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_copybara_tokens") + @patch("buildscripts.copybara.sync_repo_with_copybara.create_mongodb_bot_gitconfig") + @patch("buildscripts.copybara.sync_repo_with_copybara.ensure_copybara_checkout_and_image") + @patch("buildscripts.copybara.sync_repo_with_copybara.read_config_file") + @patch("buildscripts.copybara.sync_repo_with_copybara.os.getcwd") + def test_main_runs_prod_release_tag_sync_and_publishes_tag( + self, + mock_getcwd, + mock_read_config_file, + mock_ensure_copybara_checkout_and_image, + mock_create_mongodb_bot_gitconfig, + mock_get_copybara_tokens, + mock_fetch_remote_copybara_config_bundle, + mock_tag_exists_remote, + mock_get_remote_tag_commit, + mock_activate_new_hotfix_tasks, + mock_prepare_branch_sync, + mock_run_branch_dry_run, + mock_run_branch_migrate, + mock_publish_release_tag, + mock_ensure_generated_copybara_evergreen_is_current, + ): + mock_read_config_file.return_value = { + "project": sync_repo_with_copybara.EXPECTED_EVERGREEN_PROJECT, + } + mock_getcwd.return_value = "/repo" + tokens = { + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + } + mock_get_copybara_tokens.side_effect = [tokens, tokens] + mock_tag_exists_remote.return_value = False + mock_get_remote_tag_commit.return_value = "tagsha123" + synthetic_fragment_contents = "" + + with tempfile.TemporaryDirectory() as tmpdir: + base_config_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + base_config_path, + prod_url=sync_repo_with_copybara.TEST_REPO_URL, + ) + fragment_path = Path(tmpdir) / "master.sky" + fragment_path.write_text('sync_branch("master")\n') + mock_fetch_remote_copybara_config_bundle.return_value = ( + sync_repo_with_copybara.CopybaraConfigBundle( + config_sha="configsha123", + bundle_dir=Path(tmpdir), + base_config_path=base_config_path, + path_rules_path=Path(tmpdir) / "copybara_path_rules.json", + path_rules_module_path=Path(tmpdir) / "copybara_path_rules.bara.sky", + branch_to_fragment={"master": fragment_path}, + ) + ) + prepared_sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2.7", + source_ref="tagsha123", + config_sha="configsha123", + workflow_name="prod_r8.2.7", + config_file=Path("/tmp/generated.sky"), + preview_dir=Path("/tmp/preview"), + docker_command=("echo",), + release_tag="r8.2.7", + release_source_commit="tagsha123", + ) + mock_prepare_branch_sync.return_value = prepared_sync + mock_run_branch_dry_run.return_value = sync_repo_with_copybara.BranchDryRunResult( + branch="v8.2.7", + noop=True, + ) + + argv = [ + "buildscripts/copybara/sync_repo_with_copybara.py", + "--workflow=prod", + "--branches=r8.2.7", + ] + with patch.object(sys, "argv", argv): + sync_repo_with_copybara.main() + synthetic_fragment_contents = mock_prepare_branch_sync.call_args.kwargs[ + "fragment_path" + ].read_text() + + prepare_call = mock_prepare_branch_sync.call_args.kwargs + self.assertEqual(prepare_call["branch"], "v8.2.7") + self.assertEqual(prepare_call["source_ref"], "tagsha123") + self.assertEqual(prepare_call["release_tag"], "r8.2.7") + self.assertEqual(prepare_call["release_source_commit"], "tagsha123") + self.assertIn('sync_tag("r8.2.7")', synthetic_fragment_contents) + mock_ensure_generated_copybara_evergreen_is_current.assert_called_once_with(Path(tmpdir)) + mock_run_branch_migrate.assert_not_called() + mock_publish_release_tag.assert_called_once_with(prepared_sync, tokens) + mock_activate_new_hotfix_tasks.assert_called_once() + mock_tag_exists_remote.assert_called_once_with( + "https://x-access-token:test-token@github.com/10gen/mongo-copybara.git", + "r8.2.7", + ) + + @patch( + "buildscripts.copybara.sync_repo_with_copybara.ensure_generated_copybara_evergreen_is_current" + ) + @patch("buildscripts.copybara.sync_repo_with_copybara.prepare_branch_sync") + @patch("buildscripts.copybara.sync_repo_with_copybara.tag_exists_remote") + @patch("buildscripts.copybara.sync_repo_with_copybara.fetch_remote_copybara_config_bundle") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_copybara_tokens") + @patch("buildscripts.copybara.sync_repo_with_copybara.create_mongodb_bot_gitconfig") + @patch("buildscripts.copybara.sync_repo_with_copybara.ensure_copybara_checkout_and_image") + @patch("buildscripts.copybara.sync_repo_with_copybara.read_config_file") + @patch("buildscripts.copybara.sync_repo_with_copybara.os.getcwd") + def test_main_rejects_existing_public_release_tag( + self, + mock_getcwd, + mock_read_config_file, + mock_ensure_copybara_checkout_and_image, + mock_create_mongodb_bot_gitconfig, + mock_get_copybara_tokens, + mock_fetch_remote_copybara_config_bundle, + mock_tag_exists_remote, + mock_prepare_branch_sync, + mock_ensure_generated_copybara_evergreen_is_current, + ): + mock_read_config_file.return_value = { + "project": sync_repo_with_copybara.EXPECTED_EVERGREEN_PROJECT, + } + mock_getcwd.return_value = "/repo" + tokens = { + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + } + mock_get_copybara_tokens.return_value = tokens + mock_tag_exists_remote.return_value = True + + with tempfile.TemporaryDirectory() as tmpdir: + base_config_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + base_config_path, + prod_url=sync_repo_with_copybara.TEST_REPO_URL, + ) + fragment_path = Path(tmpdir) / "master.sky" + fragment_path.write_text('sync_branch("master")\n') + mock_fetch_remote_copybara_config_bundle.return_value = ( + sync_repo_with_copybara.CopybaraConfigBundle( + config_sha="configsha123", + bundle_dir=Path(tmpdir), + base_config_path=base_config_path, + path_rules_path=Path(tmpdir) / "copybara_path_rules.json", + path_rules_module_path=Path(tmpdir) / "copybara_path_rules.bara.sky", + branch_to_fragment={"master": fragment_path}, + ) + ) + + argv = [ + "buildscripts/copybara/sync_repo_with_copybara.py", + "--workflow=prod", + "--branches=r8.2.7", + ] + with patch.object(sys, "argv", argv): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.main() + + mock_prepare_branch_sync.assert_not_called() + mock_ensure_generated_copybara_evergreen_is_current.assert_called_once_with(Path(tmpdir)) + mock_tag_exists_remote.assert_called_once_with( + "https://x-access-token:test-token@github.com/10gen/mongo-copybara.git", + "r8.2.7", + ) + + def test_rewrite_copybara_config_refreshes_existing_tokens_and_prefix(self): + with tempfile.TemporaryDirectory() as tmpdir: + config_path = Path(tmpdir) / "copy.bara.sky" + config_path.write_text( + 'source_url = "https://x-access-token:old@github.com/10gen/mongo.git"\n' + 'prod_url = "https://github.com/mongodb/mongo.git"\n' + 'test_url = "https://x-access-token:old@github.com/10gen/mongo-copybara.git"\n' + 'test_branch_prefix = "copybara_test_branch"\n' + "\n" + "def make_workflow(workflow_name, destination_url, source_ref, destination_ref, branch_excluded_files):\n" + " pass\n" + "\n" + "def sync_branch(branch_name, branch_excluded_files = []):\n" + ' make_workflow("prod_" + branch_name, prod_url, branch_name, branch_name, branch_excluded_files)\n' + " make_workflow(\n" + ' "test_" + branch_name,\n' + " test_url,\n" + " branch_name,\n" + ' test_branch_prefix + "_" + branch_name,\n' + " branch_excluded_files,\n" + " )\n" + ) + + sync_repo_with_copybara.rewrite_copybara_config( + config_file=config_path, + tokens_map={ + sync_repo_with_copybara.SOURCE_REPO_URL: "new-source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "new-prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "new-test-token", + }, + test_branch_prefix="copybara_test_branch_patch456", + source_refs={"v8.2": "deadbeef123"}, + ) + + rewritten = config_path.read_text() self.assertIn( - 'prodRefForPinnedSourceCommit = "deadbeef123"', - generated_config, + "https://x-access-token:new-source-token@github.com/10gen/mongo.git", + rewritten, ) self.assertIn( - 'make_workflow("prod", prodUrl, prodRefForPinnedSourceCommit, "master")', - generated_config, + "https://x-access-token:new-prod-token@github.com/mongodb/mongo.git", + rewritten, ) + self.assertIn( + "https://x-access-token:new-test-token@github.com/10gen/mongo-copybara.git", + rewritten, + ) + self.assertIn('test_branch_prefix = "copybara_test_branch_patch456"', rewritten) + self.assertIn('source_refs = {"v8.2": "deadbeef123"}', rewritten) + self.assertIn("source_ref = source_refs.get(branch_name, branch_name)", rewritten) + + @patch("buildscripts.copybara.sync_repo_with_copybara.validate_preview_exclusions") + @patch("buildscripts.copybara.sync_repo_with_copybara.validate_sync_config") + @patch("buildscripts.copybara.sync_repo_with_copybara.rewrite_copybara_config") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_copybara_tokens") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_run_branch_dry_run_refreshes_tokens_on_auth_failure( + self, + mock_run_command, + mock_get_copybara_tokens, + mock_rewrite_copybara_config, + mock_validate_sync_config, + mock_validate_preview_exclusions, + ): + with tempfile.TemporaryDirectory() as tmpdir: + preview_dir = Path(tmpdir) / "preview" + preview_dir.mkdir() + stale_file = preview_dir / "stale.txt" + stale_file.write_text("stale") + sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2", + source_ref="deadbeef123", + config_sha="local", + workflow_name="prod_v8.2", + config_file=Path(tmpdir) / "copy.bara.sky", + preview_dir=preview_dir, + docker_command=("echo", "copybara"), + expansions={"version_id": "patch123"}, + ) + auth_error = subprocess.CalledProcessError( + 128, + "copybara", + output=( + "remote: Invalid username or token. Password authentication is not supported " + "for Git operations.\n" + "fatal: Authentication failed for 'https://github.com/10gen/mongo-copybara.git/'" + ), + ) + mock_run_command.side_effect = [auth_error, ""] + refreshed_tokens = { + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + } + mock_get_copybara_tokens.return_value = refreshed_tokens + + result = sync_repo_with_copybara.run_branch_dry_run(sync) + + self.assertEqual(result, sync_repo_with_copybara.BranchDryRunResult(branch="v8.2")) + self.assertFalse(stale_file.exists()) + self.assertEqual(mock_run_command.call_count, 2) + mock_get_copybara_tokens.assert_called_once_with(sync.expansions) + mock_rewrite_copybara_config.assert_called_once_with(sync.config_file, refreshed_tokens) + mock_validate_sync_config.assert_called() + mock_validate_preview_exclusions.assert_called_once_with(sync) + + @patch("buildscripts.copybara.sync_repo_with_copybara.rewrite_copybara_config") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_copybara_tokens") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_run_branch_migrate_refreshes_tokens_on_auth_failure( + self, + mock_run_command, + mock_get_copybara_tokens, + mock_rewrite_copybara_config, + ): + sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2", + source_ref="deadbeef123", + config_sha="local", + workflow_name="prod_v8.2", + config_file=Path("/tmp/copy.bara.sky"), + preview_dir=Path("/tmp/preview"), + docker_command=("echo", "copybara"), + expansions={"version_id": "patch123"}, + ) + auth_error = subprocess.CalledProcessError( + 128, + "copybara", + output=( + "remote: Invalid username or token. Password authentication is not supported " + "for Git operations.\n" + "fatal: Authentication failed for 'https://github.com/10gen/mongo-copybara.git/'" + ), + ) + mock_run_command.side_effect = [auth_error, ""] + refreshed_tokens = { + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + } + mock_get_copybara_tokens.return_value = refreshed_tokens + + sync_repo_with_copybara.run_branch_migrate(sync) + + self.assertEqual(mock_run_command.call_count, 2) + mock_get_copybara_tokens.assert_called_once_with(sync.expansions) + mock_rewrite_copybara_config.assert_called_once_with(sync.config_file, refreshed_tokens) + + @patch("buildscripts.copybara.sync_repo_with_copybara.get_remote_branch_head") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_fetch_remote_copybara_config_bundle_reads_latest_master_bundle( + self, mock_run_command, mock_get_remote_branch_head + ): + mock_get_remote_branch_head.return_value = "configsha123" + local_runner_contents = Path(sync_repo_with_copybara.__file__).resolve().read_text() + local_path_rules_contents = ( + Path(sync_repo_with_copybara.__file__).resolve().with_name("path_rules.py").read_text() + ) + template_text = ( + "rendered_from_template = True\n" + "common_files_to_include = [\n" + "{{COMMON_FILES_TO_INCLUDE}}\n" + "]\n" + "\n" + "common_files_to_exclude = [\n" + "{{COMMON_FILES_TO_EXCLUDE}}\n" + "]\n" + ) + + def run_command_side_effect(command): + if command.startswith("git init "): + return "" + if "fetch --depth 1" in command: + return "" + if f"rev-parse {sync_repo_with_copybara.COPYBARA_CONFIG_FETCH_REF}" in command: + return "configsha123\n" + if sync_repo_with_copybara.COPYBARA_SYNC_RUNNER_PATH.as_posix() in command: + return local_runner_contents + if sync_repo_with_copybara.COPYBARA_PATH_RULES_HELPER_PATH.as_posix() in command: + return local_path_rules_contents + if "ls-tree -r --name-only" in command: + return "\n".join( + [ + sync_repo_with_copybara.COPYBARA_BASE_CONFIG_PATH.as_posix(), + "buildscripts/copybara/master.sky", + "buildscripts/copybara/v8_2.sky", + ] + ) + if sync_repo_with_copybara.COPYBARA_BASE_CONFIG_PATH.as_posix() in command: + return ( + 'source_url = "https://github.com/10gen/mongo.git"\n' + 'prod_url = "https://github.com/mongodb/mongo.git"\n' + 'test_url = "https://github.com/10gen/mongo-copybara.git"\n' + 'test_branch_prefix = "copybara_test_branch"\n' + "source_refs = {}\n" + "\n" + "def make_workflow(workflow_name, destination_url, source_ref, destination_ref, branch_excluded_files):\n" + " pass\n" + "\n" + "def sync_branch(branch_name, branch_excluded_files = []):\n" + ' make_workflow("prod_" + branch_name, prod_url, branch_name, branch_name, branch_excluded_files)\n' + " make_workflow(\n" + ' "test_" + branch_name,\n' + " test_url,\n" + " branch_name,\n" + ' test_branch_prefix + "_" + branch_name,\n' + " branch_excluded_files,\n" + " )\n" + ) + if sync_repo_with_copybara.COPYBARA_PATH_RULES_PATH.as_posix() in command: + return json.dumps( + { + "common_files_to_include": ["**"], + "common_files_to_exclude": ["src/mongo/db/modules/**"], + } + ) + if sync_repo_with_copybara.COPYBARA_PATH_RULES_TEMPLATE_PATH.as_posix() in command: + return template_text + if ( + sync_repo_with_copybara.COPYBARA_GENERATED_EVERGREEN_CONFIG_PATH.as_posix() + in command + ): + return "generated Copybara Evergreen YAML\n" + if "buildscripts/copybara/master.sky" in command: + return 'sync_branch("master")\n' + if "buildscripts/copybara/v8_2.sky" in command: + return 'sync_branch("v8.2")\n' + raise AssertionError(f"Unexpected command: {command}") + + mock_run_command.side_effect = run_command_side_effect + + with tempfile.TemporaryDirectory() as tmpdir: + bundle = sync_repo_with_copybara.fetch_remote_copybara_config_bundle( + tmpdir, + "https://x-access-token:token@github.com/10gen/mongo.git", + ) + + self.assertEqual(bundle.config_sha, "configsha123") + self.assertEqual( + bundle.base_config_path.read_text().splitlines()[0], + 'source_url = "https://github.com/10gen/mongo.git"', + ) + self.assertEqual( + bundle.path_rules_path.read_text(), + json.dumps( + { + "common_files_to_include": ["**"], + "common_files_to_exclude": ["src/mongo/db/modules/**"], + } + ), + ) + self.assertIn( + "rendered_from_template = True", + bundle.path_rules_module_path.read_text(), + ) + generated_evergreen_config_path = ( + bundle.bundle_dir / sync_repo_with_copybara.COPYBARA_GENERATED_EVERGREEN_CONFIG_PATH + ) + self.assertEqual( + generated_evergreen_config_path.read_text(), + "generated Copybara Evergreen YAML\n", + ) + self.assertIn('"src/mongo/db/modules/**"', bundle.path_rules_module_path.read_text()) + self.assertIn("master", bundle.branch_to_fragment) + self.assertIn("v8.2", bundle.branch_to_fragment) + + @patch("buildscripts.copybara.sync_repo_with_copybara.get_remote_branch_head") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_fetch_remote_copybara_config_bundle_fails_for_stale_checked_out_runner( + self, mock_run_command, mock_get_remote_branch_head + ): + mock_get_remote_branch_head.return_value = "configsha123" + + def run_command_side_effect(command): + if command.startswith("git init "): + return "" + if "fetch --depth 1" in command: + return "" + if f"rev-parse {sync_repo_with_copybara.COPYBARA_CONFIG_FETCH_REF}" in command: + return "configsha123\n" + if sync_repo_with_copybara.COPYBARA_SYNC_RUNNER_PATH.as_posix() in command: + return "# stale runner from old build\n" + raise AssertionError(f"Unexpected command: {command}") + + mock_run_command.side_effect = run_command_side_effect + + with tempfile.TemporaryDirectory() as tmpdir: + stdout = io.StringIO() + with redirect_stdout(stdout): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.fetch_remote_copybara_config_bundle( + tmpdir, + "https://x-access-token:token@github.com/10gen/mongo.git", + ) + + log_output = stdout.getvalue() + self.assertIn("latest master-owned", log_output) + self.assertIn(sync_repo_with_copybara.COPYBARA_SYNC_RUNNER_PATH.as_posix(), log_output) + self.assertIn("Start a new master build", log_output) + + @patch("buildscripts.copybara.sync_repo_with_copybara.get_remote_branch_head") + @patch("buildscripts.copybara.sync_repo_with_copybara.run_command") + def test_fetch_remote_copybara_config_bundle_fails_for_stale_checked_out_path_rules( + self, mock_run_command, mock_get_remote_branch_head + ): + mock_get_remote_branch_head.return_value = "configsha123" + local_runner_contents = Path(sync_repo_with_copybara.__file__).resolve().read_text() + + def run_command_side_effect(command): + if command.startswith("git init "): + return "" + if "fetch --depth 1" in command: + return "" + if f"rev-parse {sync_repo_with_copybara.COPYBARA_CONFIG_FETCH_REF}" in command: + return "configsha123\n" + if sync_repo_with_copybara.COPYBARA_SYNC_RUNNER_PATH.as_posix() in command: + return local_runner_contents + if sync_repo_with_copybara.COPYBARA_PATH_RULES_HELPER_PATH.as_posix() in command: + return "# stale path rules helper from old build\n" + raise AssertionError(f"Unexpected command: {command}") + + mock_run_command.side_effect = run_command_side_effect + + with tempfile.TemporaryDirectory() as tmpdir: + stdout = io.StringIO() + with redirect_stdout(stdout): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.fetch_remote_copybara_config_bundle( + tmpdir, + "https://x-access-token:token@github.com/10gen/mongo.git", + ) + + log_output = stdout.getvalue() + self.assertIn("latest master-owned", log_output) + self.assertIn( + sync_repo_with_copybara.COPYBARA_PATH_RULES_HELPER_PATH.as_posix(), log_output + ) + self.assertIn("path rules helper", log_output) + self.assertIn("Start a new master build", log_output) + + def test_get_local_copybara_config_bundle_renders_checked_out_path_rules_module(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + base_config_path = get_repo_base_copybara_config_path(root) + write_base_copybara_config(base_config_path) + path_rules_path = get_repo_copybara_path_rules_path(root) + write_copybara_path_rules( + path_rules_path, + common_includes=DEFAULT_TEST_COPYBARA_PATH_RULES_INCLUDES, + common_excludes=DEFAULT_TEST_COPYBARA_PATH_RULES_EXCLUDES, + ) + path_rules_template_path = get_repo_copybara_path_rules_template_path(root) + write_copybara_path_rules_template(path_rules_template_path) + path_rules_module_path = get_repo_copybara_path_rules_module_path(root) + fragment_dir = root / "buildscripts" / "copybara" + fragment_dir.mkdir(parents=True, exist_ok=True) + (fragment_dir / "master.sky").write_text('sync_branch("master")\n') + (fragment_dir / "v8_2.sky").write_text('sync_branch("v8.2")\n') + + bundle = sync_repo_with_copybara.get_local_copybara_config_bundle(tmpdir) + + self.assertEqual(bundle.config_sha, "local") + self.assertEqual(bundle.base_config_path, base_config_path) + self.assertEqual(bundle.path_rules_path, path_rules_path) + self.assertEqual(bundle.path_rules_module_path, path_rules_module_path) + self.assertEqual( + path_rules_module_path.read_text(), + render_copybara_path_rules_module_from_files( + path_rules_template_path, + path_rules_path, + ), + ) + self.assertEqual(bundle.branch_to_fragment["master"], fragment_dir / "master.sky") + self.assertEqual(bundle.branch_to_fragment["v8.2"], fragment_dir / "v8_2.sky") + + +class TestGenerateCopybaraEvergreen(unittest.TestCase): + @staticmethod + def write_fragment(root: Path, filename: str, contents: str) -> None: + fragment_path = root / sync_repo_with_copybara.COPYBARA_CONFIG_DIRECTORY / filename + fragment_path.parent.mkdir(parents=True, exist_ok=True) + fragment_path.write_text(contents) + + def test_render_expected_copybara_evergreen_uses_branch_and_tag_fragments(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + self.write_fragment(root, "master.sky", 'sync_branch("master")\n') + self.write_fragment( + root, + "v8_2.sky", + 'sync_branch("v8.2")\n' + 'sync_branch("v8.2.7-hotfix")\n' + 'sync_tag("r8.2.7-hotfix")\n', + ) + + generated = generate_evergreen.render_expected_copybara_evergreen(root) + + self.assertIn( + "# This file is generated by buildscripts/copybara/generate_evergreen.py.", + generated, + ) + self.assertIn(" - name: sync_copybara_master", generated) + self.assertIn(" - name: sync_copybara_v8_2", generated) + self.assertIn(" - name: sync_copybara_v8_2_7_hotfix", generated) + self.assertIn(' copybara_branches: "v8.2.7-hotfix"', generated) + self.assertIn(" - name: sync_copybara_r8_2_7_hotfix", generated) + self.assertIn(' copybara_branches: "r8.2.7-hotfix"', generated) + self.assertNotIn("sync_copybara_task_template", generated) + self.assertLess( + generated.index("sync_copybara_master"), + generated.index("sync_copybara_v8_2"), + ) + self.assertLess( + generated.index("sync_copybara_v8_2_7_hotfix"), + generated.index("sync_copybara_r8_2_7_hotfix"), + ) + + def test_check_generated_copybara_evergreen_detects_stale_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + self.write_fragment(root, "master.sky", 'sync_branch("master")\n') + generated_path = root / generate_evergreen.COPYBARA_EVERGREEN_GENERATED_CONFIG_PATH + generated_path.parent.mkdir(parents=True, exist_ok=True) + generated_path.write_text("stale\n") + + stdout = io.StringIO() + with redirect_stdout(stdout): + is_current = generate_evergreen.check_generated_copybara_evergreen(root) + + self.assertFalse(is_current) + self.assertIn("Generated Copybara Evergreen config is stale", stdout.getvalue()) + + def test_check_generated_copybara_evergreen_accepts_current_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + self.write_fragment(root, "master.sky", 'sync_branch("master")\n') + generated_path = root / generate_evergreen.COPYBARA_EVERGREEN_GENERATED_CONFIG_PATH + generated_path.parent.mkdir(parents=True, exist_ok=True) + generated_path.write_text(generate_evergreen.render_expected_copybara_evergreen(root)) + + self.assertTrue(generate_evergreen.check_generated_copybara_evergreen(root)) + + @patch("buildscripts.copybara.generate_evergreen.check_generated_copybara_evergreen") + def test_sync_precheck_accepts_current_generated_evergreen(self, mock_check_generated): + mock_check_generated.return_value = True + + stdout = io.StringIO() + with redirect_stdout(stdout): + sync_repo_with_copybara.ensure_generated_copybara_evergreen_is_current("/repo") + + mock_check_generated.assert_called_once_with(Path("/repo")) + self.assertIn("Generated Copybara Evergreen config is current", stdout.getvalue()) + + @patch("buildscripts.copybara.generate_evergreen.check_generated_copybara_evergreen") + def test_sync_precheck_rejects_stale_generated_evergreen(self, mock_check_generated): + mock_check_generated.return_value = False + + with self.assertRaises(SystemExit) as context: + sync_repo_with_copybara.ensure_generated_copybara_evergreen_is_current("/repo") + + self.assertEqual(context.exception.code, 1) + mock_check_generated.assert_called_once_with(Path("/repo")) + + +class TestValidateSyncConfig(unittest.TestCase): + @patch("buildscripts.copybara.sync_repo_with_copybara.check_branch_top_level_paths_are_labeled") + @patch("buildscripts.copybara.sync_repo_with_copybara.list_top_level_paths_for_remote_ref") + def test_uses_pinned_source_ref_for_top_level_validation( + self, + mock_list_top_level_paths_for_remote_ref, + mock_check_branch_top_level_paths_are_labeled, + ): + mock_list_top_level_paths_for_remote_ref.return_value = {"README.md", "src"} + sync = sync_repo_with_copybara.PreparedBranchSync( + branch="v8.2", + source_ref="deadbeef123", + config_sha="configsha123", + workflow_name="prod_v8.2", + config_file=Path("/tmp/generated.sky"), + preview_dir=Path("/tmp/preview"), + docker_command=("echo", "copybara"), + copybara_config=sync_repo_with_copybara.CopybaraConfig( + source=sync_repo_with_copybara.CopybaraRepoConfig( + git_url="https://example.com/source.git", + repo_name=sync_repo_with_copybara.SOURCE_REPO_NAME, + branch="v8.2", + ref="deadbeef123", + ), + destination=sync_repo_with_copybara.CopybaraRepoConfig( + git_url="https://example.com/destination.git", + repo_name=sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_NAME, + branch="v8.2", + ), + ), + ) + + sync_repo_with_copybara.validate_sync_config(sync) + + mock_list_top_level_paths_for_remote_ref.assert_called_once_with( + "https://example.com/source.git", + "deadbeef123", + ) + mock_check_branch_top_level_paths_are_labeled.assert_called_once_with( + str(sync.config_file), + "v8.2", + {"README.md", "src"}, + ) + + +class TestEnsureCopybaraSourceRefSupport(unittest.TestCase): + """Verify that ensure_copybara_source_ref_support correctly injects or preserves source_refs.""" + + def _make_config_without_source_refs(self) -> str: + return textwrap.dedent("""\ + source_url = "https://github.com/10gen/mongo.git" + test_branch_prefix = "copybara_test_branch" + + def sync_branch(branch_name): + make_workflow("prod_" + branch_name, prod_url, branch_name, branch_name) + make_workflow( + "test_" + branch_name, + test_url, + branch_name, + test_branch_prefix + "_" + branch_name, + ) + """) + + def _make_config_with_source_refs(self) -> str: + return textwrap.dedent("""\ + source_url = "https://github.com/10gen/mongo.git" + test_branch_prefix = "copybara_test_branch" + source_refs = {} + + def sync_branch(branch_name): + source_ref = source_refs.get(branch_name, branch_name) + make_workflow( + "prod_" + branch_name, + prod_url, + source_ref, + branch_name, + ) + make_workflow( + "test_" + branch_name, + test_url, + source_ref, + test_branch_prefix + "_" + branch_name, + ) + """) + + def _make_legacy_config_with_source_refs(self) -> str: + return textwrap.dedent("""\ + source_url = "https://github.com/10gen/mongo.git" + test_branch_prefix = "copybara_test_branch" + source_refs = {} + + def sync_branch(branch_name, branch_excluded_files = [], branch_public_files = ["**"]): + source_ref = source_refs.get(branch_name, branch_name) + make_workflow( + "prod_" + branch_name, + prod_url, + source_ref, + branch_name, + branch_excluded_files, + branch_public_files, + ) + make_workflow( + "test_" + branch_name, + test_url, + source_ref, + test_branch_prefix + "_" + branch_name, + branch_excluded_files, + branch_public_files, + ) + """) + + def test_injects_source_refs_when_missing(self): + contents = self._make_config_without_source_refs() + result = sync_repo_with_copybara.ensure_copybara_source_ref_support( + contents, Path("test.sky") + ) + self.assertIn("source_refs = {}", result) + self.assertIn("source_ref = source_refs.get(branch_name, branch_name)", result) + self.assertIn("def sync_tag(tag_name):", result) + self.assertIn( + 'make_workflow("prod_" + branch_name, prod_url, source_ref, branch_name)', + result, + ) + self.assertIn( + 'make_workflow("prod_" + tag_name, prod_url, source_ref, branch_name)', + result, + ) + + def test_preserves_existing_source_refs(self): + contents = self._make_config_with_source_refs() + result = sync_repo_with_copybara.ensure_copybara_source_ref_support( + contents, Path("test.sky") + ) + self.assertIn("source_refs = {}", result) + self.assertIn("source_ref = source_refs.get(branch_name, branch_name)", result) + self.assertIn("def sync_branch(branch_name):", result) + + def test_preserves_legacy_source_refs_with_public_files(self): + contents = self._make_legacy_config_with_source_refs() + result = sync_repo_with_copybara.ensure_copybara_source_ref_support( + contents, Path("test.sky") + ) + self.assertIn("source_refs = {}", result) + self.assertIn("source_ref = source_refs.get(branch_name, branch_name)", result) + self.assertIn('branch_public_files = ["**"]', result) + self.assertIn("branch_public_files,\n )", result) + + def test_exits_when_sync_branch_not_found(self): + contents = textwrap.dedent("""\ + source_url = "https://github.com/10gen/mongo.git" + test_branch_prefix = "copybara_test_branch" + source_refs = {} + """) + with self.assertRaises(SystemExit): + sync_repo_with_copybara.ensure_copybara_source_ref_support(contents, Path("test.sky")) + + def test_exits_when_test_branch_prefix_not_found(self): + contents = textwrap.dedent("""\ + source_url = "https://github.com/10gen/mongo.git" + + def sync_branch(branch_name, branch_excluded_files = []): + pass + """) + with self.assertRaises(SystemExit): + sync_repo_with_copybara.ensure_copybara_source_ref_support(contents, Path("test.sky")) + + +class TestValidatePreviewExclusions(unittest.TestCase): + """Verify dry-run output validation catches forbidden files.""" + + def _make_sync_with_preview( + self, + files: list[str], + branch: str = "master", + branch_excluded_patterns: list[str] | None = None, + ) -> sync_repo_with_copybara.PreparedBranchSync: + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + root = Path(tmpdir) + + config_path = root / "copy.bara.sky" + write_base_copybara_config(config_path, common_includes=["README.md"]) + branch_excluded_patterns = branch_excluded_patterns or [] + if branch_excluded_patterns: + exclude_entries = "\n".join(f' "{pattern}",' for pattern in branch_excluded_patterns) + config_path.write_text( + config_path.read_text() + + "\nbranch_files_to_exclude = [\n" + + f"{exclude_entries}\n" + + "]\n" + + f'sync_branch("{branch}", branch_files_to_exclude)\n' + ) + else: + config_path.write_text(config_path.read_text() + f'\nsync_branch("{branch}")\n') + + preview_dir = root / "preview" / "checkout" + preview_dir.mkdir(parents=True) + for file_path in files: + full_path = preview_dir / file_path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text("content") + + return sync_repo_with_copybara.PreparedBranchSync( + branch=branch, + source_ref="abc123", + config_sha="sha123", + workflow_name=f"prod_{branch}", + config_file=config_path, + preview_dir=root / "preview", + docker_command=("echo",), + ) + + def test_passes_when_no_excluded_files_present(self): + sync = self._make_sync_with_preview( + [ + "src/mongo/db/catalog/collection.cpp", + "README.md", + "jstests/core/basic.js", + ] + ) + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_modules_directory_present(self): + sync = self._make_sync_with_preview( + [ + "src/mongo/db/modules/enterprise/something.cpp", + ] + ) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError) as ctx: + sync_repo_with_copybara.validate_preview_exclusions(sync) + self.assertIn("preview validation", ctx.exception.stage) + + def test_fails_when_agents_md_present(self): + sync = self._make_sync_with_preview(["AGENTS.md"]) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_private_third_party_present(self): + sync = self._make_sync_with_preview( + [ + "src/third_party/private/secret.h", + ] + ) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_cursor_directory_present(self): + sync = self._make_sync_with_preview( + [ + ".cursor/rules/my_rule.md", + ] + ) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_github_codeowners_present(self): + sync = self._make_sync_with_preview( + [ + ".github/CODEOWNERS", + ] + ) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_monguard_present(self): + sync = self._make_sync_with_preview( + [ + "monguard/config.yaml", + ] + ) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_branch_specific_excluded_file_present(self): + sync = self._make_sync_with_preview( + ["docs/private-notes/secret.md"], + branch="v8.2", + branch_excluded_patterns=["docs/private-notes/**"], + ) + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_private_commercial_header_present(self): + sync = self._make_sync_with_preview([]) + public_header_path = sync.preview_dir / "checkout" / "src/mongo/util/public_header.h" + public_header_path.parent.mkdir(parents=True, exist_ok=True) + public_header_path.write_text( + textwrap.dedent("""\ + /** + * Copyright (C) 2025-present MongoDB, Inc. and subject to applicable commercial license. + */ + """) + ) + + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_excluded_broken_symlink_present(self): + if sys.platform == "win32": + self.skipTest("symlink permissions vary on Windows") + + sync = self._make_sync_with_preview([]) + excluded_symlink = sync.preview_dir / "checkout" / "AGENTS.md" + os.symlink("missing-target", excluded_symlink) + + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + def test_fails_when_excluded_symlinked_directory_present(self): + if sys.platform == "win32": + self.skipTest("symlink permissions vary on Windows") + + sync = self._make_sync_with_preview([]) + excluded_symlink_dir = sync.preview_dir / "checkout" / "src/mongo/db/modules" + excluded_symlink_dir.parent.mkdir(parents=True, exist_ok=True) + os.symlink("missing-target", excluded_symlink_dir) + + with self.assertRaises(sync_repo_with_copybara.BranchSyncError): + sync_repo_with_copybara.validate_preview_exclusions(sync) + + +class TestRedactSecrets(unittest.TestCase): + """Verify token redaction in log output.""" + + def test_redacts_known_tokens(self): + original_secrets = list(sync_repo_with_copybara.REDACTED_STRINGS) + try: + sync_repo_with_copybara.REDACTED_STRINGS.clear() + sync_repo_with_copybara.REDACTED_STRINGS.append("ghp_SuperSecretToken123") + + result = sync_repo_with_copybara.redact_secrets( + "Using token ghp_SuperSecretToken123 for auth" + ) + self.assertNotIn("ghp_SuperSecretToken123", result) + self.assertIn("", result) + finally: + sync_repo_with_copybara.REDACTED_STRINGS.clear() + sync_repo_with_copybara.REDACTED_STRINGS.extend(original_secrets) + + def test_redacts_github_url_credentials(self): + url = "https://x-access-token:ghs_abc123xyz@github.com/10gen/mongo.git" + result = sync_repo_with_copybara.redact_secrets(url) + self.assertNotIn("ghs_abc123xyz", result) + self.assertIn("https://x-access-token:@github.com", result) + + def test_redacts_unknown_github_url_credentials(self): + """Credentials not in REDACTED_STRINGS are still caught by the regex fallback.""" + url = "https://x-access-token:unknown_ambient_token@github.com/mongodb/mongo.git" + result = sync_repo_with_copybara.redact_secrets(url) + self.assertNotIn("unknown_ambient_token", result) + + def test_preserves_non_sensitive_text(self): + text = "Running copybara for branch master" + result = sync_repo_with_copybara.redact_secrets(text) + self.assertEqual(text, result) + + def test_handles_empty_redacted_strings_list(self): + original_secrets = list(sync_repo_with_copybara.REDACTED_STRINGS) + try: + sync_repo_with_copybara.REDACTED_STRINGS.clear() + result = sync_repo_with_copybara.redact_secrets("no secrets here") + self.assertEqual("no secrets here", result) + finally: + sync_repo_with_copybara.REDACTED_STRINGS.clear() + sync_repo_with_copybara.REDACTED_STRINGS.extend(original_secrets) + + +class TestExtractSkyExcludedPatternsRejectsDuplicates(unittest.TestCase): + """Verify that duplicate common_files_to_exclude definitions are rejected.""" + + def test_rejects_multiple_common_files_to_exclude_definitions(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + sky_path.write_text( + textwrap.dedent("""\ + common_files_to_exclude = [ + "src/mongo/db/modules/**", + ] + + common_files_to_exclude = [ + "harmless_file.txt", + ] + """) + ) + + with self.assertRaises(SystemExit): + sync_repo_with_copybara.extract_sky_excluded_patterns(str(sky_path)) + + def test_accepts_single_definition(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config(sky_path) + patterns = sync_repo_with_copybara.extract_sky_excluded_patterns(str(sky_path)) + self.assertIsInstance(patterns, set) + self.assertTrue(len(patterns) > 0) + + def test_ignores_commented_out_definitions(self): + with tempfile.TemporaryDirectory() as tmpdir: + sky_path = Path(tmpdir) / "copy.bara.sky" + sky_path.write_text( + textwrap.dedent("""\ + # common_files_to_exclude = ["should_be_ignored/**"] + common_files_to_exclude = [ + "src/mongo/db/modules/**", + ] + """) + ) + + patterns = sync_repo_with_copybara.extract_sky_excluded_patterns(str(sky_path)) + self.assertNotIn("should_be_ignored/**", patterns) + self.assertIn("src/mongo/db/modules/**", patterns) + + +class TestMatchesExcludedPattern(unittest.TestCase): + """Verify path matching logic for excluded patterns.""" + + def test_directory_pattern_matches_files_in_subtree(self): + self.assertTrue( + sync_repo_with_copybara.matches_excluded_pattern( + "src/mongo/db/modules/enterprise/foo.cpp", "src/mongo/db/modules/" + ) + ) + + def test_directory_pattern_matches_directory_itself(self): + self.assertTrue( + sync_repo_with_copybara.matches_excluded_pattern( + "src/mongo/db/modules", "src/mongo/db/modules/" + ) + ) + + def test_directory_pattern_with_glob_suffix(self): + self.assertTrue( + sync_repo_with_copybara.matches_excluded_pattern( + "src/third_party/private/secret.h", "src/third_party/private/**" + ) + ) + + def test_directory_pattern_does_not_match_unrelated_path(self): + self.assertFalse( + sync_repo_with_copybara.matches_excluded_pattern( + "src/mongo/db/storage/wiredtiger.cpp", "src/mongo/db/modules/" + ) + ) + + def test_exact_file_pattern_matches_exact_path(self): + self.assertTrue(sync_repo_with_copybara.matches_excluded_pattern("AGENTS.md", "AGENTS.md")) + + def test_exact_file_pattern_does_not_match_different_file(self): + self.assertFalse(sync_repo_with_copybara.matches_excluded_pattern("README.md", "AGENTS.md")) + + def test_exact_file_pattern_does_not_match_subdirectory_file(self): + self.assertFalse( + sync_repo_with_copybara.matches_excluded_pattern("docs/AGENTS.md", "AGENTS.md") + ) + + def test_directory_pattern_does_not_match_prefix_overlap(self): + """monguard/ should not match monguard_extra/.""" + self.assertFalse( + sync_repo_with_copybara.matches_excluded_pattern("monguard_extra/file.txt", "monguard/") + ) + + +class TestCanonicalizeExcludedPattern(unittest.TestCase): + """Verify pattern normalization for preview exclusion matching.""" + + def test_trailing_slash_becomes_directory_pattern(self): + self.assertEqual( + sync_repo_with_copybara.canonicalize_excluded_pattern("monguard/"), + "monguard/", + ) + + def test_glob_suffix_becomes_directory_pattern(self): + self.assertEqual( + sync_repo_with_copybara.canonicalize_excluded_pattern("monguard/**"), + "monguard/", + ) + + def test_exact_file_stays_exact(self): + self.assertEqual( + sync_repo_with_copybara.canonicalize_excluded_pattern("AGENTS.md"), + "AGENTS.md", + ) + + def test_rejects_wildcard_patterns(self): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.canonicalize_excluded_pattern("*.py") + + def test_rejects_empty_pattern(self): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.canonicalize_excluded_pattern("") + + def test_rejects_slash_only_pattern(self): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.canonicalize_excluded_pattern("/") + + +class TestAssembleCopybaraConfig(unittest.TestCase): + """Verify that base config and fragments are correctly concatenated.""" + + def test_combines_base_and_single_fragment(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + base = root / "base.sky" + base.write_text('# base config\nsource_url = "foo"\n') + + fragment = root / "master.sky" + fragment.write_text('sync_branch("master")\n') + + output = root / "assembled.sky" + sync_repo_with_copybara.assemble_copybara_config(base, [fragment], output) + + assembled = output.read_text() + self.assertIn("# base config", assembled) + self.assertIn('sync_branch("master")', assembled) + self.assertIn(f"# BEGIN {fragment.as_posix()}", assembled) + self.assertIn(f"# END {fragment.as_posix()}", assembled) + + def test_combines_base_and_multiple_fragments(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + base = root / "base.sky" + base.write_text("# base\n") + + frag_a = root / "a.sky" + frag_a.write_text('sync_branch("a")\n') + frag_b = root / "b.sky" + frag_b.write_text('sync_branch("b")\n') + + output = root / "out.sky" + sync_repo_with_copybara.assemble_copybara_config(base, [frag_a, frag_b], output) + + assembled = output.read_text() + self.assertIn('sync_branch("a")', assembled) + self.assertIn('sync_branch("b")', assembled) + idx_a = assembled.index('sync_branch("a")') + idx_b = assembled.index('sync_branch("b")') + self.assertLess(idx_a, idx_b) + + def test_output_ends_with_newline(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + base = root / "base.sky" + base.write_text("# base\n") + fragment = root / "f.sky" + fragment.write_text('sync_branch("x")\n') + output = root / "out.sky" + sync_repo_with_copybara.assemble_copybara_config(base, [fragment], output) + self.assertTrue(output.read_text().endswith("\n")) + + +class TestParseBranchList(unittest.TestCase): + """Verify edge-case handling for comma-separated branch parsing.""" + + def test_returns_empty_for_none(self): + self.assertEqual(sync_repo_with_copybara.parse_branch_list(None), []) + + def test_returns_empty_for_blank_string(self): + self.assertEqual(sync_repo_with_copybara.parse_branch_list(""), []) + + def test_single_branch(self): + self.assertEqual( + sync_repo_with_copybara.parse_branch_list("master"), + ["master"], + ) + + def test_deduplicates_preserving_order(self): + self.assertEqual( + sync_repo_with_copybara.parse_branch_list("v8.2, master, v8.2"), + ["v8.2", "master"], + ) + + def test_strips_whitespace(self): + self.assertEqual( + sync_repo_with_copybara.parse_branch_list(" master , v8.0 , v8.2 "), + ["master", "v8.0", "v8.2"], + ) + + def test_skips_empty_entries_from_trailing_comma(self): + self.assertEqual( + sync_repo_with_copybara.parse_branch_list("master,v8.0,"), + ["master", "v8.0"], + ) + + def test_skips_empty_entries_from_double_comma(self): + self.assertEqual( + sync_repo_with_copybara.parse_branch_list("master,,v8.0"), + ["master", "v8.0"], + ) + + +class TestRealCopybaraSkyConfiguration(unittest.TestCase): + """Integration tests for the checked-in Copybara config files.""" + + REAL_COPYBARA_ROOT = Path(__file__).resolve().parents[2] + REAL_SKY_PATH = REAL_COPYBARA_ROOT / sync_repo_with_copybara.COPYBARA_BASE_CONFIG_PATH + REAL_TEMPLATE_PATH = ( + REAL_COPYBARA_ROOT / "buildscripts" / "copybara" / "copybara_path_rules.bara.sky.template" + ) + REAL_PATH_RULES_PATH = REAL_COPYBARA_ROOT / sync_repo_with_copybara.COPYBARA_PATH_RULES_PATH + REAL_PATH_RULES_MODULE_PATH = ( + REAL_COPYBARA_ROOT / sync_repo_with_copybara.COPYBARA_PATH_RULES_MODULE_PATH + ) + REAL_GENERATED_EVERGREEN_PATH = ( + REAL_COPYBARA_ROOT / generate_evergreen.COPYBARA_EVERGREEN_GENERATED_CONFIG_PATH + ) + + @unittest.skipUnless( + REAL_PATH_RULES_MODULE_PATH.is_file() + and REAL_TEMPLATE_PATH.is_file() + and REAL_PATH_RULES_PATH.is_file(), + "checked-in Copybara source-of-truth files not found at expected paths", + ) + def test_real_path_rules_module_matches_rendered_source_of_truth(self): + rendered = render_copybara_path_rules_module_from_files( + self.REAL_TEMPLATE_PATH, + self.REAL_PATH_RULES_PATH, + ) + self.assertEqual(self.REAL_PATH_RULES_MODULE_PATH.read_text(), rendered) + + @unittest.skipUnless( + REAL_SKY_PATH.is_file(), + "checked-in copy.bara.sky not found at expected path", + ) + def test_real_base_config_loads_generated_path_rules_module(self): + contents = self.REAL_SKY_PATH.read_text() + self.assertIn('load("copybara_path_rules"', contents) + self.assertIn('"common_files_to_include"', contents) + self.assertIn('"common_files_to_exclude"', contents) + + @unittest.skipUnless( + REAL_PATH_RULES_PATH.is_file(), + "checked-in Copybara path rules JSON not found at expected path", + ) + def test_real_path_rules_json_lists_are_sorted(self): + payload = json.loads(self.REAL_PATH_RULES_PATH.read_text()) + + for field_name in ("common_files_to_include", "common_files_to_exclude"): + with self.subTest(field_name=field_name): + self.assertEqual( + payload[field_name], + sorted(payload[field_name]), + f"{field_name} in copybara_path_rules.json must remain sorted.", + ) + + @unittest.skipUnless( + REAL_SKY_PATH.is_file(), + "checked-in copy.bara.sky not found at expected path", + ) + def test_real_fragments_match_expected_mainline_branch_allowlist(self): + branch_to_fragment = sync_repo_with_copybara.discover_copybara_branches( + str(self.REAL_COPYBARA_ROOT) + ) + mainline_branches = sorted( + branch for branch in branch_to_fragment if not branch.endswith("-hotfix") + ) + hotfix_branches = sorted( + branch for branch in branch_to_fragment if branch.endswith("-hotfix") + ) + + self.assertEqual(mainline_branches, ["master", "v7.0", "v8.0", "v8.2"]) + self.assertTrue(all(branch.endswith("-hotfix") for branch in hotfix_branches)) + + @unittest.skipUnless( + REAL_GENERATED_EVERGREEN_PATH.is_file(), + "checked-in generated Copybara Evergreen config not found at expected path", + ) + def test_real_generated_evergreen_matches_copybara_fragments(self): + self.assertEqual( + self.REAL_GENERATED_EVERGREEN_PATH.read_text(), + generate_evergreen.render_expected_copybara_evergreen(self.REAL_COPYBARA_ROOT), + ) + + +class TestEvergreenProjectGuard(unittest.TestCase): + def test_passes_for_expected_master_project(self): + sync_repo_with_copybara.ensure_expected_evergreen_project( + {"project": sync_repo_with_copybara.EXPECTED_EVERGREEN_PROJECT} + ) + + def test_fails_for_missing_project(self): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.ensure_expected_evergreen_project({}) + + def test_fails_for_non_master_project(self): + with self.assertRaises(SystemExit): + sync_repo_with_copybara.ensure_expected_evergreen_project( + {"project": "mongodb-mongo-master-nightly"} + ) + + +class TestHotfixTaskActivation(unittest.TestCase): + def test_get_hotfix_branches_for_release(self): + hotfix_branches = sync_repo_with_copybara.get_hotfix_branches_for_release( + "v8.2", + ["master", "v8.2", "v8.2-hotfix", "v8.2.6-hotfix", "v8.0.10-hotfix"], + ) + + self.assertEqual(hotfix_branches, ["v8.2-hotfix", "v8.2.6-hotfix"]) + + @patch("buildscripts.copybara.sync_repo_with_copybara.retry_operation") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_evergreen_api") + @patch("buildscripts.copybara.sync_repo_with_copybara.check_destination_branch_exists") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + def test_activate_new_hotfix_tasks_activates_unsynced_hotfix_task( + self, + mock_branch_exists_remote, + mock_check_destination_branch_exists, + mock_get_api, + mock_retry_operation, + ): + mock_branch_exists_remote.return_value = True + mock_check_destination_branch_exists.return_value = False + + inactive_task = MagicMock() + inactive_task.display_name = "sync_copybara_v8_2_6_hotfix" + inactive_task.activated = False + inactive_task.task_id = "task_1" + + mock_api = mock_get_api.return_value + mock_api.tasks_by_build.return_value = [inactive_task] + + with tempfile.TemporaryDirectory() as tmpdir: + config_file = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + config_file, + prod_url=sync_repo_with_copybara.TEST_REPO_URL, + ) + + sync_repo_with_copybara.activate_new_hotfix_tasks( + selected_branches=["v8.2"], + configured_branches=["master", "v8.2", "v8.2.6-hotfix"], + expansions={"build_id": "build_1"}, + tokens_map={ + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + }, + config_file=config_file, + ) + + mock_retry_operation.assert_called_once_with( + mock_api.configure_task, + "task_1", + activated=True, + tries=3, + delay_seconds=5, + backoff_factor=2, + ) + self.assertEqual( + mock_check_destination_branch_exists.call_args.args[0].destination.repo_name, + sync_repo_with_copybara.TEST_REPO_NAME, + ) + + @patch("buildscripts.copybara.sync_repo_with_copybara.retry_operation") + @patch("buildscripts.copybara.sync_repo_with_copybara.get_evergreen_api") + @patch("buildscripts.copybara.sync_repo_with_copybara.check_destination_branch_exists") + @patch("buildscripts.copybara.sync_repo_with_copybara.branch_exists_remote") + def test_activate_new_hotfix_tasks_skips_existing_public_branch( + self, + mock_branch_exists_remote, + mock_check_destination_branch_exists, + mock_get_api, + mock_retry_operation, + ): + mock_branch_exists_remote.return_value = True + mock_check_destination_branch_exists.return_value = True + mock_get_api.return_value.tasks_by_build.return_value = [] + + with tempfile.TemporaryDirectory() as tmpdir: + config_file = Path(tmpdir) / "copy.bara.sky" + write_base_copybara_config( + config_file, + prod_url=sync_repo_with_copybara.TEST_REPO_URL, + ) + + sync_repo_with_copybara.activate_new_hotfix_tasks( + selected_branches=["v8.2"], + configured_branches=["master", "v8.2", "v8.2.6-hotfix"], + expansions={"build_id": "build_1"}, + tokens_map={ + sync_repo_with_copybara.SOURCE_REPO_URL: "source-token", + sync_repo_with_copybara.PUBLIC_GITHUB_APP_REPO_URL: "prod-token", + sync_repo_with_copybara.TEST_REPO_URL: "test-token", + }, + config_file=config_file, + ) + + mock_retry_operation.assert_not_called() if __name__ == "__main__": diff --git a/docs/branching/README.md b/docs/branching/README.md index b0a3e84952a..0f7a52d3505 100644 --- a/docs/branching/README.md +++ b/docs/branching/README.md @@ -51,16 +51,15 @@ VERSION=8.3 ### Copybara configuration -Run the following automation and verify results: +Run the following automation in the private repo and verify results: ```sh -sed -i "s/master/v$VERSION/g" copy.bara.sky -sed -i 's/branch = "master"/branch = "v'"$VERSION"'"/' buildscripts/sync_repo_with_copybara.py +sed -i "s/master/v$VERSION/g" buildscripts/copybara/copy.bara.sky buildscripts/copybara/sync_repo_with_copybara.py ``` -For each file [`copy.bara.sky`](../../copy.bara.sky) and -[`sync_repo_with_copybara.py`](../../buildscripts/sync_repo_with_copybara.py), the "master" branch -references should be replaced with the new branch name. +In the private repo, `buildscripts/copybara/copy.bara.sky` and +`buildscripts/copybara/sync_repo_with_copybara.py` should have their `"master"` branch references +replaced with the new branch name. ### Evergreen YAML configurations diff --git a/docs/owners/owners_format.md b/docs/owners/owners_format.md index 88e7a5ba2df..7cec5cbe361 100644 --- a/docs/owners/owners_format.md +++ b/docs/owners/owners_format.md @@ -33,8 +33,11 @@ programmatically to, for example, generate a report of all the files owned by a even though that team has nominated specific engineers as approvers. `options` are not required and are various options about how to use this OWNERS.yml file. Currently -there is only a single option `no_parent_owners` which is defaulted to false. If this option is set -to true it will stop upwards OWNERS resolution. +there are two options: + +- `no_parent_owners`, which defaults to false. If set to true it stops upwards OWNERS resolution. +- `no_auto_approver`, which defaults to false. If set to true it prevents the generated `CODEOWNERS` + entry for this `OWNERS.yml` file from automatically including `@svc-auto-approve-bot`. ### Example file @@ -70,6 +73,7 @@ filters: # List of all filters - bazel-approvers options: # All options for this file no_parent_owners: false # See above for no_parent_owners. Defaulted to false so this line is not needed. + no_auto_approver: false # Prevents auto-adding @svc-auto-approve-bot for this OWNERS file. ``` ### Filter examples diff --git a/etc/BUILD.bazel b/etc/BUILD.bazel index 8aa4067245d..2525918b5f6 100644 --- a/etc/BUILD.bazel +++ b/etc/BUILD.bazel @@ -7,6 +7,7 @@ exports_files([ "tsan.suppressions", "burn_in_tests.yml", "extensions.yml", + "evergreen_yml_components/copybara/copybara_gen.yml", "backports_required_for_multiversion_tests.yml", ]) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 1b9ec910f32..5cb1e885f0e 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -97,6 +97,8 @@ include: - filename: etc/evergreen_yml_components/variants/amazon/streams/streams_dev.yml - filename: src/mongo/db/modules/atlas/atlas_dev.yml + - filename: etc/evergreen_yml_components/copybara/copybara.yml + - filename: etc/evergreen_yml_components/copybara/copybara_gen.yml - filename: monguard/.evergreen/config.yml