From 3f6bcaec41e7f7281ddef2bcdcd483ccbeb98a92 Mon Sep 17 00:00:00 2001 From: Trevor Date: Thu, 4 Apr 2024 13:13:22 -0500 Subject: [PATCH] SERVER-88711 Generate .github/CODEOWNERS from OWNERS.yml files (#20694) GitOrigin-RevId: 1f7dc5dd5e91cc885647063b888d4d9c2f61f43c --- .github/CODEOWNERS | 45 ++++ BUILD.bazel | 6 +- OWNERS.yml | 2 +- bazel/toolchains/python_toolchain.BUILD | 8 +- buildscripts/BUILD.bazel | 14 ++ buildscripts/OWNERS.yml | 1 + buildscripts/codeowners_generate.py | 218 ++++++++++++++++++ buildscripts/resmokelib/OWNERS.yml | 1 + .../tasks/misc_tasks.yml | 27 +++ 9 files changed, 319 insertions(+), 3 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 buildscripts/BUILD.bazel create mode 100644 buildscripts/codeowners_generate.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..4bd8e68cc30 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,45 @@ +# This is a generated file do not make changes to this file. +# This is generated from various OWNERS.yml files across the repo. +# To regenerate this file run `bazel run //:codeowners` +# The documentation for the OWNERS.yml files can be found here: +# https://github.com/10gen/mongo/blob/master/docs/owners_format.md + +# The following patterns are parsed from ./OWNERS.yml +OWNERS.yml @IamXander +.bazelignore alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.bazelrc alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.bazelversion alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.clang-format alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.eslintignore alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +.eslintrc.yml alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +.mypy.ini alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.prettierignore alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +.prettierrc alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +.pydocstyle alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.pylintrc alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +.style.yapf alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +BUILD.bazel alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +copybara.sky @IamXander +copybara.staging.sky alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +jsconfig.json alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +package.json alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +pnpm-lock.yaml alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com +poetry.lock alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +pyproject.toml alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +SConstruct alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com +WORKSPACE.bazel alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com + +# The following patterns are parsed from ./bazel/OWNERS.yml +/bazel/**/* alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com + +# The following patterns are parsed from ./buildfarm/OWNERS.yml +/buildfarm/**/* alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com + +# The following patterns are parsed from ./buildscripts/OWNERS.yml +/buildscripts/**/* alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com + +# The following patterns are parsed from ./buildscripts/resmokelib/OWNERS.yml +/buildscripts/resmokelib/**/* alex.neben@mongodb.com juan.gu@mongodb.com mikhail.shchatko@mongodb.com steve.gross@mongodb.com trevor.guidry@mongodb.com + +# The following patterns are parsed from ./site_scons/OWNERS.yml +/site_scons/**/* alex.neben@mongodb.com anthony.pratti@mongodb.com daniel.moody@mongodb.com steve.gross@mongodb.com thomas.langston@mongodb.com trevor.guidry@mongodb.com udita.bose@mongodb.com zack.winter@mongodb.com diff --git a/BUILD.bazel b/BUILD.bazel index 93358427201..92ece35c642 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -3,7 +3,6 @@ load("@npm//:defs.bzl", "npm_link_all_packages") package(default_visibility = ["//visibility:public"]) exports_files([ - "buildscripts/idl", "pyproject.toml", "poetry.lock", ]) @@ -14,3 +13,8 @@ alias( name = "format", actual = "//bazel/format", ) + +alias( + name = "codeowners", + actual = "//buildscripts:codeowners", +) diff --git a/OWNERS.yml b/OWNERS.yml index 795cc507f94..3c90bd114a2 100644 --- a/OWNERS.yml +++ b/OWNERS.yml @@ -19,7 +19,7 @@ filters: - ".clang-format": approvers: - devprod-build - - ".eslint-ignore": + - ".eslintignore": approvers: - devprod-correctness - ".eslintrc.yml": diff --git a/bazel/toolchains/python_toolchain.BUILD b/bazel/toolchains/python_toolchain.BUILD index af9aea235a5..46f350165a5 100644 --- a/bazel/toolchains/python_toolchain.BUILD +++ b/bazel/toolchains/python_toolchain.BUILD @@ -2,7 +2,13 @@ load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") filegroup( name = "files", - srcs = glob(["**/*"]), + srcs = glob( + include=["**/*"], + # bazel runfiles do not support paths with spaces + # https://github.com/bazelbuild/bazel/issues/4327 + # The setuptools developers will not remove the spaces from these files + # https://github.com/pypa/setuptools/issues/746 + exclude=["**/setuptools/**/* *"]), visibility = ["//visibility:public"], ) diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel new file mode 100644 index 00000000000..1ef131334d6 --- /dev/null +++ b/buildscripts/BUILD.bazel @@ -0,0 +1,14 @@ +load("@poetry//:dependencies.bzl", "dependency") + +py_binary( + name = "codeowners", + srcs = ["codeowners_generate.py"], + main = "codeowners_generate.py", + visibility = ["//visibility:public"], + deps = [ + dependency( + "pyyaml", + group = "core", + ), + ], +) diff --git a/buildscripts/OWNERS.yml b/buildscripts/OWNERS.yml index 9920c08fe54..76eb64e400f 100644 --- a/buildscripts/OWNERS.yml +++ b/buildscripts/OWNERS.yml @@ -6,5 +6,6 @@ aliases: - //bazel/devprod_build_aliases.yml filters: - "*": + approvers: - devprod-correctness - devprod-build diff --git a/buildscripts/codeowners_generate.py b/buildscripts/codeowners_generate.py new file mode 100644 index 00000000000..a9303ab9d30 --- /dev/null +++ b/buildscripts/codeowners_generate.py @@ -0,0 +1,218 @@ +import argparse +from functools import lru_cache +import glob +import os +import pathlib +import subprocess +import sys +import yaml + +OWNERS_FILE_NAME = "OWNERS.yml" + + +def add_pattern(output_lines: list[str], pattern: str, owners: set[str]) -> None: + if owners: + output_lines.append(f"{pattern} {' '.join(sorted(owners))}") + else: + output_lines.append(pattern) + + +def add_owner_line(output_lines: list[str], directory: str, pattern: str, owners: set[str]) -> None: + # ensure the path is correct and consistent on all platforms + directory = pathlib.PurePath(directory).as_posix() + + if directory == ".": + # we are in the root dir and can directly pass the pattern + parsed_pattern = pattern + elif not pattern: + # If there is no pattern add the directory as the pattern. + parsed_pattern = f"/{directory}/" + elif "/" in pattern: + # if the pattern contains a slash the pattern should be treated as relative to the + # directory it came from. + if pattern.startswith("/"): + parsed_pattern = f"/{directory}{pattern}" + else: + parsed_pattern = f"/{directory}/{pattern}" + else: + parsed_pattern = f"/{directory}/**/{pattern}" + + test_pattern = f".{parsed_pattern}" if parsed_pattern.startswith( + "/") else f"./**/{parsed_pattern}" + + # ensure at least one file patches the pattern. + first_file_found = glob.iglob(test_pattern, recursive=True) + if all(False for _ in first_file_found): + raise (RuntimeError(f"Can not find any files that match pattern: `{pattern}`")) + + add_pattern(output_lines, parsed_pattern, owners) + + +@lru_cache(maxsize=None) +def process_alias_import(path: str) -> dict[str, list[str]]: + if not path.startswith("//"): + raise RuntimeError( + f"Alias file paths must start with // and be relative to the repo root: {path}") + + # remove // from beginning of path + parsed_path = path[2::] + + if not os.path.exists(parsed_path): + raise RuntimeError(f"Could not find alias file {path}") + + with open(parsed_path, "r") as file: + contents = yaml.safe_load(file) + assert "version" in contents, f"Version not found in {path}" + assert "aliases" in contents, f"Alias not found in {path}" + assert contents["version"] == "1.0.0", f"Invalid version in {path}" + return contents["aliases"] + + +def process_owners_file(output_lines: list[str], directory: str) -> None: + owners_file_path = os.path.join(directory, OWNERS_FILE_NAME) + if not os.path.exists(owners_file_path): + return + print(f"parsing: {owners_file_path}") + output_lines.append(f"# The following patterns are parsed from {owners_file_path}") + + with open(owners_file_path, "r") as file: + contents = yaml.safe_load(file) + assert "version" in contents, f"Version not found in {owners_file_path}" + assert "filters" in contents, f"Filters not found in {owners_file_path}" + assert contents["version"] == "1.0.0", f"Invalid version in {owners_file_path}" + no_parent_owners = False + if "options" in contents: + options = contents["options"] + no_parent_owners = "no_parent_owners" in options and options["no_parent_owners"] + + if no_parent_owners: + # Specfying no owners will ensure that no file in this directory has an owner unless it + # matches one of the later patterns in the file. + add_owner_line(output_lines, directory, pattern="*", owners=None) + + aliases = {} + if "aliases" in contents: + for alias_file in contents["aliases"]: + aliases.update(process_alias_import(alias_file)) + + filters = contents["filters"] + for _filter in filters: + assert "approvers" in _filter, f"Filter in {owners_file_path} does not have approvers." + approvers = _filter["approvers"] + del _filter["approvers"] + if "emeritus_approvers" in _filter: + del _filter["emeritus_approvers"] + + # the last key remaining should be the pattern for the filter + assert len(_filter) == 1, f"Filter in {owners_file_path} has incorrect values." + pattern = next(iter(_filter)) + owners = set() + + def process_owner(owner: str): + if "@" in owner: + # approver is email, just add as is + if not owner.endswith("@mongodb.com"): + raise RuntimeError("Any emails specified must be a mongodb.com email.") + owners.add(owner) + else: + # approver is github username, need to prefix with @ + owners.add(f"@{owner}") + + for approver in approvers: + if approver in aliases: + for member in aliases[approver]: + process_owner(member) + else: + process_owner(approver) + + add_owner_line(output_lines, directory, pattern, owners) + output_lines.append("") + + +# Order matters, we need to always add the contents of the root directory to codeowners first +# and work our way to the outside directories in that order. +def process_dir(output_lines: list[str], directory: str) -> None: + process_owners_file(output_lines, directory) + for item in sorted(os.listdir(directory)): + path = os.path.join(directory, item) + if not os.path.isdir(path) or os.path.islink(path): + continue + + process_dir(output_lines, path) + + +def main(): + # If we are running in bazel, default the directory to the workspace + default_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if not default_dir: + process = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, + text=True, check=True) + default_dir = process.stdout.strip() + + parser = argparse.ArgumentParser( + prog='GenerateCodeowners', + description='This generates a CODEOWNERS file based off of our OWNERS.yml files. ' + 'Whenever changes are made to the OWNERS.yml files in the repo this script ' + 'should be run.') + + parser.add_argument("--output-file", help="Path of the CODEOWNERS file to be generated.", + default=os.path.join(".github", "CODEOWNERS")) + parser.add_argument("--repo-dir", help="Root of the repo to scan for OWNER files.", + default=default_dir) + parser.add_argument( + "--check", help= + "When set, program exits 1 when the CODEOWNERS content changes. This will skip generation", + default=False, action="store_true") + + args = parser.parse_args() + os.chdir(args.repo_dir) + + # The lines to write to the CODEOWNERS file + output_lines = [ + "# This is a generated file do not make changes to this file.", + "# This is generated from various OWNERS.yml files across the repo.", + "# To regenerate this file run `bazel run //:codeowners`", + "# The documentation for the OWNERS.yml files can be found here:", + "# https://github.com/10gen/mongo/blob/master/docs/owners_format.md", + "", + ] + + print(f"Scanning for OWNERS.yml files in {os.path.abspath(os.curdir)}") + try: + process_dir(output_lines, "./") + except Exception as ex: + print("An exception was found while generating the CODEOWNERS file.", file=sys.stderr) + print("Please refer to the docs to see the spec for OWNERS.yml files here :", + file=sys.stderr) + print("https://github.com/10gen/mongo/blob/master/docs/owners_format.md", file=sys.stderr) + raise ex + + old_contents = "" + check = args.check + output_file = args.output_file + os.makedirs(os.path.dirname(output_file), exist_ok=True) + if check and os.path.exists(output_file): + with open(output_file, "r") as file: + old_contents = file.read() + + new_contents = "\n".join(output_lines) + if check: + if new_contents != old_contents: + print("ERROR: New contents of codeowners file does not match old contents.") + print( + "If you are seeing this message in CI you likely need to run `bazel run //:codeowners`" + ) + return 1 + + print("CODEOWNERS file is up to date") + return 0 + + with open(output_file, "w") as file: + file.write(new_contents) + print(f"Successfully wrote to the CODEOWNERS file at: {os.path.abspath(output_file)}") + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/buildscripts/resmokelib/OWNERS.yml b/buildscripts/resmokelib/OWNERS.yml index 06f9646a494..ea569ecf535 100644 --- a/buildscripts/resmokelib/OWNERS.yml +++ b/buildscripts/resmokelib/OWNERS.yml @@ -3,4 +3,5 @@ aliases: - //buildscripts/resmokelib/devprod_correctness_aliases.yml filters: - "*": + approvers: - devprod-correctness diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index fdec505b7cf..9812d4cbf88 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -521,6 +521,33 @@ tasks: target: >- //:format -- --mode check +# TODO: rename if display_name appears on the evergreen UI +- name: bazel_run_//:codeowners + tags: ["assigned_to_jira_team_devprod_build", "development_critical_single_variant", "lint", "bazel_check"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - command: timeout.update + params: + # 40 minutes + exec_timeout_secs: 2400 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. + - func: "fetch bazel" + - func: "bazel run" + vars: + target: >- + //:codeowners -- --check + - name: lint_clang_format tags: ["assigned_to_jira_team_devprod_build", "development_critical_single_variant", "lint"] commands: