SERVER-118429 Add resmoke_suite_test assignment tags based on code owner (#48079)

GitOrigin-RevId: 22e946575188d32e9a479143bd543779fc89a9d9

MODULE.bazel

@@ -286,6 +286,12 @@ setup_bolt_data = use_repo_rule("//bazel/repository_rules:bolt_data.bzl", "setup
 
 setup_bolt_data(name = "bolt_data")
 
+mothra_repository = use_repo_rule("//bazel/repository_rules:mothra.bzl", "mothra_repository")
+
+# This repository is created in CI setup or by manually cloning the 10gen/mothra repo to ./mothra.
+# If the directory doesn't exist, a stub with empty teams will be created.
+mothra_repository(name = "mothra")
+
 setup_mongo_windows_toolchains_extension = use_extension("//bazel/toolchains/cc/mongo_windows:mongo_toolchain.bzl", "setup_mongo_windows_toolchain_extension")
 use_repo(setup_mongo_windows_toolchains_extension, "mongo_windows_toolchain")
 

bazel/repository_rules/mothra.bzl

@@ -0,0 +1,52 @@
+"""Repository rule for optionally loading mothra teams data."""
+
+def _mothra_repo(ctx):
+    """Creates a repository for mothra teams data.
+
+    If the mothra directory exists, it will be used.
+    Otherwise, a stub with empty teams will be created.
+    """
+    mothra_path = ctx.path(ctx.workspace_root).get_child("mothra")
+
+    if mothra_path.exists:
+        # Mothra directory exists, symlink to it
+        for item in mothra_path.readdir():
+            ctx.symlink(item, item.basename)
+
+        ctx.file(
+            "BUILD.bazel",
+            """
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "teams",
+    srcs = glob(["mothra/teams/*.yaml"]),
+)
+""",
+        )
+    else:
+        # Create a stub team file to satisfy runfiles resolution
+        ctx.file(
+            "mothra/teams/devprod.yaml",
+            """# This is an intentionally empty stub. The real mothra repo was not cloned. For the real team mappings to be used, clone 10gen/mothra into mothra.
+teams: []
+""",
+        )
+
+        ctx.file(
+            "BUILD.bazel",
+            """
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "teams",
+    srcs = glob(["mothra/teams/*.yaml"]),
+)
+""",
+        )
+
+mothra_repository = repository_rule(
+    implementation = _mothra_repo,
+    local = True,
+    configure = True,
+)

buildscripts/BUILD.bazel

@@ -303,6 +303,10 @@ py_binary(
 py_binary(
     name = "generate_result_tasks",
     srcs = ["generate_result_tasks.py"],
+    data = [
+        "@codeowners_binary//:codeowners",
+        "@mothra//:teams",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         dependency(
@@ -313,6 +317,10 @@ py_binary(
             "shrub-py",
             group = "testing",
         ),
+        dependency(
+            "bazel-runfiles",
+            group = "testing",
+        ),
     ],
 )
 

buildscripts/generate_result_tasks.py

@@ -14,12 +14,17 @@ Options:
     --outfile           File path for the generated task config.
 """
 
+import glob
 import json
 import os
 import subprocess
-from typing import List
+import sys
+from functools import cache
+from typing import List, Optional
 
+import runfiles
 import typer
+import yaml
 from shrub.v2 import FunctionCall, Task
 from typing_extensions import Annotated
 
@@ -33,7 +38,97 @@ def make_results_task(target: str) -> Task:
         FunctionCall("fetch remote test results", {"test_label": target}),
     ]
 
-    return Task(target, commands)
+    task = Task(target, commands).as_dict()
+
+    tag = get_assignment_tag(target)
+    if tag:
+        task["tags"] = [tag]
+
+    return task
+
+
+def get_assignment_tag(target: str) -> Optional[str]:
+    # Format is like "assigned_to_jira_team_devprod_build".
+    # See also docs/evergreen-testing/yaml_configuration/task_ownership_tags.md
+
+    assignment_tags = resolve_assignment_tags()
+    tags = set()
+    for codeowner in get_codeowners(target):
+        if codeowner in assignment_tags:
+            tags.add(assignment_tags[codeowner])
+    if len(tags) > 1:
+        print(
+            f"Target {target} has {len(tags)} possible assignment tags based on it's codeowner: {tags}. Picking the first encountered.",
+            file=sys.stderr,
+        )
+    return list(tags)[0] if tags else None
+
+
+def get_codeowners(target: str) -> list[str]:
+    package = target.split(":", 1)[0]
+    return resolve_codeowners().get(package)
+
+
+@cache
+def resolve_assignment_tags() -> dict[str, str]:
+    try:
+        # Find the teams directory in the runfiles. Unfortunately, resolving the
+        # directory requires resolving a specific file within the runfiles, so
+        # an arbitrary team's YAML is used.
+        r = runfiles.Create()
+        teams_dir = os.path.dirname(r.Rlocation("mothra/mothra/teams/devprod.yaml"))
+
+        teams = []
+        for file in glob.glob(teams_dir + "/*.yaml"):
+            with open(file, "rt") as f:
+                teams += yaml.safe_load(f).get("teams", [])
+
+        assignment_tags = {}
+        for team in teams:
+            evergreen_tag_name = team.get("evergreen_tag_name")
+            github_teams = team.get("code_owners", {}).get("github_teams", [])
+            for github_team in github_teams:
+                name = github_team.get("team_name")
+                if name and evergreen_tag_name:
+                    assignment_tags[name] = "assigned_to_jira_team_" + evergreen_tag_name
+        return assignment_tags
+    except Exception as e:
+        # Conservatively except any exception here. In the worst case, the contents/format of the
+        # Mothra repo could change out from under us, and it should not completely fail
+        # task generation.
+        print(f"Failed to resolve assignment tags: {e}", file=sys.stderr)
+        return {}
+
+
+@cache
+def resolve_codeowners() -> dict[str, list[str]]:
+    try:
+        result = subprocess.run(
+            'find * -name "BUILD.bazel" | xargs bazel run @codeowners_binary//:codeowners --',
+            shell=True,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+
+        codeowners_map = {}
+        for line in result.stdout.strip().split("\n"):
+            if not line.strip():
+                continue
+            # Each line is formatted like: "./buildscripts/BUILD.bazel     @owner1 @owner2 ..."
+            words = line.split()
+            package = "//" + words[0].removeprefix("./").removesuffix("/BUILD.bazel")
+            # Remove teams that don't provide a meaningful mapping to a real owner.
+            owners = set(words[1:])
+            owners.difference_update({"@svc-auto-approve-bot", "@10gen/mongo-default-approvers"})
+
+            codeowners_map[package] = [owner.removeprefix("@") for owner in owners]
+        return codeowners_map
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to resolve codeowners: {e.returncode}", file=sys.stderr)
+        print(f"STDOUT:\n{e.stdout}", file=sys.stderr)
+        print(f"STDERR:\n{e.stderr}", file=sys.stderr)
+        return {}
 
 
 def query_targets() -> List[str]:
@@ -61,7 +156,7 @@ def main(outfile: Annotated[str, typer.Option()]):
     test_targets = query_targets()
 
     tasks = [make_results_task(target) for target in test_targets]
-    project = {"tasks": [task.as_dict() for task in tasks]}
+    project = {"tasks": [task for task in tasks]}
 
     with open(outfile, "w") as f:
         f.write(json.dumps(project, indent=4))

docs/evergreen-testing/yaml_configuration/task_ownership_tags.md

@@ -12,3 +12,5 @@ If the linter configuration is missing your team:
 1. Make sure that your team configuration exists or add it in mothra
 2. Make sure that your team configuration in mothra has `evergreen_tag_name`
 3. Update the tag list with `assigned_to_jira_team_{evergreen_tag_name}` tag for your team
+
+Dynamically generated tasks for resmoke suites (i.e. the ones named like `//buildscripts/resmokeconfig:core`) will set the ownership tag based on a best effort lookup from the codeowner of the test's definition to a team name from mothra, picking the first encountered in case of multiple possible assignments.

etc/evergreen_yml_components/configuration.yml

@@ -104,6 +104,11 @@ modules:
     branch: master
     ref: v0.1.1
     auto_update: true
+  - name: mothra
+    owner: 10gen
+    repo: mothra
+    branch: main
+    prefix: ${workdir}/src
 
 # Pre task steps
 pre:

etc/evergreen_yml_components/variants/misc/task_generation.yml

@@ -11,6 +11,8 @@ buildvariants:
     activate: true
     run_on:
       - rhel8.8-medium
+    modules:
+      - mothra
     tasks:
       - name: version_gen
       - name: version_burn_in_gen

poetry.lock

@@ -99,6 +99,18 @@ files = [
 docs = ["furo", "jaraco.packaging (>=9.3)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
 testing = ["jaraco.test", "pytest (!=8.0.*)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"]
 
+[[package]]
+name = "bazel-runfiles"
+version = "1.8.3"
+description = ""
+optional = false
+python-versions = ">=3.7"
+groups = ["testing"]
+markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\""
+files = [
+    {file = "bazel_runfiles-1.8.3-py3-none-any.whl", hash = "sha256:57a2cc04e0b924606e8dd70fc8b31d157db43680a300f546dc60207b5ce7ca82"},
+]
+
 [[package]]
 name = "blinker"
 version = "1.9.0"
@@ -5884,4 +5896,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<4.0"
-content-hash = "c4a064d90a866bf24bdc2d77d8f10147a1a6cf2d94f193e064cf8405250449cd"
+content-hash = "33a897a6d5a0482f8d4d9378582c1f3a732d74fae6068e9b80a9ffb6cc7a3873"

pyproject.toml

@@ -142,6 +142,7 @@ python-msilib = { version = "^0.5.0", markers = "sys_platform == 'win32' and pyt
 cryptography = "^44.0.2"
 
 [tool.poetry.group.testing.dependencies]
+bazel-runfiles = "^1.8.3"
 curatorbin = "^1.2.4"
 PyKMIP = {git = "https://github.com/mongodb-forks/PyKMIP.git", rev = "c48cb01635819e478b573e3245ef840a11d78865"}
 kafka-python = "^2.0.2"