SERVER-101694 switch to bazel clang tidy in evergreen (#33061)

GitOrigin-RevId: 23fcb16382bb8962d5648adfa7a7a2a828c555ec

buildscripts/apply_clang_tidy_fixes.py

@@ -5,6 +5,7 @@ import argparse
 import hashlib
 import json
 import os
+import re
 import sys
 from collections import defaultdict
 
@@ -92,7 +93,15 @@ def _combine_errors(dir: str) -> str:
                 fix_msg = fix["DiagnosticMessage"]
             fix_data = (
                 all_fixes.setdefault(fix["DiagnosticName"], {})
-                .setdefault(fix_msg.get("FilePath", "FilePath Not Found"), {})
+                .setdefault(
+                    re.sub(
+                        "^.*/src/mongo/",
+                        "src/mongo/",
+                        fix_msg.get("FilePath", "FilePath Not Found"),
+                        1,
+                    ),
+                    {},
+                )
                 .setdefault(
                     str(fix_msg.get("FileOffset", "FileOffset Not Found")),
                     {
@@ -103,13 +112,23 @@ def _combine_errors(dir: str) -> str:
                     },
                 )
             )
+
             for replacement in fix_data["replacements"]:
-                if replacement.get("FilePath") and os.path.exists(replacement.get("FilePath")):
-                    with open(replacement.get("FilePath"), "rb") as contents:
-                        replacement["FileContentsMD5"] = hashlib.md5(contents.read()).hexdigest()
+                if replacement.get("FilePath"):
+                    file_path = re.sub(
+                        "^.*/src/mongo/", "src/mongo/", replacement.get("FilePath"), 1
+                    )
+                    replacement["FilePath"] = file_path
+                    if os.path.exists(file_path):
+                        with open(file_path, "rb") as contents:
+                            replacement["FileContentsMD5"] = hashlib.md5(
+                                contents.read()
+                            ).hexdigest()
 
             fix_data["count"] += 1
-            fix_data["source_files"].append(fixes["MainSourceFile"])
+            fix_data["source_files"].append(
+                re.sub("^.*/src/mongo/", "src/mongo", fixes["MainSourceFile"], 1)
+            )
 
     fixes_file = os.path.join(dir, "clang_tidy_fixes.json")
     with open(fixes_file, "w") as files_file:

etc/evergreen_yml_components/configuration.yml

@@ -103,7 +103,6 @@ post:
   - func: "upload docker compose"
   - func: "attach docker logs"
   - func: "upload jstestfuzz minimized output"
-  - func: "upload clang tidy results"
   - func: "generate and upload network diagnostics"
   - func: "kill processes"
   - func: "save code coverage data"

etc/evergreen_yml_components/definitions.yml

@@ -687,43 +687,6 @@ functions:
       content_type: text/plain
       display_name: Pip Requirements
 
-  "run clang tidy":
-    - *f_expansions_write
-    - command: subprocess.exec
-      type: test
-      params:
-        binary: bash
-        add_expansions_to_env: true
-        args:
-          - "src/evergreen/run_python_script.sh"
-          - "buildscripts/clang_tidy.py"
-          - "--split-jobs=4"
-          - "--split=${split}"
-
-  "upload clang tidy results":
-    - command: s3.put
-      display_name: "upload clang_tidy_fixes.tgz"
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/clang_tidy_fixes.tgz
-        remote_file: ${project}/${build_variant}/${version_id}/${task_id}/clang_tidy_fixes.tgz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: application/gzip
-        display_name: clang_tidy_fixes.tgz
-    - command: s3.put
-      display_name: "upload clang_tidy_fixes.json"
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/clang_tidy_fixes/clang_tidy_fixes.json
-        remote_file: ${project}/${build_variant}/${version_id}/${task_id}/clang_tidy_fixes.json
-        bucket: mciuploads
-        permissions: public-read
-        content_type: application/json
-        display_name: clang_tidy_fixes.json
-
   "generate network diagnostics": &generate_network_diagnostics
     command: subprocess.exec
     display_name: "get network diagnostics"
@@ -1497,6 +1460,20 @@ functions:
     - *f_expansions_write
     - *bazel_compile_sh
 
+  "generate clang-tidy report sh": &generate_clang_tidy_report_sh
+    command: subprocess.exec
+    display_name: "generate clang-tidy report"
+    type: test
+    params:
+      binary: bash
+      args:
+        - "src/evergreen/run_python_script.sh"
+        - "evergreen/generate_clang_tidy_report.py"
+
+  "generate clang-tidy report":
+    - *f_expansions_write
+    - *generate_clang_tidy_report_sh
+
   "bazel run sh": &bazel_run_sh
     command: subprocess.exec
     display_name: "bazel run sh"

etc/evergreen_yml_components/tasks/compile_tasks.yml

@@ -110,17 +110,6 @@ variables:
       - func: "run hang analyzer"
       - func: "wait for resmoke to shutdown"
 
-  - &clang_tidy_scons_compile_vars
-    task_compile_flags: >-
-      --build-profile=compiledb
-    targets: compiledb +mongo-tidy-tests
-    compiling_for_test: true
-    compile_flags: >-
-      --link-model=dynamic
-      -j$(grep -c ^processor /proc/cpuinfo)
-      --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars
-    show_scons_timings: false
-
   - &clang_tidy_task_template
     tags:
       [
@@ -385,11 +374,13 @@ tasks:
     name: run_bazel_clang_tidy
     commands:
       - func: "do scons setup"
-      - func: "bazel compile (clang)"
+      - func: "bazel compile"
         vars:
           # Note that we only run clang-tidy on the Mongo-authored targets, not on third-party code.
           targets: //src/mongo/...
-          args: --config=clang-tidy --compiler_type=clang
+          args: --config=clang-tidy-${clang_tidy_toolchain|v4} --mongo_toolchain_version=${clang_tidy_toolchain|v4} --keep_going
+          run_for_clang_tidy: "true"
+      - func: "generate clang-tidy report"
 
   - name: run_bazel_program_windows
     tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
@@ -551,50 +542,6 @@ tasks:
           skip_archive: on
           bazel_build_tags: --bazel-build-tag=-mongo_unittest --bazel-build-tag=mongo_binary --bazel-build-tag=mongo_benchmark --bazel-build-tag=mongo_integration_test
 
-  - <<: *clang_tidy_task_template
-    name: clang_tidy_1
-    commands:
-      - func: "do scons setup"
-      - func: "scons compile"
-        vars:
-          <<: *clang_tidy_scons_compile_vars
-      - func: "run clang tidy"
-        vars:
-          split: 1
-
-  - <<: *clang_tidy_task_template
-    name: clang_tidy_2
-    commands:
-      - func: "do scons setup"
-      - func: "scons compile"
-        vars:
-          <<: *clang_tidy_scons_compile_vars
-      - func: "run clang tidy"
-        vars:
-          split: 2
-
-  - <<: *clang_tidy_task_template
-    name: clang_tidy_3
-    commands:
-      - func: "do scons setup"
-      - func: "scons compile"
-        vars:
-          <<: *clang_tidy_scons_compile_vars
-      - func: "run clang tidy"
-        vars:
-          split: 3
-
-  - <<: *clang_tidy_task_template
-    name: clang_tidy_4
-    commands:
-      - func: "do scons setup"
-      - func: "scons compile"
-        vars:
-          <<: *clang_tidy_scons_compile_vars
-      - func: "run clang tidy"
-        vars:
-          split: 4
-
   ## compile_unittests ##
   - name: compile_unittests_future_git_tag_multiversion
     tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]

evergreen/bazel_compile.sh

@@ -112,4 +112,8 @@ for i in {1..5}; do
   echo "Bazel failed to execute, retrying..."
 done
 
+if [ "${run_for_clang_tidy:-false}" == "true" ]; then
+  exit 0
+fi
+
 exit $RET

evergreen/generate_clang_tidy_report.py

@@ -0,0 +1,39 @@
+import os
+import re
+import sys
+
+from buildscripts.simple_report import make_report, put_report, try_combine_reports
+
+failures = []
+
+for root, _, files in os.walk("bazel-bin"):
+    for name in files:
+        if name.endswith(".clang-tidy.status"):
+            with open(os.path.join(root, name)) as f:
+                if f.read().strip() == "1":
+                    tokens = name.split(".")
+                    log_file = os.path.join(root, ".".join(tokens[:-1]) + ".log")
+                    with open(log_file) as log:
+                        content = log.read()
+
+                    # exmaple
+                    # bazel-bin/src/mongo/shell/bazel_clang_tidy_src/mongo/shell/mongo_main.cpp.mongo_main_with_debug.clang-tidy.status
+                    # target is:
+                    # mongo_main.cpp
+                    source_file = ".".join(tokens[:-3])
+
+                    failures.append(
+                        [
+                            os.path.join(
+                                re.sub("^.*/bazel_clang_tidy_src/", "src/", root, 1), source_file
+                            ),
+                            content,
+                        ]
+                    )
+
+for failure in failures:
+    report = make_report(failure[0], failure[1], 1)
+    try_combine_reports(report)
+    put_report(report)
+if failures:
+    sys.exit(1)