SERVER-102731: Add OSS LLVM Clang in MacOS builds. (#36708)

GitOrigin-RevId: 73159ed1ec73a7516e478f5cd4a1e924166b6b4f

.bazelrc

@@ -62,6 +62,9 @@ test --build_tests_only
 
 common:macos --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1
 
+# Pin down the OSS LLVM Clang version for MacOS builds.
+common:macos --repo_env=LLVM_VERSION=19
+
 common:windows --features=-compiler_param_file
 
 
@@ -135,6 +138,7 @@ common --flag_alias=dbg_level=//bazel/config:dbg_level
 common --flag_alias=mongo_toolchain_version=//bazel/config:mongo_toolchain_version
 common --flag_alias=simple_build_id=//bazel/config:simple_build_id
 common --flag_alias=create_dwp=//bazel/config:create_dwp
+common --flag_alias=local_clang_compiler=//bazel/config:local_clang_compiler
 
 #############################################################################################################################
 # BUILD 'PROFILES' - this is the area to set up configurations of flags to be used by developers.

BUILD.bazel

@@ -1,6 +1,7 @@
 load("@npm//:defs.bzl", "npm_link_all_packages")
 load("@aspect_rules_js//npm:defs.bzl", "npm_link_package")
 load("//bazel/install_rules:install_rules.bzl", "TEST_TAGS", "mongo_install")
+load("//bazel:mongo_src_rules.bzl", "mongo_cc_binary")
 load("//bazel/toolchains/cc/mongo_linux:mongo_toolchain.bzl", "setup_mongo_toolchain_aliases")
 load("//bazel/config:render_template.bzl", "render_template")
 load("@npm//:eslint/package_json.bzl", eslint_bin = "bin")

MODULE.bazel

@@ -139,12 +139,16 @@ register_execution_platforms("@internal_platforms_do_not_use//host:host")
 #
 mongo_apple_toolchain_config = use_repo_rule("//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "mongo_apple_toolchain_config")
 
-mongo_apple_toolchain_setup = use_repo_rule("//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "mongo_apple_toolchain_setup")
-
 mongo_apple_toolchain_config(name = "mongo_apple_toolchain_config")
 
+mongo_apple_toolchain_setup = use_repo_rule("//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "mongo_apple_toolchain_setup")
+
 mongo_apple_toolchain_setup(name = "mongo_apple_toolchain")
 
+mongo_apple_brew_llvm_toolchain_config = use_repo_rule("//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "mongo_apple_brew_llvm_toolchain_config")
+
+mongo_apple_brew_llvm_toolchain_config(name = "mongo_apple_brew_llvm_toolchain_config")
+
 register_toolchains(
     "@mongo_apple_toolchain//...",
 )

bazel/config/BUILD.bazel

@@ -26,6 +26,7 @@ load(
     "libunwind",
     "linker",
     "linkstatic",
+    "local_clang_compiler",
     "lsan",
     "mongo_toolchain_version",
     "msan",
@@ -91,6 +92,18 @@ config_setting(
     },
 )
 
+local_clang_compiler(
+    name = "local_clang_compiler",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "use_local_clang_compiler",
+    flag_values = {
+        "//bazel/config:local_clang_compiler": "True",
+    },
+)
+
 selects.config_setting_group(
     name = "gcc_or_clang",
     match_any = [

bazel/config/configs.bzl

@@ -15,6 +15,7 @@ def compiler_type_impl(ctx):
     compiler_type_value = ctx.build_setting_value
     if compiler_type_value not in compiler_type_values:
         fail(str(ctx.label) + " compiler_type allowed to take values {" + ", ".join(compiler_type_values) + "} but was set to unallowed value " + compiler_type_value)
+
     return compiler_type_provider(compiler_type = compiler_type_value)
 
 compiler_type = rule(
@@ -22,6 +23,20 @@ compiler_type = rule(
     build_setting = config.string(flag = True),
 )
 
+# =============
+# compiler_type
+# =============
+
+local_clang_compiler_provider = provider(
+    doc = "use the local clang compiler",
+    fields = ["enabled"],
+)
+
+local_clang_compiler = rule(
+    implementation = lambda ctx: local_clang_compiler_provider(enabled = ctx.build_setting_value),
+    build_setting = config.bool(flag = True),
+)
+
 # =========
 # mongo_toolchain_version
 # =========

bazel/toolchains/cc/mongo_apple/BUILD_llvm.tpl

@@ -0,0 +1,69 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@//bazel/config:configs.bzl", "sdkroot")
+load("@//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "get_supported_apple_archs")
+load("@//bazel/toolchains/cc/mongo_apple:mongo_apple_llvm_cc_toolchain_config.bzl", "mongo_apple_llvm_cc_toolchain_config")
+load(
+    "@//bazel/toolchains/cc:mongo_custom_features.bzl",
+    "FEATURES_ATTR_NAMES",
+    "get_common_features_attrs")
+
+# Helper target for the toolchain (see below):
+filegroup(
+    name = "all_files",
+    srcs = glob(["**/*"]),
+)
+
+[
+    cc_toolchain(
+        name = "cc-compiler-" + arch,
+        all_files = ":all_files",
+        ar_files = ":all_files",
+        as_files = ":all_files",
+        compiler_files = ":all_files",
+        dwp_files = ":empty",
+        linker_files = ":all_files",
+        objcopy_files = ":empty",
+        strip_files = ":all_files",
+        supports_header_parsing = 1,
+        supports_param_files = 1,
+        toolchain_config = "llvm_" + arch,
+        toolchain_identifier = "apple_llvm_clang_toolchain_" + arch,
+    )
+    for arch, cpu in get_supported_apple_archs().items()
+]
+
+feature_attrs = get_common_features_attrs()
+
+[
+    mongo_apple_llvm_cc_toolchain_config(
+        name = "llvm_" + arch,
+        cpu = "darwin",
+        compiler = "clang",
+        toolchain_identifier = "apple_llvm_clang_toolchain_" + arch,
+        target_libc = "macosx",
+        abi_version = "darwin_" + arch,
+        abi_libc_version = "darwin_" + arch,
+        cxx_builtin_include_directories = [
+            %{cxx_builtin_include_directories}
+        ],
+        tool_paths = {
+            "ar": "%{llvm_path}/bin/llvm-ar",
+            "cpp": "%{llvm_path}/bin/clang-cpp",
+            "ld":  "%{lld_path}/bin/ld.lld",
+            "dwp": "%{llvm_path}/bin/llvm-dwp",
+            "gcc": "%{llvm_path}/bin/clang",
+            "g++": "%{llvm_path}/bin/clang++",
+            "gcov": "%{llvm_path}/bin/llvm-profdata",
+            "llvm-cov": "%{llvm_path}/bin/llvm-cov",
+            "llvm-profdata": "%{llvm_path}/bin/llvm-profdata",
+            "nm": "%{llvm_path}/bin/llvm-nm",
+            "objcopy": "%{llvm_path}/bin/llvm-objcopy",
+            "objdump": "%{llvm_path}/bin/llvm-objdump",
+            "strip": "%{llvm_path}/bin/llvm-strip",
+        },
+        builtin_sysroot = "@//bazel/config:sdkroot",
+        optimization_level = feature_attrs[FEATURES_ATTR_NAMES.OPT_LEVEL],
+    )
+    for arch, cpu in get_supported_apple_archs().items()
+]

bazel/toolchains/cc/mongo_apple/mongo_apple_toolchain.BUILD.tmpl

@@ -13,7 +13,10 @@ package(default_visibility = ["//visibility:public"])
             "@platforms//os:macos",
             "@platforms//cpu:" + cpu,
         ],
-        toolchain = "@mongo_apple_toolchain_config//:cc-compiler-" + apple_arch,
+        toolchain = select({
+            "@//bazel/config:use_local_clang_compiler": "@mongo_apple_brew_llvm_toolchain_config//:cc-compiler-" + apple_arch,
+            "@//conditions:default": "@mongo_apple_toolchain_config//:cc-compiler-" + apple_arch,
+        }),
         toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
     )
     for apple_arch, cpu in get_supported_apple_archs().items()

bazel/toolchains/cc/mongo_apple/mongo_apple_toolchain.bzl

@@ -275,7 +275,135 @@ exit $?
 
 ######
 # mongodb customization
-# everything below is modifications for mongodb build
+# Everything below is modifications for mongodb build.
+# We are going through a transition of using the LLVM
+# clang while the Apple Clang is still the default. Once
+# the transition is completed, we will remove the Apple
+# Clang toolchain.
+
+def _get_llvm_info(repository_ctx, build_file):
+    llvm_version = repository_ctx.os.environ.get("LLVM_VERSION") or ""
+
+    if llvm_version == "":
+        error_message = """
+The Apple LLVM Clang toolchain has not been defined. Please make sure
+that LLVM_VERSION has been defined in //.bazelrc.local or //.bazelrc file."""
+        return False, "", "", error_message
+
+    brew_command = [
+        "/bin/bash",
+        "-c",
+        "brew --prefix llvm@{}".format(llvm_version),
+    ]
+    result = repository_ctx.execute(brew_command)
+    if result.return_code != 0:
+        error_message = """
+Unable to find the prefix LLVM path using brew command: {}. Please make
+sure that you have installed the LLVM toolchain using Homebrew:
+    `brew install llvm@<version>`
+or update the LLVM_VERSION in the //.bazelrc file or //.bazelrc.local.""".format(" ".join(brew_command))
+        return False, "", "", error_message
+    llvm_path = result.stdout.strip()
+
+    # The path needs to be validated to ensure that it exists.
+    # The error message is injected in the build file as users may not
+    # have installed the LLVM toolchain yet. If the user attempt to use
+    # it, a build error will be raised with the message.
+    if not repository_ctx.path(llvm_path).exists:
+        error_message = """
+You have specified to use the local clang --local_clang_compiler but unable to find the LLVM path:
+    {}.
+Please make sure that you have installed the LLVM toolchain using Homebrew: 
+    `brew install llvm@<version>`.""".format(llvm_path)
+        return False, "", "", error_message
+
+    # Find the real path to the LLVM installation as we need to include the LLVM
+    # lib and headers directories as part of built-in directories.
+    command = [
+        "/bin/bash",
+        "-c",
+        "readlink -f {}".format(llvm_path),
+    ]
+    result = repository_ctx.execute(command)
+    if result.return_code != 0:
+        return False, "", "", "Failed to find the true LLVM path using command: {}".format(" ".join(command))
+    llvm_path = result.stdout.strip()
+
+    return True, llvm_path, llvm_version, ""
+
+def _get_lld_info(repository_ctx, llvm_version):
+    brew_command = [
+        "/bin/bash",
+        "-c",
+        "brew --prefix lld@{}".format(llvm_version),
+    ]
+    result = repository_ctx.execute(brew_command)
+    if result.return_code != 0:
+        return False, "", "Failed to find the prefix of LLD path using brew command: {}".format(" ".join(brew_command))
+    lld_path = result.stdout.strip()
+
+    if not repository_ctx.path(lld_path).exists:
+        return False, "", "The LLD_PATH does not exist: {}".format(lld_path)
+
+    return True, lld_path, ""
+
+def _get_llvm_clang_include_dirs(repository_ctx, llvm_path):
+    include_dirs = [
+        "/Applications/",
+        "/Library",
+    ]
+
+    user = repository_ctx.os.environ.get("USER")
+    if user:
+        include_dirs.extend([
+            "/Users/{}/Applications/".format(user),
+            "/Users/{}/Library/".format(user),
+        ])
+
+    for include_dir in ["include", "lib"]:
+        include_dirs.append(llvm_path + "/" + include_dir)
+
+    ret_include_dirs = []
+    for path in include_dirs:
+        ret_include_dirs.append(("            \"%s\"," % path))
+
+    return ret_include_dirs
+
+def _configure_oss_clang_toolchain(repository_ctx):
+    build_file = "BUILD.bazel"
+
+    success, llvm_path, llvm_version, error = _get_llvm_info(repository_ctx, build_file)
+    if not success:
+        repository_ctx.file(
+            build_file,
+            "fail(\"\"\"%s\"\"\")" % error,
+        )
+        return False
+
+    success, lld_path, error = _get_lld_info(repository_ctx, llvm_version)
+    if not success:
+        repository_ctx.file(
+            build_file,
+            "fail(\"\"\"%s\"\"\")" % error,
+        )
+        return False
+
+    include_dirs = _get_llvm_clang_include_dirs(repository_ctx, llvm_path)
+
+    repository_ctx.report_progress("Generating Apple OSS LLVM Clang Toolchain build file")
+    build_template = Label("@//bazel/toolchains/cc/mongo_apple:BUILD_llvm.tpl")
+    repository_ctx.template(
+        build_file,
+        build_template,
+        {
+            "%{llvm_path}": llvm_path,
+            "%{lld_path}": lld_path,
+            "%{cxx_builtin_include_directories}": "\n".join(include_dirs),
+        },
+    )
+
+    return True, ""
+
 def _apple_cc_autoconf_impl(repository_ctx):
     if repository_ctx.os.name.startswith("mac os"):
         success, error = configure_osx_toolchain(repository_ctx)
@@ -295,11 +423,31 @@ mongo_apple_toolchain_config = repository_rule(
         "USE_CLANG_CL",  # Kept as a hack for those who rely on this invaliding the toolchain
         "USER",  # Used to allow paths for custom toolchains to be used by C* compiles
         "XCODE_VERSION",  # Force re-computing the toolchain by including the current Xcode version info in an env var
+        "LLVM_PATH",  # Force re-compute if the user changed the location of the LLVM toolchain
+        "LLVM_VERSION",  # Force re-compute if the user changed the version of the LLVM toolchain
     ],
     implementation = _apple_cc_autoconf_impl,
     configure = True,
 )
 
+def _apple_llvm_clang_cc_autoconf_impl(repository_ctx):
+    """Configures the Apple LLVM Clang toolchain."""
+    if repository_ctx.os.name.startswith("mac os"):
+        # No failure is shown to the user as the toolchain is still being worked on it.
+        _configure_oss_clang_toolchain(repository_ctx)
+    else:
+        repository_ctx.file("BUILD", "# Apple OSS LLVM Clang autoconfiguration was disabled because you're not on macOS")
+
+mongo_apple_brew_llvm_toolchain_config = repository_rule(
+    environ = [
+        "LLVM_PATH",  # Force re-compute if the user changed the location of the LLVM toolchain
+        "LLVM_VERSION",  # Force re-compute if the user changed the version of the LLVM toolchain
+    ],
+    implementation = _apple_llvm_clang_cc_autoconf_impl,
+    configure = True,
+    local = True,
+)
+
 _ARCH_MAP = {
     "aarch64": "@platforms//cpu:arm64",
     "x86_64": "@platforms//cpu:x86_64",

src/third_party/snappy/BUILD.bazel

@@ -52,6 +52,13 @@ mongo_cc_library(
         "//bazel/config:not_windows": ["platform/build_posix"],
         "//conditions:default": [],
     }),
+    linkopts = select({
+        "@platforms//os:macos": [
+            "-lc++",
+            "-lc++abi",
+        ],
+        "//conditions:default": [],
+    }),
     local_defines = ["HAVE_CONFIG_H"],
     deps = [],
 )