mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-103927 switch from scons to bazel (#39284)

Browse Source 
GitOrigin-RevId: 698b83c9b64adf5dfe2b670d16052f24342c04e3
This commit is contained in:
Daniel Moody 2025-07-29 12:18:07 -05:00 committed by MongoDB Bot
parent f78c5baef4
commit e54f728ad4
3228 changed files with 152328 additions and 571010 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.bazelignore
View File

@ -4,7 +4,6 @@ src/third_party/abseil-cpp/dist
src/third_party/protobuf/dist
src/third_party/re2/dist
src/third_party/tcmalloc/dist
src/third_party/mozjs
# Ignore node_modules due to the following error
# ERROR: in verify_node_modules_ignored:

1
.bazeliskrc Normal file
View File

@ -0,0 +1 @@
BAZELISK_BASE_URL=https://mdb-build-public.s3.amazonaws.com/bazel_binary_waterfall_builds/9ea3a8ad9ffc29090d93780658c3ec19e75d9f17

536
.bazelrc
View File

@ -1,92 +1,508 @@
# Reference docs: https://bazel.build/run/bazelrc
# This automatically sets a config based on host platform, ex: --config=macos if the host is macos
build --enable_platform_specific_config=true
# coverity need to use the local command without remote execution auto
# enabled.
info --config=local
# Don't use legacy toolchain resolution.
common --incompatible_enable_cc_toolchain_resolution
# https://github.com/bazelbuild/bazel/issues/8766
# there is WIP to allow users to inject constraints
# this flag is required because the default for BZLMOD is @platform//host:host
build --platforms=@internal_platforms_do_not_use//host
common --enable_workspace=true
# Improves build performance after first clean pull by
# about 1 minute on developer machines
common --experimental_remote_merkle_tree_cache
# Linter will have issues with duplicate configurations due to some of our
# install targets depending on tests that have testonly set. This flag can be removed
# if we stop installing our test targets.
common --experimental_retain_test_configuration_across_testonly
# allow multiple execution infos to be set
common --incompatible_modify_execution_info_additive
# This makes builds more hermetic by preventing environment variables from leaking into the execution of rules
build --incompatible_strict_action_env=true
common --incompatible_strict_action_env=true
# allows us to use python rules. This is needed because of the bug:
# https://github.com/bazelbuild/bazel/issues/4327
build --nobuild_runfile_links
common --experimental_inprocess_symlink_creation
# cc_library transitively propagates link opts. The build uses cc_shared_library to avoid this when linking
# dynamically.
build --experimental_cc_shared_library
common --experimental_cc_shared_library
# This supports the "crosstool" feature (aka building from our toolchain).
# Currently the only platform with a custom toolchain config is linux, use the default
# toolchain_suite elsewhere.
build:linux --crosstool_top=@mongo_toolchain//:toolchain_suite
# Reuse sandboxes to save sandbox execution and deletion times.
common --experimental_reuse_sandbox_directories
# Seeing as we link locally, you'll end up with 99% of the files locally anyways.
# Using remote download all save some build time because it will immediately download an object
# file when it is available, where toplevel will wait until the local linking job needs it
# before starting the download.
common --remote_download_outputs=all
# Files we always want to download for debugging and error checking
common --remote_download_regex=.*\.(dwo|h|cpp)$
# Disable caching of test results by default since many of our tests are non-deterministic.
common --cache_test_results=no
# Stream the test output so users can see the test progression.
common --test_output=streamed
# when using remote asset api, fallback to the direct urls to download
common --experimental_remote_downloader_local_fallback
# Build only what is necessary to run the *_test and test_suite rules that were
# not filtered due to their size, timeout, tag, or language.
test --build_tests_only
# Pin down the OSS LLVM Clang version for MacOS builds.
common:macos --repo_env=LLVM_VERSION=19
# Pin down the Microsoft Visual compiler. If you would like to use
# the default compiler version installed in this host, comment the line.
common:windows --repo_env=BAZEL_VC_FULL_VERSION=14.31.31103
# Default the Visual C Redistribution to v14.3 for Windows installer.
common:windows --repo_env=MONGO_VC_REDIST_FULL_VERSION=v143
# while in hybrid build state, using local unsandboxed linking should be faster. When most of our link
# targets have been converted (i.e. unittest binaries) and we can BWOB, remote linking should be faster
# in those cases:
common --strategy=CppLink=local
common --strategy=CppArchive=local
common --strategy=SolibSymlink=local
common --strategy=StripDebuginfo=local
common --strategy=ExtractDebuginfo=local
common --strategy=TestRunner=local
common --strategy=CoverageReport=local
common --strategy=CcGenerateIntermediateDwp=local
common --strategy=CcGenerateDwp=local
common --modify_execution_info=^(TestRunner|CppLink|CppArchive|SolibSymlink|ExtractDebuginfo|StripDebuginfo|CcGenerateIntermediateDwp|CcGenerateDwp)$=+no-remote-cache
# Global clang tidy flags to avoid invalidating analysis cache between clang-tidy / regular runs
common --@bazel_clang_tidy//:clang_tidy_excludes=".pb.cc,icu_init.cpp"
common --@bazel_clang_tidy//:clang_tidy_config=//:clang_tidy_config_strict
common --@bazel_clang_tidy//:clang_tidy_executable=//:clang_tidy
common --@bazel_clang_tidy//:clang_tidy_additional_deps=//:toolchain_files
common --@bazel_clang_tidy//:clang_tidy_plugin_deps=//src/mongo/tools/mongo_tidy_checks:mongo_tidy_checks
# Aliases for config flags
common --flag_alias=linkstatic=//bazel/config:linkstatic
common --flag_alias=compiler_type=//bazel/config:compiler_type
common --flag_alias=linker=//bazel/config:linker
common --flag_alias=use_gdbserver=//bazel/config:use_gdbserver
common --flag_alias=libunwind=//bazel/config:libunwind
common --flag_alias=spider_monkey_dbg=//bazel/config:spider_monkey_dbg
common --flag_alias=allocator=//bazel/config:allocator
common --flag_alias=use_lldbserver=//bazel/config:use_lldbserver
common --flag_alias=opt=//bazel/config:opt
common --flag_alias=dbg=//bazel/config:dbg
common --flag_alias=debug_symbols=//bazel/config:debug_symbols
common --flag_alias=thin_lto=//bazel/config:thin_lto
common --flag_alias=separate_debug=//bazel/config:separate_debug
common --flag_alias=use_wait_for_debugger=//bazel/config:use_wait_for_debugger
common --flag_alias=use_ocsp_stapling=//bazel/config:use_ocsp_stapling
common --flag_alias=use_disable_ref_track=//bazel/config:use_disable_ref_track
common --flag_alias=use_wiredtiger=//bazel/config:use_wiredtiger
common --flag_alias=use_glibcxx_debug=//bazel/config:use_glibcxx_debug
common --flag_alias=use_tracing_profiler=//bazel/config:use_tracing_profiler
common --flag_alias=build_otel=//bazel/config:build_otel
common --flag_alias=detect_odr_violations=//bazel/config:detect_odr_violations
common --flag_alias=shared_archive=//bazel/config:shared_archive
common --flag_alias=skip_archive=//bazel/config:skip_archive
common --flag_alias=streams_release_build=//bazel/config:streams_release_build
common --flag_alias=disable_streams=//bazel/config:disable_streams
common --flag_alias=release=//bazel/config:release
common --flag_alias=build_enterprise=//bazel/config:build_enterprise
common --flag_alias=visibility_support=//bazel/config:visibility_support
common --flag_alias=disable_warnings_as_errors=//bazel/config:disable_warnings_as_errors
common --flag_alias=gcov=//bazel/config:gcov
common --flag_alias=pgo_profile=//bazel/config:pgo_profile
common --flag_alias=server_js=//bazel/config:server_js
common --flag_alias=ssl=//bazel/config:ssl
common --flag_alias=js_engine=//bazel/config:js_engine
common --flag_alias=use_sasl_client=//bazel/config:use_sasl_client
common --flag_alias=enterprise_feature_all=//bazel/config:enterprise_feature_all
common --flag_alias=link_timeout=//bazel/config:link_timeout
common --flag_alias=compress_debug_compile=//bazel/config:compress_debug_compile
common --flag_alias=include_mongot=//bazel/config:include_mongot
# This flag will only work if you pass it on the command line as it is parsed by bazelisk pre-bazel script
common --flag_alias=include_autogenerated_targets=//bazel/config:include_autogenerated_targets
common --flag_alias=bolt=//bazel/config:bolt
common --flag_alias=dwarf_version=//bazel/config:dwarf_version
common --flag_alias=http_client=//bazel/config:http_client
common --flag_alias=asan=//bazel/config:asan
common --flag_alias=fsan=//bazel/config:fsan
common --flag_alias=msan=//bazel/config:msan
common --flag_alias=lsan=//bazel/config:lsan
common --flag_alias=tsan=//bazel/config:tsan
common --flag_alias=ubsan=//bazel/config:ubsan
common --flag_alias=dbg_level=//bazel/config:dbg_level
common --flag_alias=mongo_toolchain_version=//bazel/config:mongo_toolchain_version
common --flag_alias=simple_build_id=//bazel/config:simple_build_id
common --flag_alias=create_dwp=//bazel/config:create_dwp
common --flag_alias=win_min_version=//bazel/config:win_min_version
#############################################################################################################################
# BUILD 'PROFILES' - this is the area to set up configurations of flags to be used by developers.
# --enable_platform_specific_config=true is passed to every profile so more hardware specific
# configurations can override the defaults in these profiles. These represent the only supported flag configurations,
# and should be the only thing passed on the command line in most scenarios
# Every profile should explicitly specify every option used by every other profile, with the exception of the
# profile modifiers
# The base profiles are fast, opt and debug - these reflect the output directories artifacts will get placed into
# and match the bazel compilation modes
# Should a profile modify another profile, it should be named {original_profile}_{modifier_name} - example fast_static
# You can use .bazelrc.local to modify the default build type you want to use locally
# Current Profiles:
# fastbuild
# dbg
# dbg_san
# dbg_tsan
# opt
# Bazel will by default strip on fastbuild, if we want stripping we will handle it ourselves
common --strip=never
# Current default profile - fastbuild - this matches the config options of fastbuild. It does not just set --config=fastbuild
# To avoid the bazel warning about passing multiple configs
common -c fastbuild
common --fission=yes
common --//bazel/config:dwarf_version=5
common --//bazel/config:compress_debug_compile=True
common --//bazel/config:opt=off
common --//bazel/config:dbg=True
common --//bazel/config:dbg_level=1
common --//bazel/config:debug_symbols=True
common --//bazel/config:separate_debug=False
common --//bazel/config:compiler_type=clang
common --//bazel/config:linker=mold
common --//bazel/config:linkstatic=True
common --//bazel/config:build_enterprise=True
common --//bazel/config:skip_archive=True
common --//bazel/config:allocator=auto
common --//bazel/config:asan=False
common --//bazel/config:tsan=False
common --//bazel/config:ubsan=False
common --//bazel/config:libunwind=auto
common --jobs=300
common:macos --jobs=auto
common:windows --jobs=auto
common --enable_platform_specific_config=true
# Profile for building fast with minimal debuggability - the build is fast ##################################################
--config=fastbuild
common:fastbuild -c fastbuild
common:fastbuild --fission=yes
common:fastbuild --//bazel/config:dwarf_version=5
common:fastbuild --//bazel/config:compress_debug_compile=True
common:fastbuild --//bazel/config:opt=off
common:fastbuild --//bazel/config:dbg=True
common:fastbuild --//bazel/config:dbg_level=1
common:fastbuild --//bazel/config:debug_symbols=True
common:fastbuild --//bazel/config:separate_debug=False
common:fastbuild --//bazel/config:compiler_type=clang
common:fastbuild --//bazel/config:linker=mold
common:fastbuild --//bazel/config:linkstatic=True
common:fastbuild --//bazel/config:build_enterprise=True
common:fastbuild --//bazel/config:skip_archive=True
common:fastbuild --//bazel/config:allocator=auto
common:fastbuild --//bazel/config:asan=False
common:fastbuild --//bazel/config:tsan=False
common:fastbuild --//bazel/config:ubsan=False
common:fastbuild --//bazel/config:libunwind=auto
common:fastbuild --enable_platform_specific_config=true
# Profile for building highly debuggable code - the build is slow, the code is slow, the binaries are large #################
--config=dbg
common:dbg -c dbg
common:dbg --fission=yes
common:dbg --//bazel/config:dwarf_version=5
common:dbg --//bazel/config:compress_debug_compile=True
common:dbg --//bazel/config:opt=off
common:dbg --//bazel/config:dbg=True
common:dbg --//bazel/config:dbg_level=2
common:dbg --//bazel/config:debug_symbols=True
common:dbg --//bazel/config:separate_debug=False
common:dbg --//bazel/config:compiler_type=clang
common:dbg --//bazel/config:linker=mold
common:dbg --//bazel/config:linkstatic=True
common:dbg --//bazel/config:build_enterprise=True
common:dbg --//bazel/config:skip_archive=True
common:dbg --//bazel/config:allocator=auto
common:dbg --//bazel/config:asan=False
common:dbg --//bazel/config:tsan=False
common:dbg --//bazel/config:ubsan=False
common:dbg --//bazel/config:libunwind=auto
common:dbg --enable_platform_specific_config=true
# Build with address and undefined sanitizers
--config=dbg_aubsan
common:dbg_aubsan --config=dbg
common:dbg_aubsan --//bazel/config:opt=debug
common:dbg_aubsan --//bazel/config:linkstatic=False
common:dbg_aubsan --//bazel/config:allocator=system
common:dbg_aubsan --//bazel/config:asan=True
common:dbg_aubsan --//bazel/config:ubsan=True
common:dbg_aubsan --enable_platform_specific_config=true
# Build with thread sanitizers
--config=dbg_tsan
common:dbg_tsan --config=dbg
common:dbg_tsan --//bazel/config:opt=debug
common:dbg_tsan --//bazel/config:linkstatic=False
common:dbg_tsan --//bazel/config:allocator=system
common:dbg_tsan --//bazel/config:tsan=True
common:dbg_tsan --//bazel/config:libunwind=off
common:dbg_tsan --//bazel/config:skip_archive=False
common:dbg_tsan --enable_platform_specific_config=true
# Profile for building optimized code - the build is slow, the code is fast #################################################
--config=opt
common:opt -c opt
common:opt --fission=yes
common:opt --//bazel/config:dwarf_version=5
common:opt --//bazel/config:compress_debug_compile=True
common:opt --//bazel/config:opt=on
common:opt --//bazel/config:dbg=False
common:opt --//bazel/config:dbg_level=2
common:opt --//bazel/config:debug_symbols=True
common:opt --//bazel/config:separate_debug=False
common:opt --//bazel/config:compiler_type=gcc
common:opt --//bazel/config:linker=auto
common:opt --//bazel/config:linkstatic=True
common:opt --//bazel/config:build_enterprise=True
common:opt --//bazel/config:skip_archive=True
common:opt --//bazel/config:allocator=auto
common:opt --//bazel/config:asan=False
common:opt --//bazel/config:tsan=False
common:opt --//bazel/config:ubsan=False
common:opt --//bazel/config:libunwind=auto
common:opt --enable_platform_specific_config=true
--config=evg
common:evg --config=opt
common:evg --//bazel/config:opt=auto
common:evg --//bazel/config:separate_debug=True
common:evg --fission=no
common:evg --enable_platform_specific_config=true
--config=evg_crypt
common:evg_crypt --config=evg
common:evg_crypt --//bazel/config:allocator=system
common:evg_crypt --//bazel/config:js_engine=none
common:evg_crypt --//bazel/config:ssl=False
common:evg_crypt --//bazel/config:http_client=False
common:evg_crypt --//bazel/config:shared_archive=True
--config=evg_crypt_dbg
common:evg_crypt_dbg --config=evg_crypt
common:evg_crypt_dbg --//bazel/config:opt=off
common:evg_crypt_dbg --//bazel/config:dbg=True
--config=evg_crypt_test
common:evg_crypt_test --config=evg_crypt
common:evg_crypt_test --//bazel/config:shared_archive=False
--config=evg_stitch
common:evg_stitch --config=evg
common:evg_stitch --//bazel/config:build_enterprise=False
common:evg_stitch --//bazel/config:ssl=False
common:evg_stitch --//bazel/config:http_client=False
common:evg_stitch --//bazel/config:shared_archive=True
--config=evg_stitch_test
common:evg_stitch_test --config=evg_stitch
common:evg_stitch_test --//bazel/config:shared_archive=False
# TODO: Build the code as we would release it
# common:opt_release --config=opt
# common:opt_release --//bazel/config:separate_debug=True
# common:opt_release --//bazel/config:build_enterprise=True
# common:opt_release --//bazel/config:release=True
# TODO: Open Source community build flags
# common:community
#############################################################################################################################
--config=remote_link
common:remote_link --strategy=CppLink=remote
common:remote_link --strategy=CppArchive=remote
common:remote_link --strategy=SolibSymlink=remote
common:remote_link --strategy=ExtractDebugInfo=remote
common:remote_link --strategy=StripDebugInfo=remote
common:remote_link --features=-thin_archive
# Coverage
coverage --config=dbg
coverage --compiler_type=clang
coverage --linkstatic=False
coverage --combined_report=lcov
coverage --experimental_use_llvm_covmap
coverage --experimental_generate_llvm_lcov
coverage --remote_download_outputs=all
coverage --fission=no
# This is set in .bazelrc.evergreen:
# coverage --config=no-remote-exec
# code ownership configuration
common --define codeowners_add_auto_approve_user=True
# Don't detect the native toolchain on linux, only use the hermetic toolchains.
# Opt out of this by passing --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0 on the command line.
common:linux --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
# Only use fission and debug compression on Linux
common:windows --fission=no
common:windows --//bazel/config:create_dwp=False
common:windows --//bazel/config:compress_debug_compile=False
common:macos --fission=no
common:macos --//bazel/config:create_dwp=False
common:macos --//bazel/config:compress_debug_compile=False
# Dynamic linking on Windows (DLL generation) is currently not supported.
build:windows --//bazel/config:linkstatic=True
common:windows --//bazel/config:linkstatic=True
# The only Windows compiler we support is MSVC.
build:windows --//bazel/config:compiler_type=msvc
common:windows --//bazel/config:compiler_type=msvc
common:windows --//bazel/config:linker=auto
# Windows and MacOS do not use the custom toolchain config, so the compiler flags need to be set here.
build:macos --cxxopt=-std=c++20
build:windows --cxxopt=/std:c++20
# Data dependencies don't work in bazel run or bazel test on windows without this
startup --windows_enable_symlinks
common:windows --enable_runfiles
build:windows --action_env=MSYS=winsymlinks:nativestrict
test:windows --action_env=MSYS=winsymlinks:nativestrict
# Set the windows version to win10 by default
# TODO(SERVER-87654): We may want to add support for other Windows versions in the future.
build:windows --cxxopt=-D_WIN32_WINNT=0x0A00
build:windows --cxxopt=-DBOOST_USE_WINAPI_VERSION=0x0A00
build:windows --cxxopt=-DNTDDI_VERSION=0x0A000000
# Always use clang on macos
common:macos --//bazel/config:compiler_type=clang
common:macos --//bazel/config:linker=auto
# remote execution is the default, but only mongodb employees will be able to access
# the engflow cluster. External builders should use the local option below
# remote execution configs
build --remote_executor=grpcs://sodalite.cluster.engflow.com
build --remote_cache=grpcs://sodalite.cluster.engflow.com
build --bes_backend=grpcs://sodalite.cluster.engflow.com
build --bes_results_url=https://sodalite.cluster.engflow.com/invocation/
build --experimental_remote_cache_compression=true
build --grpc_keepalive_time=30s
build --nolegacy_important_outputs
build --bes_keywords=repo:mongo
build --jobs=200
common:macos --macos_minimum_os=11.0
# TODO(SERVER-102959): Find out why tests fail when mac is not in compilation mode dbg
common:macos -c dbg
# Remote execution and caching is the default, but only mongodb employees will be able to access
# the engflow cluster. External builders should use the --config=local option
# Enable remote build execution:
common:linux --remote_executor=grpcs://sodalite.cluster.engflow.com
common:windows --remote_executor=
common:macos --remote_executor=
# Enable remote cache (also necessary for remote build execution):
common --experimental_remote_downloader=grpcs://sodalite.cluster.engflow.com
common --remote_cache=grpcs://sodalite.cluster.engflow.com
common --bes_backend=grpcs://sodalite.cluster.engflow.com
common --bes_results_url=https://sodalite.cluster.engflow.com/invocation/
common --remote_cache_compression=true
common --grpc_keepalive_time=30s
common --nolegacy_important_outputs
common --bes_keywords=repo:mongo
common --remote_upload_local_results=False
# Settings specific for clang-tidy
--config=clang-tidy
common:clang-tidy --config=dbg
common:clang-tidy --remote_download_outputs=all
common:clang-tidy --build_tag_filters=-third_party,-mongo-tidy-tests,-third_party_debug,-mongo-tidy-tests_debug
common:clang-tidy --//bazel/config:compiler_type=clang
common:clang-tidy --keep_going
common:clang-tidy --aspects @bazel_clang_tidy//clang_tidy:clang_tidy.bzl%clang_tidy_aspect
common:clang-tidy --output_groups=report
--config=mod-scanner
common:mod-scanner --config=dbg
common:mod-scanner --build_tag_filters=-third_party,-mongo-tidy-tests,-third_party_debug,-mongo-tidy-tests_debug
common:mod-scanner --//bazel/config:compiler_type=clang
common:mod-scanner --//bazel/config:mongo_toolchain_version=v5
common:mod-scanner --output_groups=report
common:mod-scanner --aspects //modules_poc:mod_scanner.bzl%mod_scanner_aspect
common:mod-scanner --remote_download_regex=.*\.mod_scanner_decls.json$
# if you don't have access to the remote execution cluster above, use the local config
# described below.
# pass local config to SCons like:
# > buildscripts/scons.py BAZEL_FLAGS=--config=local <others args>
# or if invoking bazel directly pass "--config=local" on the bazel command line
# by passing "--config=local" on the bazel command line
--config=local
build:local --remote_executor=
build:local --remote_cache=
build:local --bes_backend=
build:local --bes_results_url=
build:local --tls_client_certificate=
build:local --tls_client_key=
build:local --experimental_remote_cache_compression=false
build:local --grpc_keepalive_time=0s
build:local --legacy_important_outputs
build:local --//bazel/config:local_build=True
build:local --jobs=auto
common:local --remote_executor=
common:local --experimental_remote_downloader=
common:local --remote_cache=
common:local --bes_backend=
common:local --bes_results_url=
common:local --tls_client_certificate=
common:local --tls_client_key=
common:local --remote_cache_compression=false
common:local --grpc_keepalive_time=0s
common:local --legacy_important_outputs
common:local --jobs=auto
common:local --//bazel/config:local_build=True
# Disable remote execution but keep remote cache enabled
--config=no-remote-exec
common:no-remote-exec --remote_executor=
common:no-remote-exec --jobs=auto
common:no-remote-exec --//bazel/config:local_build=True
# Disable remote execution and caching for public releases
--config=public-release
build:public-release --remote_executor=
build:public-release --remote_cache=
build:public-release --bes_backend=
build:public-release --bes_results_url=
build:public-release --tls_client_certificate=
build:public-release --tls_client_key=
build:public-release --experimental_remote_cache_compression=false
build:public-release --grpc_keepalive_time=0s
build:public-release --legacy_important_outputs
common:public-release --remote_executor=
common:public-release --experimental_remote_downloader=
common:public-release --remote_cache=
common:public-release --bes_backend=
common:public-release --bes_results_url=
common:public-release --tls_client_certificate=
common:public-release --tls_client_key=
common:public-release --remote_cache_compression=false
common:public-release --grpc_keepalive_time=0s
common:public-release --legacy_important_outputs
common:public-release --//bazel/config:release=True
common:public-release --//bazel/config:local_build=True
--config=fission
common:fission --fission=yes
common:fission --remote_download_regex=.*\.dwo$
# Avoid failing builds when BES metadata fails to upload.
common --bes_upload_mode=fully_async
# Enable Build without the Bytes
common --remote_download_outputs=toplevel
# Default Mongo Version if a version is not specified.
common --define=MONGO_VERSION=8.0.6
# try to import the bazelrc files if available
# Default distmod if not specified.
common --define=MONGO_DISTMOD=""
# Default if .git directory is not present
common --define=GIT_COMMIT_HASH="nogitversion"
# TODO(WT-12780): delete this once wiredtiger switches to /.bazelrc.evergreen.
try-import %workspace%/.bazelrc.evergreen_engflow_creds
# Evergreen settings, ex. in evergreen this will set the key/cert options for access to the cluster
try-import %workspace%/.bazelrc.evergreen
# local default dev settings
try-import %workspace%/.bazelrc.common_bes
# local git version info
try-import %workspace%/.bazelrc.git
# Used for build profiles and any settings a user wants to consistently use
try-import %workspace%/.bazelrc.local
# in evergreen this will set the key/cert options for access to the cluster
try-import %workspace%/.bazelrc.evergreen_engflow_creds
# Flag as built with bazelisk
try-import %workspace%/.bazelrc.bazelisk
test --test_timeout=900
common:macos --copt=-DBOOST_NO_CXX98_FUNCTION_BASE
common --flag_alias=use_diagnostic_latches=//bazel/config:use_diagnostic_latches

4
.bazelversion
View File

@ -1,3 +1 @@
6.4.0
# TODO(SERVER-86050): s390x and ppc64le are using custom built Bazel v6.4.0 binaries
# Do not update this without updating the custom built binaries until Bazelisk is supported on s390x & ppc64le.
7.5.0-mongo_9ea3a8ad9f

10
.gitattributes vendored
View File

@ -1,3 +1,13 @@
*.vcproj -crlf -diff -merge
*.pbxproj -crlf -diff -merge
*.sh text eol=lf
**/third_party/**/* rules-lint-ignored=true
**/third_party/**/BUILD.bazel rules-lint-ignored=false
external rules-lint-ignored=true
**/*.tpl.h rules-lint-ignored=true
**/*.tpl.cpp rules-lint-ignored=true
src/mongo/bson/util/bson_column_compressed_data.inl rules-lint-ignored=true
src/mongo/bson/util/simple8b.inl rules-lint-ignored=true
*.idl linguist-language=yaml
MODULE.bazel.lock rules-lint-ignored=true linguist-generated

2
.github/CODEOWNERS vendored
View File

@ -1,5 +1,5 @@
# All backports must be approved by the server release team
* @10gen/server-release
* @10gen/devprod-build
# Exclude some test files and READMEs from the backport approvals
/etc/backports_required_for_multiversion_tests.yml

8
.gitignore vendored
View File

@ -286,13 +286,15 @@ buildozer
.bazelrc.bazelisk
*.bazel_info_for_ninja.txt
.ninja_last_command_line_targets.txt
.bazelrc.common_bes
.bazelrc.sync
bazel/coverity/analysis/BUILD.bazel
.bazel_include_info.json
.bazel_header_list_cache
.bazel_real
MODULE.bazel
MODULE.bazel.lock
.compiledb
.mongo_checks_module_path
# generated configs for external fixture suites
docker_compose/

1
.npmrc Normal file
View File

@ -0,0 +1 @@
hoist=false

2
.prettierignore
View File

@ -9,6 +9,8 @@
# We are only going to do markdown file formatting for now
# Hopefully we will use prettier for more file types in the future
!*.md
# TODO SERVER-106654
docs/logging.md
# Ignore all golden test output files
jstests/*golden*/expected_output/*

440
BUILD.bazel
View File

@ -1,16 +1,450 @@
load("@npm//:defs.bzl", "npm_link_all_packages")
load("@aspect_rules_js//npm:defs.bzl", "npm_link_package")
load("//bazel/install_rules:install_rules.bzl", "TEST_TAGS", "mongo_install")
load("//bazel:mongo_src_rules.bzl", "mongo_cc_binary")
load("//bazel/toolchains/cc/mongo_linux:mongo_toolchain.bzl", "setup_mongo_toolchain_aliases")
load("//bazel/config:render_template.bzl", "render_template")
load("@npm//:eslint/package_json.bzl", eslint_bin = "bin")
load("//bazel:mongo_js_rules.bzl", "mongo_js_library")
load("@npm//:prettier/package_json.bzl", prettier = "bin")
load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
package(default_visibility = ["//visibility:public"])
package(
default_visibility = ["//visibility:public"],
)
exports_files([
"buildscripts/idl",
".prettierrc",
"pyproject.toml",
"poetry.lock",
])
npm_link_all_packages(name = "node_modules")
npm_link_package(
name = "node_modules/eslint-plugin-mongodb",
src = "//buildscripts/eslint-plugin-mongodb:npm_package",
)
mongo_js_library(
name = "eslintrc",
srcs = ["eslint.config.mjs"],
deps = [
":node_modules/@eslint/eslintrc",
":node_modules/@eslint/js",
":node_modules/eslint-plugin-mongodb",
":node_modules/globals",
],
)
eslint_bin.eslint_binary(
name = "eslint",
data = [
"//:eslintrc",
],
)
prettier.prettier_binary(
name = "prettier",
# Include this js_library and its dependencies in the runfiles (runtime dependencies)
data = ["//:.prettierrc"],
# Allow the binary to be run outside bazel
# See more details about this by commenting this out and running `bazel run //:format`
env = {"BAZEL_BINDIR": "."},
fixed_args = [
# `require` statements in the config file will be resolved relative to its location
# Therefore to make it hermetic, prettier must be pointed at the copy of the config file
# in the runfiles folder rather than the one in the source folder.
"--config=\"$$JS_BINARY__RUNFILES\"/$(rlocationpath //:.prettierrc)",
# default log level is "log" which spams on success
# https://prettier.io/docs/en/cli.html#--log-level
# NB: prettier 2 names this loglevel, in prettier 3 it's renamed log-level, see
# https://prettier.io/blog/2023/07/05/3.0.0.html#cli-1
"--log-level=warn",
# Speed this up a little bit with caching
"--cache",
],
)
alias(
name = "format",
actual = "//bazel/format",
actual = "//bazel/format:rules_lint_format_wrapper",
)
alias(
name = "lint",
actual = "//bazel:lint",
)
alias(
name = "codeowners",
actual = "@bazel_rules_mongo//codeowners:codeowners",
)
alias(
name = "engflow_auth",
actual = "@bazel_rules_mongo//engflow_auth:engflow_auth",
)
alias(
name = "mongo-tidy-test",
actual = "//src/mongo/tools/mongo_tidy_checks/tests:MongoTidyCheck_unittest",
)
setup_mongo_toolchain_aliases()
render_template(
name = "clang_tidy_config",
srcs = [
".clang-tidy.in",
"//buildscripts:clang_tidy_config_gen.py",
],
cmd = [
"$(location //buildscripts:clang_tidy_config_gen.py)",
"--input=$(location .clang-tidy.in)",
"--output=$(location .clang-tidy)",
],
output = ".clang-tidy",
)
render_template(
name = "clang_tidy_config_strict",
srcs = [
".clang-tidy.in",
"//buildscripts:clang_tidy_config_gen.py",
],
cmd = [
"$(location //buildscripts:clang_tidy_config_gen.py)",
"--input=$(location .clang-tidy.in)",
"--output=$(location .clang-tidy.strict)",
"--warnings-as-errors",
],
output = ".clang-tidy.strict",
)
genrule(
name = "compiledb",
srcs = ["compile_commands.json"],
outs = ["compile_commands_done"],
cmd = "echo noop > $(location :compile_commands_done)",
)
# This sets up targets for install-wiredtiger and archive-wiredtiger
mongo_install(
name = "wiredtiger",
srcs = [
"//src/third_party/wiredtiger:wt",
],
)
# This sets up targets for install-mongod and archive-mongod
mongo_install(
name = "mongod",
srcs = [
"//src/mongo/db:mongod",
] + select({
"@platforms//os:windows": ["@windows_sasl//:bins"],
"//conditions:default": [],
}),
)
# This sets up targets for install-mongos and archive-mongos
mongo_install(
name = "mongos",
srcs = [
"//src/mongo/s:mongos",
],
)
# This sets up targets for install-mongo and archive-mongo
mongo_install(
name = "mongo",
srcs = [
"//src/mongo/shell:mongo",
],
)
# This sets up targets for install-core and archive-core
mongo_install(
name = "core",
srcs = [],
deps = [
"mongod",
"mongos",
],
)
# This sets up targets for install-devcore and archive-devcore
mongo_install(
name = "devcore",
srcs = [],
deps = [
"mongo",
"mongod",
"mongos",
],
)
# This sets up targets for install-mongocryptd and archive-mongocryptd
mongo_install(
name = "mongocryptd",
srcs = [],
deps = [
"//src/mongo/db/modules/enterprise:mongocryptd",
],
)
# This reflects the package that actually gets released during packaging. Be careful
# when changing this definition
# This sets up targets for install-dist and archive-dist
mongo_install(
name = "dist",
srcs = [
"//src/mongo/db:mongod",
"//src/mongo/installer/compass:compass_files",
"//src/mongo/s:mongos",
] + select({
"@platforms//os:windows": ["@local_windows_msvc//:vc_redist_x64"],
"//conditions:default": [],
}),
package_extract_name = select({
"//bazel/config:build_enterprise_linux_enabled": "mongodb-linux-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//bazel/config:build_enterprise_linux_disabled": "mongodb-linux-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//bazel/config:build_enterprise_windows_enabled": "mongodb-win32-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//bazel/config:build_enterprise_windows_disabled": "mongodb-win32-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//bazel/config:build_enterprise_mac_enabled": "mongodb-macos-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//bazel/config:build_enterprise_mac_disabled": "mongodb-macos-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//conditions:default": "mongodb-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
}),
publish_debug_in_stripped = select({
"@platforms//os:windows": True,
"//conditions:default": False,
}),
deps = [
"//src/mongo/db/modules/enterprise:dist",
],
)
filegroup(
name = "archive_license_files",
srcs = [
"//distsrc:mpl",
"//distsrc:readme",
"//distsrc:third_party_notices",
] + select({
"//bazel/config:build_enterprise_enabled": ["//src/mongo/db/modules/enterprise/distsrc:enterprise_license"],
"//conditions:default": ["//distsrc:community_license"],
}) + select({
"//bazel/config:build_enterprise_windows_enabled": ["//src/mongo/db/modules/enterprise/distsrc:windows_notices"],
"//conditions:default": [],
}) + select({
"@platforms//os:macos": ["//etc:mac_plist"],
"//conditions:default": [],
}),
)
copy_to_directory(
name = "mongot_folder",
srcs = select({
"//bazel/config:include_mongot_enabled": ["@mongot_localdev//:mongot_binaries"],
"//conditions:default": [],
}),
out = "mongot-localdev",
include_external_repositories = ["mongot_localdev"],
tags = [
"local",
"no-cache",
],
)
# This sets up targets for install-dist-test and archive-dist-test
mongo_install(
name = "dist-test",
srcs = [
"//src/mongo/db:mongod",
"//src/mongo/db:mongotrafficreader",
"//src/mongo/db/query/optimizer:optimizer_gdb_test_program",
"//src/mongo/db/query/search/mongotmock",
"//src/mongo/db/storage:ksdecode",
"//src/mongo/s:mongos",
"//src/mongo/shell:mongo",
"//src/mongo/tools/mongobridge_tool:mongobridge",
"//src/mongo/util:pretty_printer_test_program",
"//src/third_party/wiredtiger:wt",
] + select({
"@platforms//os:windows": ["@windows_sasl//:bins"],
"//conditions:default": [],
}) + select({
"//bazel/config:include_mongot_enabled": ["//:mongot_folder"],
"//conditions:default": [],
}),
pretty_printer_tests = {
"//src/mongo/util:pretty_printer_test.py": "//src/mongo/util:pretty_printer_test_program",
"//src/mongo/db/concurrency:lock_gdb_test.py": "//src/mongo/db:mongod",
"//src/mongo/db/query/optimizer:optimizer_gdb_test.py": "//src/mongo/db/query/optimizer:optimizer_gdb_test_program",
},
deps = [
"//src/mongo/db/modules/enterprise:dist-test",
],
)
# This sets up targets for install-mongo_crypt and archive-mongo_crypt
mongo_install(
name = "mongo_crypt",
srcs = [],
package_extract_name = "/",
deps = [
"//src/mongo/db/modules/enterprise:mongo_crypt",
],
)
# This sets up targets for install-mongo_crypt_shlib_test and archive-mongo_crypt_shlib_test
mongo_install(
name = "mongo_crypt_shlib_test",
srcs = [],
deps = [
"//src/mongo/db/modules/enterprise:mongo_crypt_shlib_test",
],
)
# This sets up targets for install-dbtest and archive-dbtest
mongo_install(
name = "dbtest",
srcs = [
"//src/mongo/dbtests:dbtest",
] + select({
"@platforms//os:windows": ["@windows_sasl//:bins"],
"//conditions:default": [],
}),
)
# This sets up targets for install-sdam_json_test and archive-sdam_json_test
mongo_install(
name = "sdam_json_test",
srcs = [
"//src/mongo/client/sdam:sdam_json_test",
],
)
# This sets up targets for install-server_selection_json_test and archive-server_selection_json_test
mongo_install(
name = "server_selection_json_test",
srcs = [
"//src/mongo/client/sdam:server_selection_json_test",
],
)
# This is because the install system hardlinks files so permission changes end up
# being reflected on the source code version without this copy
copy_file(
name = "stitch_header",
src = "//src/mongo/embedded/stitch_support:stitch_support.h",
out = "copied_files/stitch_support.h",
)
# This sets up targets for install-stitch_support and archive-stitch_support
mongo_install(
name = "stitch_support",
srcs = ["//src/mongo/embedded/stitch_support"],
package_extract_name = "/",
root_files = {
"stitch_header": "include/stitch_support/v1/stitch_support",
},
)
# This sets up targets for install-stitch_support_test and archive-stitch_support_test
mongo_install(
name = "stitch_support_test",
testonly = True,
srcs = [
"//src/mongo/embedded/stitch_support:stitch_support_test",
],
)
# This sets up targets for install-compass and archive-compass
mongo_install(
name = "compass",
srcs = ["//src/mongo/installer/compass:compass_files"],
)
[
mongo_install(
name = target_name,
testonly = True,
srcs = select({
"//bazel/config:include_autogenerated_targets_enabled": ["//src/mongo/db/modules/enterprise/autogenerated_targets:" + target_name],
"//conditions:default": [],
}) + select({
"@platforms//os:windows": ["@windows_sasl//:bins"],
"//conditions:default": [],
}),
)
for target_name in TEST_TAGS.keys()
if target_name != "mongo_integration_test"
]
# This sets up targets for install-mongo_integration_test and archive-mongo_integration_test
mongo_install(
name = "mongo_integration_test",
testonly = True,
srcs = select({
"//bazel/config:include_autogenerated_targets_enabled": [
"//src/mongo/db:mongod",
"//src/mongo/db/modules/enterprise/autogenerated_targets:mongo_integration_test",
"//src/mongo/s:mongos",
"//src/mongo/shell:mongo",
],
"//conditions:default": [],
}),
)
# This sets up targets for install-mongotmock and archive-mongotmock
mongo_install(
name = "mongotmock",
srcs = [
"//src/mongo/db/query/search/mongotmock",
],
)
# Pre-built binaries downloaded from an archive-dist-test task.
filegroup(
name = "installed-dist-test",
srcs = glob([
"dist-test/**",
]),
)
# Jstests generated by jstestfuzz or downloaded from an Evergreen task.
filegroup(
name = "jstestfuzz_generated_tests",
srcs = glob([
"jstestfuzz/out/*.js",
]),
)
# Multiversion binaries downloaded using db-contrib-tool, setting --linkDir to this directory.
filegroup(
name = "multiversion_binaries",
srcs = glob([
"multiversion_binaries/**",
]),
)
sh_binary(
name = "scip-clang",
srcs = ["//bazel:run_under_working_dir.sh"],
data = ["@scip-clang//file"],
env = {
"BINARY_PATH": "$(location @scip-clang//file)",
},
)
sh_binary(
name = "scip-src",
srcs = ["//bazel:run_under_working_dir.sh"],
data = ["@scip-src//file"],
env = {
"BINARY_PATH": "$(location @scip-src//file)",
},
)

214
MODULE.bazel Normal file
View File

@ -0,0 +1,214 @@
# Apple cc toolchain needs to be loaded before regular cc toolchain or else Apple will just use regular cc toolchain
bazel_dep(name = "apple_support", version = "1.17.1", repo_name = "build_bazel_apple_support")
bazel_dep(name = "bazel_features", version = "1.10.0")
single_version_override(
module_name = "bazel_features",
version = "1.10.0",
)
bazel_features_deps = use_extension("//bazel:bzlmod.bzl", "bazel_features_deps")
use_repo(bazel_features_deps, "bazel_features_globals", "bazel_features_version")
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
http_archive(
name = "bazel_clang_tidy",
integrity = "sha256-A+TGGfuHdS7vWT20eyAEiA3u4YXFtuzsJDcpTEmDMS0=",
strip_prefix = "bazel_clang_tidy-10c4bf70a8946789e3932f30e29dfba2dfb78e67",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/mongodb-forks/bazel_clang_tidy/archive/10c4bf70a8946789e3932f30e29dfba2dfb78e67.tar.gz",
] * 5,
)
# SourceGraph indexer
http_file(
name = "scip-clang",
executable = True,
sha256 = "9fc7d1c309f8adee1b8663efa802efa7363ef22ee06f4711117535e406690689",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/sourcegraph/scip-clang/releases/download/v0.3.2/scip-clang-x86_64-linux",
] * 5,
)
# SourceGraph CLI
http_file(
name = "scip-src",
executable = True,
sha256 = "eb3d627c4dad56b9f5d06634b90384104db065d410c4b338fa5e6558be4f3ad9",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/sourcegraph/src-cli/releases/download/6.4.0/src_linux_amd64",
] * 5,
)
bazel_dep(name = "platforms", version = "0.0.9")
single_version_override(
module_name = "platforms",
version = "0.0.9",
)
# TODO move over from WORKSPACE
# buf does not support s390x
#bazel_dep(name = "aspect_rules_lint", version = "1.1.0")
#single_version_override(
# module_name = "aspect_rules_lint",
# version = "1.1.0",
#)
bazel_dep(name = "rules_multirun", version = "0.9.0")
bazel_dep(name = "aspect_bazel_lib", version = "2.13.0")
single_version_override(
module_name = "aspect_bazel_lib",
version = "2.13.0",
)
bazel_dep(name = "aspect_rules_js", version = "2.1.3")
single_version_override(
module_name = "aspect_rules_js",
version = "2.1.3",
)
bazel_dep(name = "rules_nodejs", version = "6.3.0")
single_version_override(
module_name = "rules_nodejs",
version = "6.3.0",
)
bazel_dep(name = "rules_poetry", version = "")
git_override(
module_name = "rules_poetry",
commit = "917630033c736c188605cf0f558c34afc1eca540",
patches = ["//bazel/rules_poetry:rules_poetry.patch"],
remote = "https://github.com/mongodb-forks/rules_poetry",
)
#
## We need skylib to be able to use config_setting_group in rule_poetry below
## https://github.com/bazelbuild/bazel-skylib/blob/main/docs/selects_doc.md#selectsconfig_setting_group
bazel_dep(name = "bazel_skylib", version = "1.7.1")
single_version_override(
module_name = "bazel_skylib",
version = "1.7.1",
)
bazel_dep(name = "rules_pkg", version = "1.0.1")
bazel_dep(name = "zlib", version = "1.3.1")
single_version_override(
module_name = "zlib",
version = "1.3.1",
)
bazel_dep(name = "abseil-cpp", version = "", repo_name = "com_google_absl")
local_path_override(
module_name = "abseil-cpp",
path = "src/third_party/abseil-cpp/dist",
)
bazel_dep(name = "protobuf", version = "", repo_name = "com_google_protobuf")
local_path_override(
module_name = "protobuf",
path = "src/third_party/protobuf/dist",
)
bazel_dep(name = "grpc", version = "", repo_name = "com_github_grpc_grpc")
local_path_override(
module_name = "grpc",
path = "src/third_party/grpc/dist",
)
bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
single_version_override(
module_name = "buildifier_prebuilt",
version = "6.4.0",
)
bazel_dep(name = "rules_multitool", version = "0.4.0")
multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool")
multitool.hub(lockfile = "//tools/lint:multitool.lock.json")
use_repo(multitool, "multitool")
# TODO port over from WORKSPACE
# currently breaks pyright
##node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node", dev_dependency = True)
##node.toolchain(node_version = "18.20.4")
##
##npm = use_extension("@aspect_rules_js//npm:extensions.bzl", "npm", dev_dependency = True)
##npm.npm_translate_lock(
## name = "npm",
## npmrc = "//:.npmrc",
## pnpm_lock = "//:pnpm-lock.yaml",
## verify_node_modules_ignored = "//:.bazelignore",
##)
##use_repo(npm, "npm")
##
##pnpm = use_extension("@aspect_rules_js//npm:extensions.bzl", "pnpm")
##use_repo(pnpm, "pnpm")
#
setup_local_config_platform = use_repo_rule("//bazel/platforms:local_config_platform.bzl", "setup_local_config_platform")
setup_local_config_platform(name = "internal_platforms_do_not_use")
register_execution_platforms("@internal_platforms_do_not_use//host:host")
# TODO port over from WORKSPACE
# cannot be moved without changing the path or creating a symlink
# to _main~setup_mongo_toolchains~mongo_toolchain_v4 and _main~setup_mongo_toolchains~mongo_toolchain_v5
##setup_mongo_toolchains_extension = use_extension("//bazel/toolchains:mongo_toolchain.bzl", "setup_mongo_toolchains_extension")
##use_repo(
## setup_mongo_toolchains_extension,
## "mongo_toolchain_v4",
## "mongo_toolchain_v5",
##)
#
setup_mongo_apple_toolchains_extension = use_extension("//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "setup_mongo_apple_toolchain_extension")
use_repo(setup_mongo_apple_toolchains_extension, "mongo_apple_toolchain")
register_toolchains(
"@mongo_apple_toolchain//...",
)
setup_mongo_python_toolchains = use_extension("//bazel:bzlmod.bzl", "setup_mongo_python_toolchains")
use_repo(
setup_mongo_python_toolchains,
"py_host",
"py_linux_arm64",
"py_linux_ppc64le",
"py_linux_s390x",
"py_linux_x86_64",
"py_macos_arm64",
"py_macos_x86_64",
"py_windows_x86_64",
)
register_toolchains(
"@py_linux_arm64//:python_toolchain",
"@py_linux_ppc64le//:python_toolchain",
"@py_linux_s390x//:python_toolchain",
"@py_linux_x86_64//:python_toolchain",
"@py_macos_arm64//:python_toolchain",
"@py_macos_x86_64//:python_toolchain",
"@py_windows_x86_64//:python_toolchain",
)
# broken on windows currently
setup_local_host_values = use_repo_rule("//bazel/platforms:local_host_values.bzl", "setup_local_host_values")
setup_local_host_values(name = "local_host_values")
setup_evergreen_variables = use_repo_rule("//bazel/repository_rules:evergreen_variables.bzl", "setup_evergreen_variables")
setup_evergreen_variables(name = "evergreen_variables")
setup_mongo_windows_toolchains_extension = use_extension("//bazel/toolchains/cc/mongo_windows:mongo_toolchain.bzl", "setup_mongo_windows_toolchain_extension")
use_repo(setup_mongo_windows_toolchains_extension, "mongo_windows_toolchain")
register_toolchains("@mongo_windows_toolchain//...")

1067
MODULE.bazel.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

22
README.md
View File

@ -4,15 +4,15 @@ Welcome to MongoDB!
## Components
- `mongod` - The database server.
- `mongos` - Sharding router.
- `mongo` - The database shell (uses interactive javascript).
- `mongod` - The database server.
- `mongos` - Sharding router.
- `mongo` - The database shell (uses interactive javascript).
## Download MongoDB
- https://www.mongodb.com/try/download/community
- Using homebrew `brew tap mongodb/brew`
- Using docker image `docker pull mongo`
- https://www.mongodb.com/try/download/community
- Using homebrew `brew tap mongodb/brew`
- Using docker image `docker pull mongo`
## Building
@ -65,9 +65,9 @@ This will generate RPM and Debian packages.
## Learn MongoDB
- Documentation - https://docs.mongodb.com/manual/
- Developer Center - https://www.mongodb.com/developer/
- MongoDB University - https://learn.mongodb.com
- Documentation - https://docs.mongodb.com/manual/
- Developer Center - https://www.mongodb.com/developer/
- MongoDB University - https://learn.mongodb.com
## Cloud Hosted MongoDB
@ -75,11 +75,11 @@ https://www.mongodb.com/cloud/atlas
## Forums
- https://mongodb.com/community/forums/
- https://mongodb.com/community/forums/
Technical questions about using MongoDB.
- https://mongodb.com/community/forums/c/server-dev
- https://mongodb.com/community/forums/c/server-dev
Technical questions about building and developing MongoDB.

289
WORKSPACE.bazel
View File

@ -1,129 +1,138 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//bazel/platforms:local_config_platform.bzl", "setup_local_config_platform")
load("//bazel/toolchains:mongo_toolchain.bzl", "toolchain_download")
setup_local_config_platform(name = "local_config_platform")
toolchain_download(name = "mongo_toolchain")
register_toolchains("@mongo_toolchain//:mongo_toolchain")
load("//bazel/toolchains:python_toolchain.bzl", "setup_mongo_python_toolchains")
[register_toolchains(toolchain) for toolchain in setup_mongo_python_toolchains()]
load("//bazel/toolchains/cc/mongo_linux:mongo_toolchain.bzl", "setup_mongo_toolchains")
load("//bazel/toolchains/cc/mongo_linux:mongo_gdb.bzl", "gdb_download")
http_archive(
name = "rules_python",
sha256 = "8c8fe44ef0a9afc256d1e75ad5f448bb59b81aba149b8958f02f7b3a98f5d9b4",
strip_prefix = "rules_python-0.13.0",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz",
"https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz",
"https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz",
"https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz",
"https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz",
],
name = "aspect_rules_lint",
sha256 = "f60e4a737a5e09402f5fa3bd182efa80dac5523ca4b9bc5c6fa8c06fbfb46630",
strip_prefix = "rules_lint-1.1.0",
url = "https://github.com/aspect-build/rules_lint/releases/download/v1.1.0/rules_lint-v1.1.0.tar.gz",
)
setup_mongo_toolchains()
gdb_download(
name = "gdb",
version = "v5",
)
http_archive(
name = "platforms",
sha256 = "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
],
name = "windows_sasl",
build_file_content = """
package(default_visibility = ["//visibility:public"])
filegroup(
name = "includes",
srcs = select({
"@platforms//os:windows": glob(["include/**/*.h"]),
"//conditions:default": [],
}),
)
filegroup(
name = "libraries",
srcs = select({
"@platforms//os:windows": glob(["lib/**/*"]),
"//conditions:default": [],
}),
)
filegroup(
name = "bins",
srcs = select({
"@platforms//os:windows": glob(["bin/**/*"]),
"//conditions:default": [],
}),
)
""",
sha256 = "3e22e2b16f802277123590f64dfda44f1c9c8a2b7e758180cd956d8ab0965817",
urls = [
"https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
] * 5,
)
# We need skylib to be able to use config_setting_group in rule_poetry below
# https://github.com/bazelbuild/bazel-skylib/blob/main/docs/selects_doc.md#selectsconfig_setting_group
http_archive(
name = "bazel_skylib",
sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
],
name = "wix_toolset",
build_file_content = """
package(default_visibility = ["//visibility:public"])
filegroup(
name = "wix_binaries",
srcs = select({
"@platforms//os:windows": glob(["*"]),
"//conditions:default": [],
}),
)
# Poetry rules for managing Python dependencies
http_archive(
name = "rules_poetry",
sha256 = "59b6f654885059465880d641dbf45a1babad512c6ae7347b01ad03eebd615e0a",
strip_prefix = "rules_poetry-fc645faf7fb2e7f6c8fce968801e2a63dc1332bb",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/mongodb-forks/rules_poetry/archive/fc645faf7fb2e7f6c8fce968801e2a63dc1332bb.tar.gz",
"https://github.com/mongodb-forks/rules_poetry/archive/fc645faf7fb2e7f6c8fce968801e2a63dc1332bb.tar.gz",
"https://github.com/mongodb-forks/rules_poetry/archive/fc645faf7fb2e7f6c8fce968801e2a63dc1332bb.tar.gz",
"https://github.com/mongodb-forks/rules_poetry/archive/fc645faf7fb2e7f6c8fce968801e2a63dc1332bb.tar.gz",
"https://github.com/mongodb-forks/rules_poetry/archive/fc645faf7fb2e7f6c8fce968801e2a63dc1332bb.tar.gz",
],
filegroup(
name = "candle",
srcs = select({
"@platforms//os:windows": ["candle.exe"],
"//conditions:default": [],
}),
data = select({
"@platforms//os:windows": [":wix_binaries"],
"//conditions:default": [],
}),
)
load("@rules_poetry//rules_poetry:defs.bzl", "poetry_deps")
filegroup(
name = "light",
srcs = select({
"@platforms//os:windows": ["light.exe"],
"//conditions:default": [],
}),
data = select({
"@platforms//os:windows": [":wix_binaries"],
"//conditions:default": [],
}),
)
""",
sha256 = "6ac824e1642d6f7277d0ed7ea09411a508f6116ba6fae0aa5f2c7daa2ff43d31",
urls = [
"https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip",
] * 5,
)
poetry_deps()
load("//bazel/install_rules:windows_msvc.bzl", "windows_msvc")
windows_msvc(
name = "local_windows_msvc",
)
load("//bazel/coverity:coverity_toolchain.bzl", "coverity_toolchain")
coverity_toolchain(
name = "rules_coverity",
)
load("@rules_coverity//coverity:repositories.bzl", "rules_coverity_toolchains")
rules_coverity_toolchains()
load("@rules_poetry//rules_poetry:poetry.bzl", "poetry")
poetry(
name = "poetry",
excludes = [
"mdit-py-plugins", # plugins for markdown-it-py. Introduces an optional circular dependency that bazel does not like.
],
lockfile = "//:poetry.lock",
pyproject = "//:pyproject.toml",
python_interpreter_target_default = "@py_host//:bin/python3",
python_interpreter_target_mac = "@py_host//:bin/python3",
python_interpreter_target_win = "@py_host//:python.exe",
python_interpreter_target_default = "@py_host//:dist/bin/python3",
python_interpreter_target_mac = "@py_host//:dist/bin/python3",
python_interpreter_target_win = "@py_host//:dist/python.exe",
)
http_archive(
name = "aspect_rules_js",
sha256 = "630a71aba66c4023a5b16ab3efafaeed8b1a2865ccd168a34611eb73876b3fc4",
strip_prefix = "rules_js-1.37.1",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/aspect-build/rules_js/releases/download/v1.37.1/rules_js-v1.37.1.tar.gz",
"https://github.com/aspect-build/rules_js/releases/download/v1.37.1/rules_js-v1.37.1.tar.gz",
"https://github.com/aspect-build/rules_js/releases/download/v1.37.1/rules_js-v1.37.1.tar.gz",
"https://github.com/aspect-build/rules_js/releases/download/v1.37.1/rules_js-v1.37.1.tar.gz",
"https://github.com/aspect-build/rules_js/releases/download/v1.37.1/rules_js-v1.37.1.tar.gz",
],
)
load("@aspect_rules_js//js:toolchains.bzl", "DEFAULT_NODE_VERSION", "rules_js_register_toolchains")
load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies")
rules_js_dependencies()
load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains")
nodejs_register_toolchains(
name = "nodejs",
node_version = DEFAULT_NODE_VERSION,
)
rules_js_register_toolchains(node_version = DEFAULT_NODE_VERSION)
load("@aspect_rules_js//npm:repositories.bzl", "npm_translate_lock")
npm_translate_lock(
name = "npm",
npmrc = "//:.npmrc",
pnpm_lock = "//:pnpm-lock.yaml",
verify_node_modules_ignored = "//:.bazelignore",
)
@ -132,20 +141,74 @@ load("@npm//:repositories.bzl", "npm_repositories")
npm_repositories()
# TODO: This comes with a lot of built in formatters
# Since we are only using prettier for now we do not use the others
# See https://github.com/aspect-build/rules_lint/releases/tag/v0.11.0 for all supported formatters
http_archive(
name = "aspect_rules_lint",
sha256 = "41fad363f11ccab46a244f93f8ccb0f442bc235e606d2fad87801987ad0759b1",
strip_prefix = "rules_lint-0.12.0",
urls = [
# Implements retry by relisting each url multiple times to be used as a failover.
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
"https://github.com/aspect-build/rules_lint/releases/download/v0.12.0/rules_lint-v0.12.0.tar.gz",
"https://github.com/aspect-build/rules_lint/releases/download/v0.12.0/rules_lint-v0.12.0.tar.gz",
"https://github.com/aspect-build/rules_lint/releases/download/v0.12.0/rules_lint-v0.12.0.tar.gz",
"https://github.com/aspect-build/rules_lint/releases/download/v0.12.0/rules_lint-v0.12.0.tar.gz",
"https://github.com/aspect-build/rules_lint/releases/download/v0.12.0/rules_lint-v0.12.0.tar.gz",
],
# Sub in the system openssl for boringssl since we don't want two implementations of
# ssl in the same address space.
new_local_repository(
name = "boringssl",
build_file_content = """
cc_library(
name = "crypto",
linkopts = ["-lcrypto"],
visibility = ["//visibility:public"],
)
cc_library(
name = "ssl",
linkopts = ["-lssl"],
visibility = ["//visibility:public"],
)
""",
path = "bazel/_openssl_placeholder_for_grpc",
)
# Overloads for the vendored repositories.
#
# WARNING: Don't change the order of the deps() calls and local_repositories.
# They're read linearly dependencies that come first override later
# ones. Dependency updates might change the correct order, though it's
# unlikely. This is obviously a temporary solution and will no longer
# be necessary once migration to bzlmod is complete.
# Note: rules_python is implicitly loaded with a grpc-compatible version.
load("//bazel/install_rules:pigz.bzl", "setup_pigz")
setup_pigz(
name = "pigz",
)
# This repository is normally created by db-contrib-tool or manually extracting the binaries at the proper location
new_local_repository(
name = "mongot_localdev",
build_file_content = """
package(default_visibility = ["//visibility:public"])
filegroup(
name = "mongot_binaries",
srcs = glob(["**"], exclude = ["BUILD.bazel", "WORKSPACE", "MODULE.bazel", "MODULE.bazel.lock"]),
)
""",
path = "mongot-localdev",
)
local_repository(
name = "bazel_rules_mongo",
path = "buildscripts/bazel_rules_mongo",
repo_mapping = {"@poetry": "@poetry_bazel_rules_mongo"},
)
load("@bazel_rules_mongo//codeowners:codeowners_validator.bzl", "codeowners_validator")
codeowners_validator()
load("@bazel_rules_mongo//codeowners:codeowners_binary.bzl", "codeowners_binary")
codeowners_binary()
poetry(
name = "poetry_bazel_rules_mongo",
lockfile = "@bazel_rules_mongo//:poetry.lock",
pyproject = "@bazel_rules_mongo//:pyproject.toml",
)
load("//bazel/format:shfmt.bzl", "shfmt")
shfmt()

18
bazel/BUILD.bazel
View File

@ -0,0 +1,18 @@
package(default_visibility = ["//visibility:public"])
# Expose script for external usage through bazel.
exports_files([
"install_rules.py",
"run_under_working_dir.sh",
])
sh_binary(
name = "lint",
srcs = ["lint.sh"],
visibility = ["//visibility:public"],
)
py_binary(
name = "bazelisk",
srcs = ["bazelisk.py"],
)

4
bazel/OWNERS.yml
View File

@ -1,7 +1,5 @@
version: 1.0.0
aliases:
- //bazel/devprod_build_aliases.yml
filters:
- "*":
approvers:
- devprod-build
- 10gen/devprod-build

4
bazel/README.md
View File

@ -1,4 +1,4 @@
# MongoDB Bazel Documentation
- [Developer Workflow](docs/developer_workflow.md)
- [Best Practices](docs/best_practices.md)
- [Developer Workflow](docs/developer_workflow.md)
- [Best Practices](docs/best_practices.md)

1
bazel/_openssl_placeholder_for_grpc/.placeholder Normal file
View File

@ -0,0 +1 @@
# placeholder

544
bazel/bazelisk.py Executable file
View File

@ -0,0 +1,544 @@
#!/usr/bin/env python3
"""
Copyright 2018 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import base64
import hashlib
import json
import netrc
import os
import os.path
import platform
import re
import shutil
import subprocess
import sys
import time
from contextlib import closing
try:
from urllib.error import HTTPError
from urllib.parse import urlparse
from urllib.request import Request, urlopen
except ImportError:
# Python 2.x compatibility hack.
# http://python-future.org/compatible_idioms.html?highlight=urllib#urllib-module
from urllib2 import HTTPError, Request, urlopen
from urlparse import urlparse
FileNotFoundError = IOError
ONE_HOUR = 1 * 60 * 60
LATEST_PATTERN = re.compile(r"latest(-(?P<offset>\d+))?$")
LAST_GREEN_COMMIT_PATH = "https://storage.googleapis.com/bazel-builds/last_green_commit/github.com/bazelbuild/bazel.git/publish-bazel-binaries"
BAZEL_GCS_PATH_PATTERN = (
"https://storage.googleapis.com/bazel-builds/artifacts/{platform}/{commit}/bazel"
)
SUPPORTED_PLATFORMS = {"linux": "ubuntu1404", "windows": "windows", "darwin": "macos"}
TOOLS_BAZEL_PATH = "./tools/bazel"
BAZEL_REAL = "BAZEL_REAL"
BAZEL_UPSTREAM = "bazelbuild"
_dotfiles_dict = None
def get_dotfiles_dict():
"""Loads all supported dotfiles and returns a unified name=value dictionary
for their config settings. The dictionary is only loaded on the first call
to this function; subsequent calls used a cached result, so won't change.
"""
global _dotfiles_dict
if _dotfiles_dict is not None:
return _dotfiles_dict
_dotfiles_dict = dict()
env_files = []
# Here we're only checking the workspace root. Ideally, we would also check
# the user's home directory to match the Go version. When making that edit,
# be sure to obey the correctly prioritization of workspace / home rcfiles.
root = find_workspace_root()
if root:
env_files.append(os.path.join(root, ".bazeliskrc"))
for env_file in env_files:
try:
with open(env_file, "r") as f:
rcdata = f.read()
except Exception:
continue
for line in rcdata.splitlines():
line = line.split("#", 1)[0].strip()
if not line:
continue
some_name, some_value = line.split("=", 1)
_dotfiles_dict[some_name] = some_value
return _dotfiles_dict
def get_env_or_config(name, default=None):
"""Reads a configuration value from the environment, but falls back to
reading it from .bazeliskrc in the workspace root."""
if name in os.environ:
return os.environ[name]
dotfiles = get_dotfiles_dict()
if name in dotfiles:
return dotfiles[name]
return default
def decide_which_bazel_version_to_use():
# Check in this order:
# - env var "USE_BAZEL_VERSION" is set to a specific version.
# - env var "USE_NIGHTLY_BAZEL" or "USE_BAZEL_NIGHTLY" is set -> latest
# nightly. (TODO)
# - env var "USE_CANARY_BAZEL" or "USE_BAZEL_CANARY" is set -> latest
# rc. (TODO)
# - the file workspace_root/tools/bazel exists -> that version. (TODO)
# - workspace_root/.bazelversion exists -> read contents, that version.
# - workspace_root/WORKSPACE contains a version -> that version. (TODO)
# - fallback: latest release
use_bazel_version = get_env_or_config("USE_BAZEL_VERSION")
if use_bazel_version is not None:
return use_bazel_version
workspace_root = find_workspace_root()
if workspace_root:
bazelversion_path = os.path.join(workspace_root, ".bazelversion")
if os.path.exists(bazelversion_path):
with open(bazelversion_path, "r") as f:
for ln in f.readlines():
ln = ln.strip()
if ln:
return ln
return "latest"
def find_workspace_root(root=None):
if root is None:
root = os.getcwd()
for boundary in ["MODULE.bazel", "REPO.bazel", "WORKSPACE.bazel", "WORKSPACE"]:
path = os.path.join(root, boundary)
if os.path.exists(path) and not os.path.isdir(path):
return root
new_root = os.path.dirname(root)
return find_workspace_root(new_root) if new_root != root else None
def resolve_version_label_to_number_or_commit(bazelisk_directory, version):
"""Resolves the given label to a released version of Bazel or a commit.
Args:
bazelisk_directory: string; path to a directory that can store
temporary data for Bazelisk.
version: string; the version label that should be resolved.
Returns:
A (string, bool) tuple that consists of two parts:
1. the resolved number of a Bazel release (candidate), or the commit
of an unreleased Bazel binary,
2. An indicator for whether the returned version refers to a commit.
"""
if version == "last_green":
return get_last_green_commit(), True
if "latest" in version:
match = LATEST_PATTERN.match(version)
if not match:
raise Exception(
'Invalid version "{}". In addition to using a version '
'number such as "0.20.0", you can use values such as '
'"latest" and "latest-N", with N being a non-negative '
"integer.".format(version)
)
history = get_version_history(bazelisk_directory)
offset = int(match.group("offset") or "0")
return resolve_latest_version(history, offset), False
return version, False
def get_last_green_commit():
commit = read_remote_text_file(LAST_GREEN_COMMIT_PATH).strip()
if not re.match(r"^[0-9a-f]{40}$", commit):
raise Exception("Invalid commit hash: {}".format(commit))
return commit
def get_releases_json(bazelisk_directory):
"""Returns the most recent versions of Bazel, in descending order."""
releases = os.path.join(bazelisk_directory, "releases.json")
# Use a cached version if it's fresh enough.
if os.path.exists(releases):
if abs(time.time() - os.path.getmtime(releases)) < ONE_HOUR:
with open(releases, "rb") as f:
try:
return json.loads(f.read().decode("utf-8"))
except ValueError:
print("WARN: Could not parse cached releases.json.")
pass
with open(releases, "wb") as f:
body = read_remote_text_file("https://api.github.com/repos/bazelbuild/bazel/releases")
f.write(body.encode("utf-8"))
return json.loads(body)
def read_remote_text_file(url):
with closing(urlopen(url)) as res:
body = res.read()
try:
return body.decode(res.info().get_content_charset("iso-8859-1"))
except AttributeError:
# Python 2.x compatibility hack
return body.decode(res.info().getparam("charset") or "iso-8859-1")
def get_version_history(bazelisk_directory):
return sorted(
(
release["tag_name"]
for release in get_releases_json(bazelisk_directory)
if not release["prerelease"]
),
# This only handles versions with numeric components, but that is fine
# since prerelease versions have been excluded.
key=lambda version: tuple(int(component) for component in version.split(".")),
reverse=True,
)
def resolve_latest_version(version_history, offset):
if offset >= len(version_history):
version = "latest-{}".format(offset) if offset else "latest"
raise Exception(
'Cannot resolve version "{}": There are only {} Bazel releases.'.format(
version, len(version_history)
)
)
# This only works since we store the history in descending order.
return version_history[offset]
def get_operating_system():
operating_system = platform.system().lower()
if operating_system not in ("linux", "darwin", "windows"):
raise Exception(
'Unsupported operating system "{}". '
"Bazel currently only supports Linux, macOS and Windows.".format(operating_system)
)
return operating_system
def determine_executable_filename_suffix():
operating_system = get_operating_system()
return ".exe" if operating_system == "windows" else ""
def determine_bazel_filename(version):
operating_system = get_operating_system()
supported_machines = get_supported_machine_archs(version, operating_system)
machine = normalized_machine_arch_name()
if machine not in supported_machines:
raise Exception(
'Unsupported machine architecture "{}". Bazel {} only supports {} on {}.'.format(
machine, version, ", ".join(supported_machines), operating_system.capitalize()
)
)
filename_suffix = determine_executable_filename_suffix()
bazel_flavor = "bazel"
if get_env_or_config("BAZELISK_NOJDK", "0") != "0":
bazel_flavor = "bazel_nojdk"
return "{}-{}-{}-{}{}".format(bazel_flavor, version, operating_system, machine, filename_suffix)
def get_supported_machine_archs(version, operating_system):
supported_machines = ["x86_64"]
if operating_system == "linux":
supported_machines += ["s390x", "ppc64le"]
versions = version.split(".")[:2]
if len(versions) == 2:
# released version
major, minor = int(versions[0]), int(versions[1])
if (
operating_system == "darwin"
and (major > 4 or major == 4 and minor >= 1)
or operating_system == "linux"
and (major > 3 or major == 3 and minor >= 4)
):
# Linux arm64 was supported since 3.4.0.
# Darwin arm64 was supported since 4.1.0.
supported_machines.append("arm64")
elif operating_system in ("darwin", "linux"):
# This is needed to run bazelisk_test.sh on Linux and Darwin arm64 machines, which are
# becoming more and more popular.
# It works because all recent commits of Bazel support arm64 on Darwin and Linux.
# However, this would add arm64 by mistake if the commit is too old, which should be
# a rare scenario.
supported_machines.append("arm64")
return supported_machines
def normalized_machine_arch_name():
machine = platform.machine().lower()
if machine == "amd64":
machine = "x86_64"
elif machine == "aarch64":
machine = "arm64"
return machine
def determine_url(version, is_commit, bazel_filename):
if is_commit:
sys.stderr.write("Using unreleased version at commit {}\n".format(version))
# No need to validate the platform thanks to determine_bazel_filename().
return BAZEL_GCS_PATH_PATTERN.format(
platform=SUPPORTED_PLATFORMS[platform.system().lower()], commit=version
)
# Split version into base version and optional additional identifier.
# Example: '0.19.1' -> ('0.19.1', None), '0.20.0rc1' -> ('0.20.0', 'rc1')
(version, rc, mongo_version) = re.match(
r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?(-mongo_\w+)?", version
).groups()
bazelisk_base_url = get_env_or_config("BAZELISK_BASE_URL")
if bazelisk_base_url is not None:
if mongo_version:
version += mongo_version
return "{}/{}/{}".format(bazelisk_base_url, version, bazel_filename)
else:
return "https://releases.bazel.build/{}/{}/{}".format(
version, rc if rc else "release", bazel_filename
)
def trim_suffix(string, suffix):
if string.endswith(suffix):
return string[: len(string) - len(suffix)]
else:
return string
def download_bazel_into_directory(version, is_commit, directory):
bazel_filename = determine_bazel_filename(version)
bazel_url = determine_url(version, is_commit, bazel_filename)
filename_suffix = determine_executable_filename_suffix()
bazel_directory_name = trim_suffix(bazel_filename, filename_suffix)
destination_dir = os.path.join(directory, bazel_directory_name, "bin")
maybe_makedirs(destination_dir)
destination_path = os.path.join(destination_dir, "bazel" + filename_suffix)
if not os.path.exists(destination_path):
download(bazel_url, destination_path)
os.chmod(destination_path, 0o755)
sha256_path = destination_path + ".sha256"
expected_hash = ""
if not os.path.exists(sha256_path):
try:
download(bazel_url + ".sha256", sha256_path)
except HTTPError as e:
if e.code == 404:
sys.stderr.write(
"The Bazel mirror does not have a checksum file; skipping checksum verification."
)
return destination_path
raise e
with open(sha256_path, "r") as sha_file:
expected_hash = sha_file.read().split()[0]
sha256_hash = hashlib.sha256()
with open(destination_path, "rb") as bazel_file:
for byte_block in iter(lambda: bazel_file.read(4096), b""):
sha256_hash.update(byte_block)
actual_hash = sha256_hash.hexdigest()
if actual_hash != expected_hash:
os.remove(destination_path)
os.remove(sha256_path)
print(
"The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Please try again.".format(
expected_hash, actual_hash
)
)
# Exiting with a special exit code not used by Bazel, so the calling process may retry based on that.
# https://docs.bazel.build/versions/0.21.0/guide.html#what-exit-code-will-i-get
sys.exit(22)
return destination_path
def download(url, destination_path):
sys.stderr.write("Downloading {}...\n".format(url))
request = Request(url)
if get_env_or_config("BAZELISK_BASE_URL") is not None:
parts = urlparse(url)
creds = None
try:
creds = netrc.netrc().hosts.get(parts.netloc)
except Exception:
pass
if creds is not None:
auth = base64.b64encode(("%s:%s" % (creds[0], creds[2])).encode("ascii"))
request.add_header("Authorization", "Basic %s" % auth.decode("utf-8"))
with closing(urlopen(request)) as response, open(destination_path, "wb") as file:
shutil.copyfileobj(response, file)
def get_bazelisk_directory():
bazelisk_home = get_env_or_config("BAZELISK_HOME")
if bazelisk_home is not None:
return bazelisk_home
operating_system = get_operating_system()
base_dir = None
if operating_system == "windows":
base_dir = os.environ.get("LocalAppData")
if base_dir is None:
raise Exception("%LocalAppData% is not defined")
elif operating_system == "darwin":
base_dir = os.environ.get("HOME")
if base_dir is None:
raise Exception("$HOME is not defined")
base_dir = os.path.join(base_dir, "Library/Caches")
elif operating_system == "linux":
base_dir = os.environ.get("XDG_CACHE_HOME")
if base_dir is None:
base_dir = os.environ.get("HOME")
if base_dir is None:
raise Exception("neither $XDG_CACHE_HOME nor $HOME are defined")
base_dir = os.path.join(base_dir, ".cache")
else:
raise Exception("Unsupported operating system '{}'".format(operating_system))
return os.path.join(base_dir, "bazelisk")
def maybe_makedirs(path):
"""
Creates a directory and its parents if necessary.
"""
try:
os.makedirs(path)
except OSError as e:
if not os.path.isdir(path):
raise e
def delegate_tools_bazel(bazel_path):
"""Match Bazel's own delegation behavior in the builds distributed by most
package managers: use tools/bazel if it's present, executable, and not this
script.
"""
root = find_workspace_root()
if root:
wrapper = os.path.join(root, TOOLS_BAZEL_PATH)
if os.path.exists(wrapper) and os.access(wrapper, os.X_OK):
try:
if not os.path.samefile(wrapper, __file__):
return wrapper
except AttributeError:
# Python 2 on Windows does not support os.path.samefile
if os.path.abspath(wrapper) != os.path.abspath(__file__):
return wrapper
return None
def prepend_directory_to_path(env, directory):
"""
Prepend binary directory to PATH
"""
if "PATH" in env:
env["PATH"] = directory + os.pathsep + env["PATH"]
else:
env["PATH"] = directory
def make_bazel_cmd(bazel_path, argv):
env = os.environ.copy()
wrapper = delegate_tools_bazel(bazel_path)
if wrapper:
env[BAZEL_REAL] = bazel_path
env["BAZELISK_SKIP_WRAPPER"] = "1"
bazel_path = wrapper
directory = os.path.dirname(bazel_path)
prepend_directory_to_path(env, directory)
return {
"exec": bazel_path,
"args": argv,
"env": env,
}
def execute_bazel(bazel_path, argv):
cmd = make_bazel_cmd(bazel_path, argv)
# We cannot use close_fds on Windows, so disable it there.
p = subprocess.Popen([cmd["exec"]] + cmd["args"], close_fds=os.name != "nt", env=cmd["env"])
while True:
try:
return p.wait()
except KeyboardInterrupt:
# Bazel will also get the signal and terminate.
# We should continue waiting until it does so.
pass
def get_bazel_path():
bazelisk_directory = get_bazelisk_directory()
maybe_makedirs(bazelisk_directory)
bazel_version = decide_which_bazel_version_to_use()
bazel_version, is_commit = resolve_version_label_to_number_or_commit(
bazelisk_directory, bazel_version
)
# TODO: Support other forks just like Go version
bazel_directory = os.path.join(bazelisk_directory, "downloads", BAZEL_UPSTREAM)
return download_bazel_into_directory(bazel_version, is_commit, bazel_directory)
def main(argv=None):
if argv is None:
argv = sys.argv
bazel_path = get_bazel_path()
argv = argv[1:]
if argv and argv[0] == "--print_env":
cmd = make_bazel_cmd(bazel_path, argv)
env = cmd["env"]
for key in env:
print("{}={}".format(key, env[key]))
return 0
return execute_bazel(bazel_path, argv)
if __name__ == "__main__":
sys.exit(main())

26
bazel/bzlmod.bzl Normal file
View File

@ -0,0 +1,26 @@
load("@bazel_features//:deps.bzl", _bazel_features_deps = "bazel_features_deps")
load("//bazel/platforms:local_config_platform.bzl", "setup_local_config_platform")
load("//bazel/toolchains/python:python_toolchain.bzl", _setup_mongo_python_toolchains = "setup_mongo_python_toolchains")
load("//bazel/toolchains/cc/mongo_linux:mongo_toolchain.bzl", _setup_mongo_toolchains = "setup_mongo_toolchains")
def _bazel_features_deps_impl(_ctx):
_bazel_features_deps()
bazel_features_deps = module_extension(
implementation = _bazel_features_deps_impl,
)
def _setup_mongo_python_toolchains_impl(_ctx):
_setup_mongo_python_toolchains()
setup_mongo_python_toolchains = module_extension(
implementation = _setup_mongo_python_toolchains_impl,
)
def _setup_mongo_toolchains_impl(_ctx):
setup_local_config_platform(name = "internal_platforms_do_not_use")
_setup_mongo_toolchains()
setup_mongo_toolchains = module_extension(
implementation = _setup_mongo_toolchains_impl,
)

1470
bazel/config/BUILD.bazel
View File

File diff suppressed because it is too large Load Diff

342
bazel/config/configs.bzl
View File

@ -4,13 +4,44 @@
# compiler_type
# =============
compiler_type_values = ["gcc", "clang", "msvc"]
compiler_type_provider = provider(
doc = "Select the compiler (e.g.: gcc)",
fields = {"compiler_type": "Choose one of [gcc, clang]"},
fields = {"compiler_type": "Choose one of " + ".".join(compiler_type_values)},
)
def compiler_type_impl(ctx):
compiler_type_value = ctx.build_setting_value
if compiler_type_value not in compiler_type_values:
fail(str(ctx.label) + " compiler_type allowed to take values {" + ", ".join(compiler_type_values) + "} but was set to unallowed value " + compiler_type_value)
return compiler_type_provider(compiler_type = compiler_type_value)
compiler_type = rule(
implementation = lambda ctx: compiler_type_provider(compiler_type = ctx.build_setting_value),
implementation = compiler_type_impl,
build_setting = config.string(flag = True),
)
# =========
# mongo_toolchain_version
# =========
mongo_toolchain_version_values = ["v4"]
mongo_toolchain_version_provider = provider(
doc = "Select the mongo toolchain version (e.g.: v4)",
fields = {"mongo_toolchain_version": "Choose one of " + ".".join(mongo_toolchain_version_values)},
)
def mongo_toolchain_version_impl(ctx):
mongo_toolchain_version_value = ctx.build_setting_value
if mongo_toolchain_version_value not in mongo_toolchain_version_values:
fail(str(ctx.label) + " mongo_toolchain_version allowed to take values {" + ", ".join(mongo_toolchain_version_values) + "} but was set to unallowed value " + mongo_toolchain_version_value)
return mongo_toolchain_version_provider(mongo_toolchain_version = mongo_toolchain_version_value)
mongo_toolchain_version = rule(
implementation = mongo_toolchain_version_impl,
build_setting = config.string(flag = True),
)
@ -18,7 +49,7 @@ compiler_type = rule(
# linker
# ==========
linker_values = ["auto", "gold", "lld"]
linker_values = ["auto", "gold", "lld", "mold"]
linker_provider = provider(
doc = "Specify the type of linker to use.",
@ -83,6 +114,28 @@ spider_monkey_dbg = rule(
build_setting = config.bool(flag = True),
)
# =========
# include_mongot
# =========
include_mongot_provider = provider(doc = "Enable including mongot in the final binary archive from a local mongot-localdev folder.", fields = ["enabled"])
include_mongot = rule(
implementation = lambda ctx: include_mongot_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# include_autogenerated_targets
# =========
include_autogenerated_targets_provider = provider(doc = "Enable using autogenerated build targets.", fields = ["enabled"])
include_autogenerated_targets = rule(
implementation = lambda ctx: include_autogenerated_targets_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# allocator
# =========
@ -191,16 +244,16 @@ use_glibcxx_debug = rule(
)
# =========
# libc++
# otel
# =========
use_libcxx_provider = provider(
doc = """use libc++ (experimental, requires clang)""",
build_otel_provider = provider(
doc = """Enable building otel and protobuf compiler. This has no effect on non-linux operating systems.""",
fields = ["enabled"],
)
use_libcxx = rule(
implementation = lambda ctx: use_libcxx_provider(enabled = ctx.build_setting_value),
build_otel = rule(
implementation = lambda ctx: build_otel_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
@ -297,20 +350,6 @@ linkstatic = rule(
build_setting = config.bool(flag = True),
)
# =========
# use-diagnostic-latches
# =========
use_diagnostic_latches_provider = provider(
doc = "Enable annotated Mutex types.",
fields = ["enabled"],
)
use_diagnostic_latches = rule(
implementation = lambda ctx: use_diagnostic_latches_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# shared_archive
# =========
@ -325,6 +364,48 @@ shared_archive = rule(
build_setting = config.bool(flag = True),
)
# =========
# skip_archive
# =========
skip_archive_provider = provider(
doc = "Skip generating archives in favor of using --start-lib --end-lib",
fields = ["enabled"],
)
skip_archive = rule(
implementation = lambda ctx: skip_archive_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# compress_debug_compile
# =========
compress_debug_compile_provider = provider(
doc = "Compress the debug sections outputted by the compiler.",
fields = ["enabled"],
)
compress_debug_compile = rule(
implementation = lambda ctx: compress_debug_compile_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# simple_build_id
# =========
simple_build_id_provider = provider(
doc = "Replace linker build-id with a simpler one based off output file name.",
fields = ["enabled"],
)
simple_build_id = rule(
implementation = lambda ctx: simple_build_id_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# detect_odr_violations
# =========
@ -343,12 +424,6 @@ detect_odr_violations = rule(
# build_enterprise_module
# =========
# Original documentation is:
# Comma-separated list of modules to build. Empty means none. Default is all.
# As Bazel will not support the module building in the same way as Scons, the only
# module is supported at present is the enterprise
# more: https://mongodb.slack.com/archives/C05V4F6GZ6J/p1705687513581639
build_enterprise_provider = provider(
doc = """Build enterprise module""",
fields = ["enabled"],
@ -372,6 +447,19 @@ streams_release_build = rule(
build_setting = config.bool(flag = True),
)
# =========
# disable-streams
# =========
disable_streams_provider = provider(
doc = """If set, will exclude the enterprise streams module in a non-release build.""",
fields = ["enabled"],
)
disable_streams = rule(
implementation = lambda ctx: disable_streams_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# visibility-support
# =========
@ -407,6 +495,41 @@ dbg = rule(
build_setting = config.bool(flag = True),
)
# ==========
# dbg_level
# ==========
dbg_level_values = ["0", "1", "2", "3"]
dbg_level_provider = provider(
doc = "Sets the level of debug information generated.",
fields = ["level"],
)
def dbg_level_support_impl(ctx):
dbg_level_value = ctx.build_setting_value
if dbg_level_value not in dbg_level_values:
fail(str(ctx.label) + "dbg_level allowed to take values {" + ", ".join(dbg_level_values) + "} but was set to unallowed value " + dbg_level_value)
return dbg_level_provider(level = dbg_level_value)
dbg_level = rule(
implementation = dbg_level_support_impl,
build_setting = config.string(flag = True),
)
# =========
# debug symbols
# =========
debug_symbols_provider = provider(
doc = """Enable the production of debug symbols.""",
fields = ["enabled"],
)
debug_symbols = rule(
implementation = lambda ctx: debug_symbols_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# opt
# =========
@ -441,19 +564,6 @@ release = rule(
build_setting = config.bool(flag = True),
)
# =========
# local_build
# =========
local_build_provider = provider(
doc = """Allows configurations based on local builds""",
fields = ["enabled"],
)
local_build = rule(
implementation = lambda ctx: local_build_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# dwarf_version
# =========
@ -474,3 +584,153 @@ dwarf_version = rule(
implementation = dwarf_version_impl,
build_setting = config.string(flag = True),
)
# =========
# disable-warnings-as-errors
# =========
disable_warnings_as_errors_provider = provider(
doc = """Don't add a warnings-as-errors flag to compiler command lines""",
fields = ["enabled"],
)
disable_warnings_as_errors = rule(
implementation = lambda ctx: disable_warnings_as_errors_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# ssl
# =========
ssl_provider = provider(doc = "Enable or Disable SSL", fields = ["enabled"])
ssl = rule(
implementation = lambda ctx: ssl_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# thin_lto
# =========
thin_lto_provider = provider(
doc = """Enable thin link time optimization (LTO) (experimental)""",
fields = ["enabled"],
)
thin_lto = rule(
implementation = lambda ctx: thin_lto_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# gcov
# =========
gcov_provider = provider(
doc = "Choose if gcov should be used",
fields = ["enabled"],
)
gcov = rule(
implementation = lambda ctx: gcov_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# pgo_profile
# =========
pgo_profile_provider = provider(
doc = "Choose if pgo profiling should be generated",
fields = ["enabled"],
)
pgo_profile = rule(
implementation = lambda ctx: pgo_profile_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =============
# sdkroot
# =============
sdkroot_provider = provider(
doc = "The path to the sdk, e.g. SDKROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk",
fields = {"path": "sdk root.]"},
)
sdkroot = rule(
implementation = lambda ctx: sdkroot_provider(path = ctx.build_setting_value),
build_setting = config.string(flag = True),
)
# =========
# js_engine
# =========
js_engine_provider = provider(
doc = "JavaScript scripting engine implementation",
fields = {"engine": "Javascript scripting engine."},
)
js_engine = rule(
implementation = lambda ctx: js_engine_provider(engine = ctx.build_setting_value),
build_setting = config.string(flag = True),
)
# =========
# server_js
# =========
server_js_provider = provider(
doc = "Build mongod without JavaScript support",
fields = ["enabled"],
)
server_js = rule(
implementation = lambda ctx: server_js_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# create_dwp
# =========
create_dwp_provider = provider(
doc = "Create dwp files when using install targets",
fields = ["enabled"],
)
create_dwp = rule(
implementation = lambda ctx: create_dwp_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# use-diagnostic-latches
# =========
use_diagnostic_latches_provider = provider(
doc = "Enable annotated Mutex types.",
fields = ["enabled"],
)
use_diagnostic_latches = rule(
implementation = lambda ctx: use_diagnostic_latches_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)
# =========
# local_build
# =========
local_build_provider = provider(
doc = """Allows configurations based on local builds""",
fields = ["enabled"],
)
local_build = rule(
implementation = lambda ctx: local_build_provider(enabled = ctx.build_setting_value),
build_setting = config.bool(flag = True),
)

155
bazel/config/generate_config_header.bzl Normal file
View File

@ -0,0 +1,155 @@
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
load("//bazel/config:configs.bzl", "sdkroot_provider")
load("//bazel:mongo_src_rules.bzl", "write_target")
def generate_config_header_impl(ctx):
cc_toolchain = find_cpp_toolchain(ctx)
compiler_bin = cc_toolchain.compiler_executable
input = ctx.attr.template.files.to_list()[0].path
checks = ctx.attr.checks.files.to_list()[0].path
# Generate compiler flags we need to make clang/gcc/msvc actually compile.
feature_configuration = cc_common.configure_features(
ctx = ctx,
cc_toolchain = cc_toolchain,
)
compile_variables = cc_common.create_compile_variables(
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
user_compile_flags = ctx.fragments.cpp.cxxopts + ctx.fragments.cpp.copts,
)
compiler_flags = cc_common.get_memory_inefficient_command_line(
feature_configuration = feature_configuration,
action_name = ACTION_NAMES.cpp_compile,
variables = compile_variables,
)
link_flags = cc_common.get_memory_inefficient_command_line(
feature_configuration = feature_configuration,
action_name = ACTION_NAMES.cpp_link_executable,
variables = compile_variables,
)
env_flags = cc_common.get_environment_variables(
feature_configuration = feature_configuration,
action_name = ACTION_NAMES.cpp_compile,
variables = compile_variables,
)
expanded_extra_definitions = {}
for key, val in ctx.attr.extra_definitions.items():
# Bazel throws an error if you try to call this on a location var
if "$(location" not in val:
expanded_extra_definitions |= {
key: ctx.expand_make_variables("generate_config_header_expand", val, ctx.var),
}
expanded_extra_definitions |= {
"compile_variables": " ".join(compiler_flags + ctx.attr.cpp_opts),
"linkflags": " ".join(link_flags + ctx.attr.cpp_linkflags),
"cpp_defines": " ".join(ctx.attr.cpp_defines),
}
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
generator_script = ctx.attr.generator_script.files.to_list()[0].path
additional_inputs = []
additional_inputs_depsets = []
for additional_input in ctx.attr.additional_inputs:
files = additional_input.files.to_list()
additional_inputs_depsets.append(additional_input.files)
for file in files:
additional_inputs.append("--additional-input")
additional_inputs.append(file.path)
ctx.actions.run(
executable = python.interpreter.path,
outputs = [ctx.outputs.output, ctx.outputs.logfile],
mnemonic = "ConfigHeaderGen",
inputs = depset(transitive = [
cc_toolchain.all_files,
python.files,
ctx.attr.generator_script.files,
ctx.attr.template.files,
ctx.attr.checks.files,
] + additional_inputs_depsets),
arguments = [
generator_script, # bazel/config/mongo_config_header.py
"--output-path",
ctx.outputs.output.path,
"--template-path",
input,
"--check-path",
checks,
"--log-path",
ctx.outputs.logfile.path,
"--compiler-path",
compiler_bin,
"--extra-definitions",
json.encode(expanded_extra_definitions),
] +
additional_inputs +
[
"--compiler-args",
" ".join(compiler_flags),
"--env-vars",
json.encode(env_flags | {"SDKROOT": ctx.attr._sdkroot[sdkroot_provider].path}),
],
)
return [DefaultInfo(files = depset([ctx.outputs.output]))]
generate_config_header_rule = rule(
generate_config_header_impl,
attrs = {
"output": attr.output(
doc = "The output of this rule.",
mandatory = True,
),
"logfile": attr.output(
doc = "The logfile of this rule.",
mandatory = True,
),
"template": attr.label(
doc = "The template file used to generate the header.",
allow_single_file = True,
),
"checks": attr.label(
doc = "The input checks python script to run for this rule.",
allow_single_file = True,
),
"extra_definitions": attr.string_dict(
doc = "Extra definitions to set.",
default = {},
),
"additional_inputs": attr.label_list(
doc = "Additional inputs to this rule.",
allow_files = True,
),
"cpp_linkflags": attr.string_list(
doc = "C++ linkflags.",
),
"cpp_opts": attr.string_list(
doc = "C++ opts.",
),
"cpp_defines": attr.string_list(
doc = "C++ defines.",
),
"generator_script": attr.label(
doc = "The python generator script to use.",
default = "//bazel/config:generate_config_header.py",
allow_single_file = True,
),
"_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),
"_sdkroot": attr.label(default = "//bazel/config:sdkroot"),
},
fragments = ["cpp"],
toolchains = ["@bazel_tools//tools/cpp:toolchain_type", "@bazel_tools//tools/python:toolchain_type"],
output_to_genfiles = True,
)
def generate_config_header(name, tags = [], **kwargs):
generate_config_header_rule(
name = name,
tags = tags + ["gen_source"],
**kwargs
)

79
bazel/config/generate_config_header.py Normal file
View File

@ -0,0 +1,79 @@
# Generates "mongo.h" config header file containing feature flags generated by checking for the availability of certain compiler features.
# This script is invoked by the Bazel build system to generate the "mongo.h" file automatically as part of the build.
# Example usage:
# python generate_config_header.py \
# --compiler-path /usr/bin/gcc --compiler-args "-O2 -Wall" \
# --output-path mongo.h --template-path mongo.h.in \
# --check-path mongo_checks.py --log-path mongo.h.log
import argparse
import inspect
import os
import sys
import textwrap
from typing import Dict
def write_config_header(input_path: str, output_path: str, definitions: Dict[str, str]) -> None:
with open(input_path) as in_file:
content = in_file.read()
with open(output_path, "w", newline="\n") as file:
output_content = content
for key, value in definitions.items():
output_content = output_content.replace(key, value)
file.write(output_content)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Generate a config header file")
parser.add_argument("--compiler-path", help="Path to the compiler executable", required=True)
parser.add_argument("--compiler-args", help="Extra compiler arguments", required=True)
parser.add_argument("--env-vars", help="Extra environment variables", required=True)
parser.add_argument(
"--output-path", help="Path to the output config header file", required=True
)
parser.add_argument(
"--template-path", help="Path to the config header's template file", required=True
)
parser.add_argument("--extra-definitions", help="Extra header definitions")
parser.add_argument("--check-path", help="Path to the suppored configure checks", required=True)
parser.add_argument("--log-path", help="Path to the suppored configure checks", required=True)
parser.add_argument("--additional-input", help="extra files", action="append")
args = parser.parse_args()
sys.path.append(".")
generate_config_module = os.path.splitext(args.check_path)[0].replace("/", ".")
module = __import__(generate_config_module, fromlist=["generate_config_header"])
generate_config_header_func = getattr(module, "generate_config_header")
try:
generate_config_header_args = {
"compiler_path": args.compiler_path,
"compiler_args": args.compiler_args,
"env_vars": args.env_vars,
"logpath": args.log_path,
"additional_inputs": args.additional_input,
"extra_definitions": args.extra_definitions,
}
definitions = generate_config_header_func(**generate_config_header_args)
write_config_header(args.template_path, args.output_path, definitions)
except TypeError as exc:
called_args = inspect.getfullargspec(generate_config_header_func).args
print(
textwrap.dedent(
"""\
ERROR: called import config header generator:
%s
imported "generate_config_header" requires has these args:
%s
but was passed these args:
%s
"""
% (generate_config_module, called_args, list(generate_config_header_args.keys()))
)
)
raise exc

57
bazel/config/render_template.bzl Normal file
View File

@ -0,0 +1,57 @@
load("//bazel:mongo_src_rules.bzl", "write_target")
def render_template_impl(ctx):
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
python_libs = [py_dep[PyInfo].transitive_sources for py_dep in ctx.attr.python_libs]
python_path = []
for py_dep in ctx.attr.python_libs:
for path in py_dep[PyInfo].imports.to_list():
if path not in python_path:
python_path.append(ctx.expand_make_variables("python_library_imports", "$(BINDIR)/external/" + path, ctx.var))
expanded_args = [
ctx.expand_make_variables("render_template_expand", ctx.expand_location(arg, ctx.attr.srcs), ctx.var)
for arg in ctx.attr.cmd
]
ctx.actions.run(
executable = python.interpreter.path,
outputs = [ctx.outputs.output],
inputs = depset(transitive = [python.files, depset([arg.files.to_list()[0] for arg in ctx.attr.srcs])] + python_libs),
arguments = expanded_args,
env = {"PYTHONPATH": ctx.configuration.host_path_separator.join(python_path)},
mnemonic = "TemplateRenderer",
)
return [DefaultInfo(files = depset([ctx.outputs.output]))]
render_template_rule = rule(
render_template_impl,
attrs = {
"srcs": attr.label_list(
doc = "The input files of this rule.",
allow_files = True,
),
"output": attr.output(
doc = "The output of this rule.",
mandatory = True,
),
"cmd": attr.string_list(
doc = "The command line arguments to pass to python",
),
"python_libs": attr.label_list(
providers = [PyInfo],
default = [],
),
},
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
output_to_genfiles = True,
)
def render_template(name, tags = [], **kwargs):
render_template_rule(
name = name,
tags = tags + ["gen_source"],
**kwargs
)

5
bazel/coverity/BUILD.bazel Normal file
View File

@ -0,0 +1,5 @@
py_binary(
name = "generate_coverity_command",
srcs = ["generate_coverity_command.py"],
visibility = ["//visibility:public"],
)

0
src/third_party/scons-4.9.1/scons-local-4.9.1/SCons/Utilities/__init__.py → bazel/coverity/coverity_toolchain.BUILD
View File

35
bazel/coverity/coverity_toolchain.bzl Normal file
View File

@ -0,0 +1,35 @@
def _coverity_toolchain(ctx):
retCode = 1
if "COVERITY_INSTALL_ROOT" in ctx.os.environ:
result = ctx.execute([
"ls",
ctx.getenv("COVERITY_INSTALL_ROOT") + "/bin/cov-build",
])
retCode = result.return_code
if retCode == 0:
ctx.report_progress("extracting coverity rules...")
result = ctx.download_and_extract("file://" + ctx.getenv("COVERITY_INSTALL_ROOT") + "/bazel/rules_coverity.tar.gz")
else:
ctx.template(
"coverity/BUILD.bazel",
ctx.attr.build_tpl,
)
ctx.template(
"coverity/repositories.bzl",
ctx.attr.repositories_tpl,
)
coverity_toolchain = repository_rule(
implementation = _coverity_toolchain,
attrs = {
"build_tpl": attr.label(
default = "//bazel/coverity:coverity_toolchain.BUILD",
doc = "Label denoting the BUILD file template that gets installed in the repo.",
),
"repositories_tpl": attr.label(
default = "//bazel/coverity:repositories.bzl",
doc = "Label denoting the repositories files the gets installed to the repo.",
),
},
)

61
bazel/coverity/generate_coverity_command.py Normal file
View File

@ -0,0 +1,61 @@
import argparse
import os
import subprocess
parser = argparse.ArgumentParser(description="Generate coverity build file.")
parser.add_argument("--bazel_executable", required=True)
parser.add_argument("--bazel_cache", required=True)
parser.add_argument("--bazel_query", required=True)
args, unknownargs = parser.parse_known_args()
bazel_cmd_args = unknownargs
bazel_executable = os.path.expanduser(args.bazel_executable)
bazel_cache = os.path.expanduser(args.bazel_cache)
# coverity requires a single target which has dependencies on all
# the cc_library and cc_binaries in our build. There is not a good way from
# within the build to get all those targets, so we will generate the list via query
# https://sig-product-docs.synopsys.com/bundle/coverity-docs/page/coverity-analysis/topics/building_with_bazel.html#build_with_bazel
proc = subprocess.run(
[
bazel_executable,
bazel_cache,
"aquery",
]
+ bazel_cmd_args
+ [args.bazel_query],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
targets = set()
for line in proc.stdout.splitlines():
if line.startswith(" Target: "):
targets.add(line.split()[-1])
enterprise_coverity_dir = os.path.join("src", "mongo", "db", "modules", "enterprise", "coverity")
os.makedirs(enterprise_coverity_dir, exist_ok=True)
with open(os.path.join(enterprise_coverity_dir, "BUILD.bazel"), "w") as buildfile:
buildfile.write("""\
load("@rules_coverity//coverity:defs.bzl", "cov_gen_script")
cov_gen_script(
name="enterprise_coverity_build",
testonly=True,
tags=["coverity"],
deps=[
""")
for target in targets:
buildfile.write(
"""\
"%s",
"""
% target
)
buildfile.write("""\
],
)
""")

2
bazel/coverity/repositories.bzl Normal file
View File

@ -0,0 +1,2 @@
def rules_coverity_toolchains():
return None

12
bazel/docs/architecture/ppc64le_build_from_source.md
View File

@ -6,14 +6,14 @@ Bazel doesn't release to the PPC64LE architecture. To address this, MongoDB main
Bazel usually comes with a built-in JDK. However, the tooling used to build the built-in JDK doesn't support PPC64LE. To get around this, an external JDK must be present on both the system compiling the Bazel executable itself as well as the host running Bazel as a build system.
On the MongoDB PPC64LE Evergreen static hosts and dev hosts, the OpenJDK 11 installation exists at:
On the MongoDB PPC64LE Evergreen static hosts and dev hosts, the OpenJDK 21 installation exists at:
/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le
/usr/lib/jvm/java-21-openjdk
To compile with on these platforms, the developer must set JAVA_HOME before invoking Bazel.
# Bazel v6.4.0 Compilation Steps
# Bazel v7.2.1 Compilation Steps
curl -O -L https://github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-dist.zip
unzip bazel-6.4.0-dist.zip
JAVA_HOME=/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le ./compile.sh
curl -O -L https://github.com/bazelbuild/bazel/releases/download/7.2.1/bazel-7.2.1-dist.zip
unzip bazel-7.2.1-dist.zip
JAVA_HOME=/usr/lib/jvm/java-21-openjdk ./compile.sh

12
bazel/docs/architecture/s390x_build_from_source.md
View File

@ -6,14 +6,14 @@ Bazel doesn't release to the S390X architecture. To address this, MongoDB mainta
Bazel usually comes with a built-in JDK. However, the tooling used to build the built-in JDK doesn't support S390X. To get around this, an external JDK must be present on both the system compiling the Bazel executable itself as well as the host running Bazel as a build system.
On the MongoDB S390X Evergreen static hosts and dev hosts, the OpenJDK 11 installation exists at:
On the MongoDB S390X Evergreen static hosts and dev hosts, the OpenJDK 21 installation exists at:
/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x
/usr/lib/jvm/java-21-openjdk-21
To compile with on these platforms, the developer must set JAVA_HOME before invoking Bazel.
# Bazel v6.4.0 Compilation Steps
# Bazel v7.2.1 Compilation Steps
curl -O -L https://github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-dist.zip
unzip bazel-6.4.0-dist.zip
JAVA_HOME=/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x ./compile.sh
curl -O -L https://github.com/bazelbuild/bazel/releases/download/7.2.1/bazel-7.2.1-dist.zip
unzip bazel-7.2.1-dist.zip
JAVA_HOME=/usr/lib/jvm/java-21-openjdk-21 ./compile.sh

95
bazel/docs/developer_workflow.md
View File

@ -4,26 +4,24 @@ This document describes the Server Developer workflow for modifying Bazel build
# Creating a new BUILD.bazel file
Similar to SCons, a build target is defined in the directory where its source code exists. To create a target that compiles **src/mongo/hello_world.cpp**, you would create **src/mongo/BUILD.bazel**.
The Bazel equivalent of SConscript files are BUILD.bazel files.
A build target is defined in the directory where its source code exists. To create a target that compiles **src/mongo/hello_world.cpp**, you would create **src/mongo/BUILD.bazel**.
src/mongo/BUILD.bazel would contain:
mongo_cc_binary(
name = "hello_world",
srcs = [
"hello_world.cpp"
],
"hello_world.cpp"
],
}
Once you've obtained bazelisk by running **evergreen/get_bazelisk.sh**, you can then build this target via "bazelisk build":
Once you've obtained bazel by running **python buildscripts/install_bazel.py**, you can then build this target via "bazel build":
./bazelisk build //src/mongo:hello_world
bazel build //src/mongo:hello_world
Or run this target via "bazelisk run":
Or run this target via "bazel run":
./bazelisk run //src/mongo:hello_world
bazel run //src/mongo:hello_world
The full target name is a combination between the directory of the BUILD.bazel file and the target name:
@ -33,39 +31,37 @@ The full target name is a combination between the directory of the BUILD.bazel f
Bazel makes use of static analysis wherever possible to improve execution and querying speed. As part of this, source and header files must not be declared dynamically (ex. glob, wildcard, etc). Instead, you'll need to manually add a reference to each header or source file you add into your build target.
The divergence from SCons is that now source files have to be declared in addition to header files.
mongo_cc_binary(
name = "hello_world",
srcs = [
"hello_world.cpp",
"new_source.cpp" # If adding a source file
],
"hello_world.cpp",
"new_source.cpp" # If adding a source file
],
hdrs = [
"new_header.h" # If adding a header file
],
"new_header.h" # If adding a header file
],
}
## Adding a New Library
The DevProd Build Team created MongoDB-specific macros for the different types of build targets you may want to specify. These include:
- mongo_cc_binary
- mongo_cc_library
- idl_generator
- mongo_cc_binary
- mongo_cc_library
- idl_generator
Creating a new library is similar to the steps above for creating a new binary. A new **mongo_cc_library** definition would be created in the BUILD.bazel file.
mongo_cc_library(
name = "new_library",
srcs = [
"new_library_source_file.cpp"
]
"new_library_source_file.cpp"
]
}
## Declaring Dependencies
If a library or binary depends on another library, this must be declared in the **deps** section of the target. The syntax for referring to the library is the same syntax used in the bazelisk build/run command.
If a library or binary depends on another library, this must be declared in the **deps** section of the target. The syntax for referring to the library is the same syntax used in the bazel build/run command.
mongo_cc_library(
name = "new_library",
@ -75,27 +71,46 @@ If a library or binary depends on another library, this must be declared in the
mongo_cc_binary(
name = "hello_world",
srcs = [
"hello_world.cpp",
],
"hello_world.cpp",
],
deps = [
":new_library", # if referring to the library declared in the same directory as this build file
# "//src/mongo:new_library" # absolute path
# "sub_directory:new_library" # relative path of a subdirectory
],
":new_library", # if referring to the library declared in the same directory as this build file
# "//src/mongo:new_library" # absolute path
# "sub_directory:new_library" # relative path of a subdirectory
],
}
## Depending on a Bazel Library in a SCons Build Target
## Running clang-tidy via Bazel
During migration from SCons to Bazel, the Build Team has created an integration layer between the two while working towards converting all SCons targets to Bazel targets.
Note: This feature is still in development; see https://jira.mongodb.org/browse/SERVER-80396 for details)
This allows SCons build targets to depend on Bazel build targets directly. The Bazel targets depended on by the SCons build target will be built with the normal scons.py invocation automatically.
To run clang-tidy via Bazel, do the following:
env.BazelLibrary(
target='fsync_locked',
source=[
'fsync_locked.cpp',
],
LIBDEPS=[
'new_library', # depend on the bazel "new_library" target defined above
],
)
1. To analyze all code, run `bazel build --config=clang-tidy src/...`
2. To analyze a single target (e.g.: `fsync_locked`), run the following command (note that `_with_debug` suffix on the target): `bazel build --config=clang-tidy src/mongo/db/commands:fsync_locked_with_debug`
Testing notes:
- If you want to test whether clang-tidy is in fact finding bugs, you can inject the following code into a `cpp` file to generate a `bugprone-incorrect-roundings` warning:
```
const double f = 1.0;
const int foo = (int)(f + 0.5);
```
# Frequently Asked Questions
### The header I want to add is referenced in several places, how do I figure out where to add references to it in the BUILD.bazel files?
Follow this loop to figure out where the header needs to be added
1. Build directly with bazel to speed up the loop: `bazel build //src/...`
2. This will fail on the first missing header dependency, search the bazel build files for the library the header is defined on. Currently there are cases where headers are incorrectly located so you'll need to use your best judgement. If the header exists on some library, add that library as a dep, for example `scoped_timer.h` is part of `scope_timer` library so add `//src/mongo/db/exec:scoped_timer` to deps field (this will take care of `scoped_timer.h` transitive dependencies). If not add the header directly to the hdrs field of the library that's failing to compile.
3. Build directly with bazel `bazel build //src/...`
4. If there is a cycle remove the dependency from Step #2, add the header as direct dependency to the hdrs field, and then start back at Step #1
### The header I want to add is referenced in dozens or more locations, and adding it to the proper location requires a large refactor that is blocking critical work, what should I do?
If you've put in a significant amount of work to try to get a header added and have found to get it added to the right place (usually alongside the associated .cpp file, having all dependents add that library as a dep) will take a significant refactor, create a SERVER ticket explaining the problem, solution, and complexity required to resolve it. Then, open up src/mongo/BUILD.bazel and add the header to "core_headers" file group referencing your ticket in a TODO comment.
This is very much a last resort and should only be done if the refactor will take a very significant amount of time and is blocking other work.

21
bazel/docs/engflow_credential_setup.md
View File

@ -2,23 +2,6 @@
MongoDB uses EngFlow to enable remote execution with Bazel. This dramatically speeds up the build process, but is only available to internal MongoDB employees.
To install the necessary credentials to enable remote execution, run scons.py with any build command, then follow the setup instructions it prints out. Or:
Bazel uses a wrapper script to check the credentials on each invocation, if for some reason thats not working, you can also manually perform this process with this command alternatively:
(Only if not in the Engineering org)
- Request access to the MANA group https://mana.corp.mongodbgov.com/resources/659ec4b9bccf3819e5608712
(For everyone)
- Go to https://sodalite.cluster.engflow.com/gettingstarted
- Login with OKTA, then click the "GENERATE AND DOWNLOAD MTLS CERTIFICATE" button
- (If logging in with OKTA doesn't work) Login with Google using your MongoDB email, then click the "GENERATE AND DOWNLOAD MTLS CERTIFICATE" button
- On your local system (usually your MacBook), open a shell terminal and, after setting the variables on the first three lines, run:
REMOTE_USER=<SSH User from https://spruce.mongodb.com/spawn/host>
REMOTE_HOST=<DNS Name from https://spruce.mongodb.com/spawn/host>
ZIP_FILE=~/Downloads/engflow-mTLS.zip
curl https://raw.githubusercontent.com/mongodb/mongo/master/buildscripts/setup_engflow_creds.sh -o setup_engflow_creds.sh
chmod +x ./setup_engflow_creds.sh
./setup_engflow_creds.sh $REMOTE_USER $REMOTE_HOST $ZIP_FILE
python buildscripts/engflow_auth.py

8
bazel/docs/header_cycle_resolution.md Normal file
View File

@ -0,0 +1,8 @@
# Header Relocation and Cycle Resolution
1. Locate all the targets that reference the header file in BUILD.bazel files.
2. Find an ideal target to declare the header under. This is usually under the target that features the .cpp file of the same name. Otherwise, the header can be placed in its own library.
3. Ensure that all the targets that need this header can depend on the target the header was moved to.
4. Run `bazel build //src/...` to check for build failures (look for failures related to dependency cycles).
5. If the build fails because of a dependency cycle, you may need to split up the dependent library or relocate the header.
6. Once the build succeeds, please create a PR and include `devprod-build` for review.

15
bazel/docs/rbe_images.md Normal file
View File

@ -0,0 +1,15 @@
# Remote execution images
The Dockerfiles for remote execution images are autogenerated to pin all
versions and allow for updates at the same time. To repin the image hashes and
package versions:
```bash
# With Bazel
bazel run //bazel/remote_execution_container:repin_dockerfiles --config=local
# Without Bazel
./repin_dockerfiles.sh
```
To change the image contents, edit [repin_dockerfiles.sh](./repin_dockerfiles.sh).

60
bazel/format/BUILD.bazel
View File

@ -1,19 +1,47 @@
load("@aspect_rules_lint//format:defs.bzl", "multi_formatter_binary")
load("@npm//:prettier/package_json.bzl", prettier = "bin")
load("@aspect_rules_lint//format:defs.bzl", "format_multirun", "format_test")
# TODO: SERVER-82329 eslint binary should almost exactly mirror prettier binary
# To update prettier change the version in package.json
# Run `pnpm install`
# Commit changes
prettier.prettier_binary(
name = "prettier",
# Allow the binary to be run outside bazel
# See more details about this by commenting this out and running `bazel run //:format`
env = {"BAZEL_BINDIR": "."},
)
multi_formatter_binary(
name = "format",
markdown = ":prettier",
py_binary(
name = "rules_lint_format_wrapper",
srcs = ["rules_lint_format_wrapper.py"],
args = [
"--prettier",
"$(location //:prettier)",
"--shellscripts-linters",
"$(location //buildscripts:shellscripts_linters)",
"--rules-lint-format",
"$(location :rules_lint_format)",
"--rules-lint-format-check",
"$(location :rules_lint_format.check)",
],
data = [
":rules_lint_format",
":rules_lint_format.check",
"//:prettier",
"//buildscripts:shellscripts_linters",
"@shfmt",
],
env = {
"SHFMT_PATH": "$(execpath @shfmt//:shfmt)",
},
main = "rules_lint_format_wrapper.py",
visibility = ["//visibility:public"],
deps = [
"//buildscripts:unittest_grouper",
],
)
format_multirun(
name = "rules_lint_format",
# disabled until it stops formatting third party code
c = "//:clang_format",
cc = "//:clang_format",
css = "//:prettier",
graphql = "//:prettier",
html = "//:prettier",
markdown = "//:prettier",
sql = "//:prettier",
starlark = "@buildifier_prebuilt//:buildifier",
visibility = ["//visibility:public"],
# TODO(SERVER-101034): Enable rules_lint shfmt after sh files are reformatted with .editorconfig
# shell = "@shfmt//:shfmt",
)

234
bazel/format/rules_lint_format_wrapper.py Normal file
View File

@ -0,0 +1,234 @@
import argparse
import os
import pathlib
import subprocess
from typing import List, Union
from buildscripts.unittest_grouper import validate_bazel_groups
def _git_distance(args: list) -> int:
command = ["git", "rev-list", "--count"] + args
try:
result = subprocess.run(command, capture_output=True, text=True, check=True)
except subprocess.CalledProcessError as e:
print(f"Error running git command: {' '.join(command)}")
print(f"stderr: {e.stderr.strip()}")
print(f"stdout: {e.stdout.strip()}")
raise
return int(result.stdout.strip())
def _get_merge_base(args: list) -> str:
command = ["git", "merge-base"] + args
result = subprocess.run(command, capture_output=True, text=True, check=True)
return result.stdout.strip()
def _git_diff(args: list) -> str:
command = ["git", "diff"] + args
result = subprocess.run(command, capture_output=True, text=True, check=True)
return result.stdout.strip() + os.linesep
def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> List[str]:
"""Query git to get a list of files in the repo from a diff."""
# There are 3 diffs we run:
# 1. List of commits between origin/master and HEAD of current branch
# 2. Cached/Staged files (--cached)
# 3. Working Tree files git tracks
fork_point = _get_merge_base(["HEAD", origin_branch])
diff_files = _git_diff(["--name-only", f"{fork_point}..HEAD"])
diff_files += _git_diff(["--name-only", "--cached"])
diff_files += _git_diff(["--name-only"])
file_set = {
os.path.normpath(os.path.join(os.curdir, line.rstrip()))
for line in diff_files.splitlines()
if line
}
return list(file_set)
def run_rules_lint(
rules_lint_format_path: pathlib.Path,
rules_lint_format_check_path: pathlib.Path,
check: bool,
files_to_format: Union[List[str], str] = "all",
) -> bool:
try:
if check:
command = [str(rules_lint_format_check_path)]
print("Running rules_lint formatter in check mode")
else:
command = [str(rules_lint_format_path)]
print("Running rules_lint formatter")
if files_to_format != "all":
command += files_to_format
repo_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
subprocess.run(command, check=True, env=os.environ, cwd=repo_path)
except subprocess.CalledProcessError:
return False
return True
def run_shellscripts_linters(shellscripts_linters: pathlib.Path, check: bool) -> bool:
try:
command = [str(shellscripts_linters)]
if not check:
print("Running shellscripts formatter")
command.append("fix")
else:
print("Running shellscripts linter")
repo_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
subprocess.run(command, check=True, env=os.environ, cwd=repo_path)
except subprocess.CalledProcessError:
return False
return True
def run_prettier(
prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all"
) -> bool:
# Explicitly ignore anything in the output directories or any symlinks in the root of the repository
# to prevent bad symlinks from failing the run, see https://github.com/prettier/prettier/issues/11568 as
# to why it the paths being present in .prettierignore isn't sufficient
force_exclude_dirs = {
"!./build",
"!./bazel-bin",
"!./bazel-out",
"!./bazel-mongo",
"!./external",
}
for path in pathlib.Path(".").iterdir():
if path.is_symlink():
force_exclude_dirs.add(f"!./{path}")
try:
command = [
str(prettier),
"--cache",
"--log-level",
"warn",
]
if files_to_format == "all":
command += ["."]
else:
command += files_to_format
command += list(force_exclude_dirs)
if check:
command.append("--check")
else:
command.append("--write")
print("Running prettier")
subprocess.run(command, check=True)
except subprocess.CalledProcessError:
print("Found formatting errors. Run 'bazel run //:format' to fix")
print("*** IF BAZEL IS NOT INSTALLED, RUN THE FOLLOWING: ***\n")
print("python buildscripts/install_bazel.py")
if os.path.exists("external"):
print(
"\nexternal exists which may be causing issues in the linter, please try running:\n"
)
print("\tunlink external")
return False
if check:
print("No formatting errors")
return True
def main() -> int:
# If we are running in bazel, default the directory to the workspace
default_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY")
if not default_dir:
print("This script must be run though bazel. Please run 'bazel run //:format' instead")
print("*** IF BAZEL IS NOT INSTALLED, RUN THE FOLLOWING: ***\n")
print("python buildscripts/install_bazel.py")
return 1
parser = argparse.ArgumentParser(
prog="Format", description="This script formats code in mongodb"
)
parser.add_argument("--check", help="Run in check mode", default=False, action="store_true")
parser.add_argument(
"--prettier", help="Set the path to prettier", required=True, type=pathlib.Path
)
parser.add_argument(
"--shellscripts-linters",
help="Set the path to shellscripts_linters",
required=True,
type=pathlib.Path,
)
parser.add_argument(
"--rules-lint-format",
help="Set the path to rules_lint's formatter",
required=True,
type=pathlib.Path,
)
parser.add_argument(
"--rules-lint-format-check",
help="Set the path to rules_lint's formatter check script",
required=True,
type=pathlib.Path,
)
parser.add_argument(
"--all",
help="Format all files instead of just formatting files that have changed since the fork point",
action="store_true",
)
parser.add_argument(
"--origin-branch",
help="The branch to use as the fork point for changed files",
default="origin/v8.0",
)
args = parser.parse_args()
prettier_path: pathlib.Path = args.prettier.resolve()
shellscripts_linters_path: pathlib.Path = args.shellscripts_linters.resolve()
os.chdir(default_dir)
files_to_format = "all"
if not args.all:
distance = _git_distance([f"{args.origin_branch}..HEAD"])
if distance > 100:
print(
f"The number of commits between current branch and origin branch ({args.origin_branch}) is too large: {distance} commits"
)
print("WARNING!!! Defaulting to formatting all files, this may take a while.")
print(
"Please update your local branch with the latest changes from origin, or use `bazel run format -- --origin-branch other_branch` to select a different origin branch"
)
args.all = True
else:
files_to_format = _get_files_changed_since_fork_point(args.origin_branch)
def files_to_format_contains_bazel_file(files: Union[List[str], str]) -> bool:
if files == "all":
return True
return any(file.endswith(".bazel") or "BUILD" in file for file in files)
if files_to_format_contains_bazel_file(files_to_format):
validate_bazel_groups(generate_report=True, fix=not args.check)
if files_to_format != "all":
files_to_format = [str(file) for file in files_to_format if os.path.isfile(file)]
return (
0
if run_rules_lint(
args.rules_lint_format, args.rules_lint_format_check, args.check, files_to_format
)
and run_shellscripts_linters(shellscripts_linters_path, args.check)
and run_prettier(prettier_path, args.check, files_to_format)
else 1
)
if __name__ == "__main__":
exit(main())

66
bazel/format/shfmt.bzl Normal file
View File

@ -0,0 +1,66 @@
"""Repository rules for shfmt binary download"""
load("//bazel:utils.bzl", "retry_download")
load("@bazel_rules_mongo//utils:platforms_normalize.bzl", "ARCH_NORMALIZE_MAP", "OS_NORMALIZE_MAP")
URLS_MAP = {
"linux_aarch64": {
"sha": "9d23013d56640e228732fd2a04a9ede0ab46bc2d764bf22a4a35fb1b14d707a8",
"url": "https://github.com/mvdan/sh/releases/download/v3.10.0/shfmt_v3.10.0_linux_arm64",
},
"linux_x86_64": {
"sha": "1f57a384d59542f8fac5f503da1f3ea44242f46dff969569e80b524d64b71dbc",
"url": "https://github.com/mvdan/sh/releases/download/v3.10.0/shfmt_v3.10.0_linux_amd64",
},
"macos_aarch64": {
"sha": "86030533a823c0a7cd92dee0f74094e5b901c3277b43def6337d5e19e56fe553",
"url": "https://github.com/mvdan/sh/releases/download/v3.10.0/shfmt_v3.10.0_darwin_arm64",
},
"macos_x86_64": {
"sha": "ef8d970b3f695a7e8e7d40730eedd2d935ab9599f78a365f319c515bc59d4c83",
"url": "https://github.com/mvdan/sh/releases/download/v3.10.0/shfmt_v3.10.0_darwin_amd64",
},
"windows_x86_64": {
"sha": "6e4c6acd38de7b4b1ba8f8082b9e688df8c9b861d3f8b2e7bb1b7270201a3587",
"url": "https://github.com/mvdan/sh/releases/download/v3.10.0/shfmt_v3.10.0_windows_amd64.exe",
},
}
def _shfmt_download(ctx):
"""
Downloads a shfmt binary
Args:
ctx: Repository context.
"""
os = ctx.os.name
arch = ctx.os.arch
os_constraint = OS_NORMALIZE_MAP[os]
arch_constraint = ARCH_NORMALIZE_MAP[arch]
platform_info = URLS_MAP["{os}_{arch}".format(os = os_constraint, arch = arch_constraint)]
ctx.report_progress("downloading shfmt")
retry_download(
ctx = ctx,
executable = True,
output = "shfmt",
tries = 5,
url = platform_info["url"],
sha256 = platform_info["sha"],
)
ctx.file(
"BUILD.bazel",
"""
package(default_visibility = ["//visibility:public"])
exports_files(["shfmt"])
""",
)
return None
_shfmt = repository_rule(
implementation = _shfmt_download,
attrs = {},
)
def shfmt():
_shfmt(name = "shfmt")

50
bazel/header_deps.bzl Normal file
View File

@ -0,0 +1,50 @@
HEADER_DEP_SUFFIX = "_header_dep"
def create_header_dep_impl(ctx):
compilation_context = cc_common.create_compilation_context(
includes = depset(transitive = [header_dep[CcInfo].compilation_context.includes for header_dep in ctx.attr.header_deps]),
headers = depset(transitive = [header_dep[CcInfo].compilation_context.headers for header_dep in ctx.attr.header_deps]),
)
return CcInfo(compilation_context = compilation_context)
create_header_dep = rule(
create_header_dep_impl,
attrs = {
"header_deps": attr.label_list(providers = [CcInfo]),
},
doc = "create header only CcInfo",
fragments = ["cpp"],
)
def create_link_dep_impl(ctx):
deps = []
for dep in ctx.attr.link_deps:
if dep[CcInfo].linking_context:
for input in dep[CcInfo].linking_context.linker_inputs.to_list():
for library in input.libraries:
if library.dynamic_library and library.resolved_symlink_dynamic_library:
dep = library.resolved_symlink_dynamic_library.path
if dep not in deps:
deps.append(library.resolved_symlink_dynamic_library.path)
if library.static_library:
dep = library.static_library.path
if dep not in deps:
deps.append(library.static_library.path)
link_list = ctx.actions.declare_file(ctx.attr.target_name + "_links.list")
ctx.actions.write(
output = link_list,
content = "\n".join(deps),
)
return DefaultInfo(files = depset([link_list]))
create_link_deps = rule(
create_link_dep_impl,
attrs = {
"target_name": attr.string(),
"link_deps": attr.label_list(providers = [CcInfo]),
},
doc = "create a pseudo target to query link deps for",
)

7
bazel/install_rules/BUILD.bazel Normal file
View File

@ -0,0 +1,7 @@
package(default_visibility = ["//visibility:public"])
# Expose script for external usage through bazel.
exports_files([
"install_rules.py",
"pretty_printer_test_creator.py",
])

596
bazel/install_rules/install_rules.bzl Normal file
View File

@ -0,0 +1,596 @@
"""
Sets up install and archive rules.
"""
load("@rules_pkg//:pkg.bzl", "pkg_tar", "pkg_zip")
load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files")
load("@rules_pkg//pkg:providers.bzl", "PackageFilesInfo")
load("@bazel_skylib//lib:paths.bzl", "paths")
load("//bazel:mongo_src_rules.bzl", "SANITIZER_DATA", "SANITIZER_ENV")
load("//bazel:separate_debug.bzl", "TagInfo")
load("//bazel/install_rules:pretty_printer_tests.bzl", "mongo_pretty_printer_test")
load("//bazel/install_rules:providers.bzl", "TestBinaryInfo")
load("//bazel/toolchains/cc:mongo_errors.bzl", "DWP_ERROR_MESSAGE")
# Used to skip rules on certain OS architectures
def _empty_rule_impl(ctx):
pass
empty_rule = rule(
implementation = _empty_rule_impl,
)
MongoInstallInfo = provider(
doc = "A install rule provider to pass around deps files",
fields = {
"deps_files": "Install rule file describing the files installed for passing to script",
"test_file": "File containing list of installed tests",
"src_map": "contents of the dep file for use in rules",
},
)
# This is a dictionary because there are no sets in bazel
# and we want to look up if the value exists quickly
TEST_TAGS = {
"bsoncolumn_bm": 1,
"mongo_benchmark": 1,
"mongo_fuzzer_test": 1,
"mongo_integration_test": 1,
"mongo_unittest": 1,
"mongo_unittest_first_group": 1,
"mongo_unittest_second_group": 1,
"mongo_unittest_third_group": 1,
"mongo_unittest_fourth_group": 1,
"mongo_unittest_fifth_group": 1,
"mongo_unittest_sixth_group": 1,
"mongo_unittest_seventh_group": 1,
"mongo_unittest_eighth_group": 1,
"query_bm": 1,
"repl_bm": 1,
"sharding_bm": 1,
"sep_bm": 1,
"storage_bm": 1,
"first_half_bm": 1,
}
def test_binary_aspect_impl(target, ctx):
"""Collect all test binaries from transitive srcs and deps
Args:
target: current target
ctx: context of current target
Returns:
struct containing collected test binaries
"""
transitive_deps = []
if TestBinaryInfo in target:
return []
if TagInfo in target:
for tag in target[TagInfo].tags:
if tag in TEST_TAGS:
transitive_deps.append(target.files)
break
if hasattr(ctx.rule.attr, "srcs"):
for src in ctx.rule.attr.srcs:
if TestBinaryInfo in src:
transitive_deps.append(src[TestBinaryInfo].test_binaries)
if hasattr(ctx.rule.attr, "deps"):
for dep in ctx.rule.attr.deps:
if TestBinaryInfo in dep:
transitive_deps.append(dep[TestBinaryInfo].test_binaries)
test_binaries = depset(transitive = transitive_deps)
return [TestBinaryInfo(test_binaries = test_binaries)]
test_binary_aspect = aspect(
implementation = test_binary_aspect_impl,
attr_aspects = ["srcs", "deps"],
)
def get_constraints(ctx):
"""Return rule time constaints.
Args:
ctx: rule ctx
Returns:
3 element tuple with each OS constraint.
"""
linux_constraint = ctx.attr._linux_constraint[platform_common.ConstraintValueInfo]
macos_constraint = ctx.attr._macos_constraint[platform_common.ConstraintValueInfo]
windows_constraint = ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]
return linux_constraint, macos_constraint, windows_constraint
def is_binary_file(ctx, basename):
"""Check if file looks like a binary
Args:
ctx: rule ctx
file: file to check
Returns:
True if it looks like a binary, False otherwise
"""
linux_constraint, macos_constraint, windows_constraint = get_constraints(ctx)
if ctx.target_platform_has_constraint(linux_constraint):
return not (basename.startswith("lib") or basename.startswith("mongo_crypt_v") or basename.startswith("stitch_support.so"))
elif ctx.target_platform_has_constraint(macos_constraint):
return not (basename.startswith("lib") or basename.startswith("mongo_crypt_v") or basename.startswith("stitch_support.dylib"))
elif ctx.target_platform_has_constraint(windows_constraint):
return basename.endswith(".exe") or basename.endswith(".pdb") or basename.endswith(".dll") or basename.endswith(".ps1")
else:
ctx.fail("Unknown OS")
return False
def is_debug_file(ctx, basename):
"""Check if file looks a debug file
Args:
ctx: rule ctx
file: file to check
Returns:
True if it looks like a debug file, False otherwise
"""
linux_constraint, macos_constraint, windows_constraint = get_constraints(ctx)
if ctx.target_platform_has_constraint(linux_constraint):
return basename.endswith(".debug") or basename.endswith(".dwp")
elif ctx.target_platform_has_constraint(macos_constraint):
return basename.endswith(".dSYM")
elif ctx.target_platform_has_constraint(windows_constraint):
return basename.endswith(".pdb")
else:
ctx.fail("Unknown OS")
return False
def declare_output(ctx, output, is_directory):
"""Declare an output as either a file or directory
Args:
ctx: rule ctx
output: output file to declare
is_directory: determines if the file is a directory
Returns:
File object representing output
"""
if is_directory:
return ctx.actions.declare_directory(output)
else:
return ctx.actions.declare_file(output)
def sort_file(ctx, file, install_dir, file_map, is_directory):
"""Determine location a file should be installed to
Args:
ctx: rule ctx
file: file to sort
install_dir: the directory to install to
file_map: dict containing specific file designations
is_directory: determines if the file is a directory
"""
_, macos_constraint, _ = get_constraints(ctx)
basename = paths.basename(file)
bin_install = install_dir + "/bin/" + basename
if bin_install.endswith(".dwp"):
# Due to us creating our binaries using the _with_debug name
# the dwp files also contain it. Strip the _with_debug from the name
bin_install = bin_install.replace("_with_debug.dwp", ".dwp")
lib_install = install_dir + "/lib/" + basename
if is_binary_file(ctx, basename) or basename.endswith(".py"):
if not is_debug_file(ctx, basename):
if ctx.attr.debug != "debug":
file_map["binaries"][file] = declare_output(ctx, bin_install, is_directory)
elif ctx.attr.debug != "stripped" or ctx.attr.publish_debug_in_stripped:
file_map["binaries_debug"][file] = declare_output(ctx, bin_install, is_directory)
elif not is_debug_file(ctx, basename):
if ctx.attr.debug != "debug":
file_map["dynamic_libs"][file] = declare_output(ctx, lib_install, is_directory)
elif ctx.attr.debug != "stripped" or ctx.attr.publish_debug_in_stripped:
file_map["dynamic_libs_debug"][file] = declare_output(ctx, lib_install, is_directory)
def mongo_install_rule_impl(ctx):
"""Perform install actions
Args:
ctx: rule ctx
Returns:
DefaultInfo: with dep files and output file
PackageFilesInfo: with a mapping for creating the archive
"""
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
file_map = {
"binaries": {},
"binaries_debug": {},
"dynamic_libs_debug": {},
"dynamic_libs": {},
"root_files": {},
}
test_files = []
outputs = []
dwps = []
install_dir = ctx.label.name
# sort direct sources
for input_bin in ctx.attr.srcs:
if DebugPackageInfo in input_bin and ctx.attr.create_dwp and ctx.attr.debug != "stripped":
bin = input_bin[DebugPackageInfo].dwp_file
dwps.append(bin)
sort_file(ctx, bin.path, install_dir, file_map, bin.is_directory)
test_files.extend(input_bin[TestBinaryInfo].test_binaries.to_list())
for bin in input_bin.files.to_list():
sort_file(ctx, bin.path, install_dir, file_map, bin.is_directory)
for input_label, output_folder in ctx.attr.root_files.items():
for file in input_label.files.to_list():
file_map["root_files"][file.path] = declare_output(ctx, install_dir + "/" + output_folder + "/" + file.basename, file.is_directory)
# sort dependency install files
for dep in ctx.attr.deps:
test_files.extend(dep[TestBinaryInfo].test_binaries.to_list())
# Create a map of filename to if its a directory, ie. { coolfolder: True, coolfile: False } as the json loses that info
file_directory_map = {file_dep.basename: file_dep.is_directory for file_dep in dep[DefaultInfo].files.to_list()}
src_map = json.decode(dep[MongoInstallInfo].src_map.to_list()[0])
files = []
for key in src_map:
if key != "roots":
files.extend(src_map[key])
for file in files:
filename = file.split("/")[-1]
# Due to us creating our binaries using the _with_debug name
# the dwp files also contain it. Strip the _with_debug from the name
filename = filename.replace("_with_debug.dwp", ".dwp")
sort_file(ctx, file, install_dir, file_map, file_directory_map[filename])
for file, folder in src_map["roots"].items():
filename = file.split("/")[-1]
file_map["root_files"][file] = declare_output(ctx, install_dir + "/" + folder + "/" + filename, file_directory_map[filename])
# aggregate based on type of installs
if ctx.attr.debug == "stripped" and not ctx.attr.publish_debug_in_stripped:
bins = [bin for bin in file_map["binaries"]]
libs = [lib for lib in file_map["dynamic_libs"]]
elif ctx.attr.debug == "debug":
bins = [bin for bin in file_map["binaries_debug"]]
libs = [lib for lib in file_map["dynamic_libs_debug"]]
else:
bins = [bin for bin in file_map["binaries"]] + [bin for bin in file_map["binaries_debug"]]
libs = [lib for lib in file_map["dynamic_libs"]] + [lib for lib in file_map["dynamic_libs_debug"]]
root_files = [root_file for root_file in file_map["root_files"]]
unittest_bin = None
if len(bins) == 1 and ctx.attr.debug != "debug" and file_map["binaries"][bins[0]].basename.endswith("_test") and len(root_files) == 0:
unittest_bin = file_map["binaries"][bins[0]]
# Write installed_tests.txt which contains the list of all test files installed
input_deps = []
installed_tests = []
for file in test_files:
if not is_debug_file(ctx, file.basename) and ctx.attr.debug != "debug":
if is_binary_file(ctx, file.basename) or file.basename.endswith(".py"):
test_path = file_map["binaries"][file.path].path
# point at the binaries in bazel-bin/install/ rather than bazel-out/<some-arch>/bin/<some-install>/
split_test_path = test_path.split("/")
test_path = "bazel-bin/install/" + "/".join(split_test_path[4:])
installed_tests.append(test_path)
installed_test_list_file = None
if len(installed_tests) > 0:
installed_test_list_file = ctx.actions.declare_file("install_deps/" + install_dir + "_test_list.txt")
ctx.actions.write(
output = installed_test_list_file,
content = "\n".join(installed_tests),
)
input_deps.append(installed_test_list_file)
# create a dep file for passing all the files we intend to install
# to the python script
name = ctx.label.package + "_" + install_dir
name = name.replace("/", "_")
deps_file = ctx.actions.declare_file("install_deps/" + name + "/" + install_dir)
# The roots are in the format { file : folder } so we can add arbitrary files to the install directory
roots = {} if installed_test_list_file == None else {installed_test_list_file.path: ""}
for file in root_files:
path = file_map["root_files"][file].short_path
folder_index_start = path.find(install_dir) + len(install_dir) + 1
folder_index_end = path.rfind("/")
roots[file] = path[folder_index_start:folder_index_end]
json_out = struct(
roots = roots,
bins = bins,
libs = libs,
)
ctx.actions.write(
output = deps_file,
content = json_out.to_json(),
)
# create a mapping of source location to install location
pkg_dict = {}
flat_map = {}
for file_type in file_map:
flat_map |= file_map[file_type]
for file in bins:
pkg_dict["bin/" + flat_map[file].basename] = flat_map[file]
outputs.append(flat_map[file])
for file in libs:
pkg_dict["lib/" + flat_map[file].basename] = flat_map[file]
outputs.append(flat_map[file])
for root_file in root_files:
pkg_dict[flat_map[root_file].basename] = flat_map[root_file]
outputs.append(flat_map[root_file])
if len(installed_tests) > 0:
real_test_list_output_location = ctx.actions.declare_file(install_dir + "/" + installed_test_list_file.basename)
pkg_dict[real_test_list_output_location.basename] = real_test_list_output_location
outputs.append(real_test_list_output_location)
# resolve full install dir for python script input
full_install_dir = ctx.bin_dir.path
if ctx.label.package:
full_install_dir += "/" + ctx.label.package
full_install_dir += "/" + install_dir
input_deps.append(deps_file)
inputs = depset(direct = input_deps, transitive = [
ctx.attr._install_script.files,
python.files,
] + [f.files for f in ctx.attr.srcs] + [r.files for r in ctx.attr.root_files.keys()] + [dep[MongoInstallInfo].deps_files for dep in ctx.attr.deps] + [dep[DefaultInfo].files for dep in ctx.attr.deps] + [depset(dwps)])
if outputs:
ctx.actions.run(
executable = python.interpreter.path,
outputs = outputs,
inputs = inputs,
arguments = [
ctx.attr._install_script.files.to_list()[0].path,
"--depfile=" + deps_file.path,
"--install-dir=" + full_install_dir,
] + ["--depfile=" + str(dep[MongoInstallInfo].deps_files.to_list()[0].path) for dep in ctx.attr.deps],
mnemonic = "MongoInstallRule",
execution_requirements = {
"no-cache": "1",
"no-sandbox": "1",
"no-remote": "1",
"local": "1",
},
)
runfiles = ctx.runfiles(files = outputs)
for input_bin in ctx.attr.srcs:
runfiles = runfiles.merge(input_bin[DefaultInfo].data_runfiles)
if unittest_bin:
outputs = depset([unittest_bin])
else:
outputs = depset(outputs)
return [
DefaultInfo(
files = outputs,
executable = unittest_bin,
runfiles = runfiles,
),
PackageFilesInfo(
dest_src_map = pkg_dict,
),
MongoInstallInfo(
deps_files = depset([deps_file], transitive = [dep[MongoInstallInfo].deps_files for dep in ctx.attr.deps]),
test_file = installed_test_list_file,
src_map = depset([json_out.to_json()]),
),
]
mongo_install_rule = rule(
mongo_install_rule_impl,
attrs = {
"srcs": attr.label_list(aspects = [test_binary_aspect]),
"deps": attr.label_list(providers = [PackageFilesInfo], aspects = [test_binary_aspect]),
"debug": attr.string(),
"root_files": attr.label_keyed_string_dict(allow_files = True),
"publish_debug_in_stripped": attr.bool(),
"create_dwp": attr.bool(),
"_install_script": attr.label(allow_single_file = True, default = "//bazel/install_rules:install_rules.py"),
"_linux_constraint": attr.label(default = "@platforms//os:linux"),
"_macos_constraint": attr.label(default = "@platforms//os:macos"),
"_windows_constraint": attr.label(default = "@platforms//os:windows"),
},
doc = "Install targets",
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
)
def mongo_install(
name,
srcs,
deps = [],
root_files = {},
target_compatible_with = [],
testonly = False,
pretty_printer_tests = {},
archive_license_files = ["//:archive_license_files"],
package_extract_name = "dist-test",
publish_debug_in_stripped = False,
**kwargs):
"""Perform install actions
Args:
name: standard target name
srcs: list of targets that should be installed
deps: other install rule targets that should be installed
**kwargs: other args to pass to underlying rules
target_compatible_with: forward target_compatible_with args to the rules
"""
compressor = select({
"@pigz//:pigz_tool_available": "@pigz//:bin",
"//conditions:default": None,
})
# this macro create several install targets for each instance of an install:
# "": normal install includes bins and debug info
# stripped: only install bins, only available with separate_debug=True
# debug: only install debug, only available with separate_debug=True
for install_type in ["", "-stripped", "-debug"]:
modified_srcs = srcs
install_target = "install-" + name + install_type
debug = ""
if install_type:
debug = install_type[1:]
# The macro names are base names, where install and archive are prefixed.
# this means for deps of install rule types we need to append a prefix
# and this means that the deps feild is restricted from selects.
#
# if a select is needed it can be applied at the srcs level
#
# however we add a special case this for enteprise install packages
dep_targets = []
community_dep_targets = []
for dep in deps:
if ":" in dep:
dep_basename = dep.split(":")[1]
dep_package = dep.split(":")[0]
if "modules/enterprise" not in native.package_name() and "modules/enterprise" not in dep_package:
community_dep_targets.append(dep_package + ":install-" + dep_basename)
dep_targets.append(dep_package + ":install-" + dep_basename)
else:
if "modules/enterprise" not in native.package_name():
community_dep_targets.append("install-" + dep)
dep_targets.append("install-" + dep)
# separate debug is required to make stripped or debug packages
seperate_debug_incompat = []
# TODO(SERVER-102851): This is commented out because CI AUBSAN builds currently
# try to build stripped/debug but don't separate them, that should be fixed
#if install_type:
# seperate_debug_incompat = ["@platforms//:incompatible"]
if len(pretty_printer_tests) > 0 and install_type != "-debug":
for test_script, test_binary in pretty_printer_tests.items():
pretty_printer_name = install_target + "-" + test_script.split(":")[-1].split(".")[0]
mongo_pretty_printer_test(
name = "real_" + pretty_printer_name,
test_script = test_script,
test_binary = test_binary,
testonly = True,
)
# This is a hacky way to not produce the pretty printer test files on windows and
# mac - you can't use target_compatible_with because it will skip downstream rules,
# and in the downstream rules you can't use select or you have nested selects
empty_rule(
name = "fake_" + pretty_printer_name,
)
native.alias(
name = pretty_printer_name,
actual = select({
"@platforms//os:linux": "real_" + pretty_printer_name,
"//conditions:default": "fake_" + pretty_printer_name,
}),
)
modified_srcs = modified_srcs + [pretty_printer_name]
testonly = True
mongo_install_rule(
name = install_target,
srcs = modified_srcs,
root_files = root_files,
debug = debug,
create_dwp = select({
"//bazel/config:dwp_supported": True,
"//bazel/config:create_dwp_disabled": False,
}, no_match_error = DWP_ERROR_MESSAGE),
deps = select({
"//bazel/config:build_enterprise_enabled": dep_targets,
"//conditions:default": community_dep_targets,
}),
target_compatible_with = target_compatible_with + select({
"//bazel/config:separate_debug_enabled": [],
"//conditions:default": seperate_debug_incompat,
}),
publish_debug_in_stripped = publish_debug_in_stripped,
testonly = testonly,
**kwargs
)
# This is so the README files dont end up looking like executables
pkg_files(
name = install_target + "_licenses",
srcs = archive_license_files,
attributes = pkg_attributes(mode = "644"),
)
pkg_files(
name = install_target + "_files",
srcs = [install_target],
attributes = pkg_attributes(mode = "755"),
strip_prefix = install_target,
testonly = testonly,
)
# package up the the install into an archive.
pkg_tar(
name = "archive-" + name + install_type + "_tar",
srcs = [install_target + "_files", install_target + "_licenses"],
compressor = compressor,
package_dir = package_extract_name,
package_file_name = name + install_type + ".tgz",
extension = "tgz",
exec_properties = {
"no-cache": "1",
"no-sandbox": "1",
"no-remote": "1",
"local": "1",
},
testonly = testonly,
target_compatible_with = select({
"@platforms//os:windows": ["@platforms//:incompatible"],
"//conditions:default": [],
}),
**kwargs
)
pkg_zip(
name = "archive-" + name + install_type + "_zip",
srcs = [install_target + "_files", install_target + "_licenses"],
package_dir = package_extract_name,
package_file_name = name + install_type + ".zip",
exec_properties = {
"no-cache": "1",
"no-sandbox": "1",
"no-remote": "1",
"local": "1",
},
testonly = testonly,
target_compatible_with = select({
"@platforms//os:windows": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
**kwargs
)
# Used to run zip on windows and tar on every other os
native.alias(
name = "archive-" + name + install_type,
actual = select({
"@platforms//os:windows": "archive-" + name + install_type + "_zip",
"//conditions:default": "archive-" + name + install_type + "_tar",
}),
)

108
bazel/install_rules/install_rules.py Normal file
View File

@ -0,0 +1,108 @@
import argparse
import json
import os
import shutil
parser = argparse.ArgumentParser()
parser.add_argument("--depfile", action="append")
parser.add_argument("--install-dir")
parser.add_argument("--install-mode", choices=["copy", "symlink", "hardlink"], default="hardlink")
args = parser.parse_args()
if os.path.exists(args.install_dir):
os.chmod(args.install_dir, 0o755)
for root, dirs, files in os.walk(args.install_dir):
for name in files:
try:
os.chmod(os.path.join(root, name), 0o755)
os.unlink(os.path.join(root, name))
# Sometimes we find files that don't exist
# from os.walk - not sure why
except FileNotFoundError:
continue
for name in dirs:
try:
os.chmod(os.path.join(root, name), 0o755)
except FileNotFoundError:
continue
shutil.rmtree(args.install_dir)
os.makedirs(args.install_dir, exist_ok=True)
install_link = args.install_dir + "/../install"
os.makedirs(install_link, exist_ok=True)
def install(src, install_type):
install_dst = os.path.join(args.install_dir, install_type, os.path.basename(src))
link_dst = os.path.join(install_link, install_type, os.path.basename(src))
if src.endswith(".dwp"):
# Due to us creating our binaries using the _with_debug name
# the dwp files also contain it. Strip the _with_debug from the name
install_dst = install_dst.replace("_with_debug.dwp", ".dwp")
link_dst = link_dst.replace("_with_debug.dwp", ".dwp")
for dst in [install_dst, link_dst]:
os.makedirs(os.path.dirname(dst), exist_ok=True)
if args.install_mode == "hardlink":
if os.path.exists(dst) and not os.path.samefile(src, dst):
try:
if os.path.isdir(dst):
shutil.rmtree(dst)
else:
os.chmod(dst, 0o755)
os.unlink(dst)
except FileNotFoundError as exc:
if link_dst == dst:
# if multiple installs are requested at once and happen
# to install the same file, it is ambiguous
# when one should be linked into the general install dir
# so we pass on exceptions
pass
else:
raise exc
if not os.path.exists(dst):
try:
if os.path.isdir(src):
for root, _, files in os.walk(src):
for name in files:
dest_dir = os.path.dirname(os.path.join(root, name)).replace(
src, dst
)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
os.link(os.path.join(root, name), os.path.join(dest_dir, name))
else:
try:
os.link(src, dst)
# If you try hardlinking across drives link will fail
except OSError:
try:
shutil.copy(src, dst)
except shutil.SameFileError:
pass
except FileExistsError as exc:
if link_dst == dst:
# if multiple installs are requested at once and happen
# to install the same file, it is ambiguous
# when one should be linked into the general install dir
# so we pass on exceptions
pass
else:
raise exc
else:
raise Exception("Only hardlink mode is currently implemented.")
return install_dst
for depfile in args.depfile:
with open(depfile) as f:
content = json.load(f)
for binary in content["bins"]:
install(binary, "bin")
for lib in content["libs"]:
install(lib, "lib")
for file, folder in content["roots"].items():
install(file, folder)

169
bazel/install_rules/msi.bzl Normal file
View File

@ -0,0 +1,169 @@
# Build an msi using the wix toolset.
# Building the msi involves running candle.exe -> light.exe -> msitrim.py
def mongo_msi_impl(ctx):
candle_in = []
candle_out = []
candle_arguments = [ctx.attr._candle_wrapper_script.files.to_list()[0].path, ctx.executable._candle.path, "-wx"]
# pass in version variables
# eg. 8.1.0-alpha
mongo_version = ctx.attr.mongo_version
if mongo_version in ctx.var:
mongo_version = ctx.var[mongo_version]
# eg. 8.1
mongo_major_version = ".".join(mongo_version.split(".")[:2])
candle_arguments.append("-dMongoDBMajorVersion=" + mongo_major_version)
# eg. 8.1.0
mongo_no_revision_version = mongo_version.split("-")[0]
candle_arguments.append("-dMongoDBVersion=" + mongo_no_revision_version)
# pass in folder variables needed by wix
for var, label in ctx.attr.deps.items():
folder = ""
for file in label.files.to_list():
symlink = ctx.actions.declare_file(var + "/" + file.basename)
ctx.actions.symlink(output = symlink, target_file = file)
candle_in.append(symlink)
folder = symlink.dirname
candle_arguments.append("-d" + var + "=" + folder + "/")
# pass in string variables needed by wix
for var, value in ctx.attr.wix_vars.items():
candle_arguments.append("-d" + var + "=" + value)
# pass in custom action
for file in ctx.attr.custom_action.files.to_list():
if file.extension == "dll":
candle_in.append(file)
candle_arguments.append("-dCustomActionDll=" + file.path)
# pass in merge module if needed
if ctx.attr.use_merge_modules:
for file in ctx.attr._merge_modules.files.to_list():
if file.basename.find("CRT") > -1 and file.basename.find(ctx.attr.arch) > -1:
candle_in.append(file)
candle_arguments.append("-dMergeModulesBasePath=" + file.dirname)
candle_arguments.append("-dMergeModuleFileCRT=" + file.basename)
break
# pass in architecture
candle_arguments.append("-dPlatform=" + ctx.attr.arch)
candle_arguments.append("-arch")
candle_arguments.append(ctx.attr.arch)
# pass in extension files needed by wix
for extension in ctx.attr.extensions:
candle_arguments.append("-ext")
candle_arguments.append(extension)
# pass in .wxs files
output_directory = ""
for label in ctx.attr.srcs:
for file in label.files.to_list():
candle_in.append(file)
candle_arguments.append(file.path)
# wix output files are the input files with wixobj extension instead
output_file = ctx.actions.declare_file(file.basename.split(".")[0] + ".wixobj")
candle_out.append(output_file)
output_directory = output_file.dirname
candle_arguments.append("-dOutDir=" + output_directory)
candle_arguments.append("-dTargetDir=" + output_directory)
candle_arguments.append("-out")
candle_arguments.append(output_directory + "/")
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
ctx.actions.run(
outputs = candle_out,
inputs = depset(transitive = [depset(candle_in), ctx.attr._candle_wrapper_script.files, python.files, ctx.attr._candle.files]),
executable = python.interpreter.path,
arguments = candle_arguments,
)
light_in = candle_out
light_out = []
light_arguments = ["-wx", "-cultures:null"]
# pass in extension files needed by wix
for extension in ctx.attr.extensions:
light_arguments.append("-ext")
light_arguments.append(extension)
# internal consistency evaluators to skip
for sice in ctx.attr.light_sice:
light_arguments.append("-sice:" + sice)
for file in candle_out:
light_arguments.append(file.path)
output_filename = ctx.label.name + "-" + mongo_version
light_msi = ctx.actions.declare_file(output_filename + ".pre.msi")
light_out.append(light_msi)
light_arguments.append("-out")
light_arguments.append(light_msi.path)
ctx.actions.run(
outputs = light_out,
inputs = light_in,
executable = ctx.executable._light,
arguments = light_arguments,
)
output_msi = ctx.actions.declare_file(output_filename + ".msi")
msi_trim_script = ctx.attr._msi_trim_script.files.to_list()[0].path
ctx.actions.run(
outputs = [output_msi],
inputs = depset(transitive = [depset(light_out), ctx.attr._msi_trim_script.files, python.files]),
executable = python.interpreter.path,
arguments = [msi_trim_script, light_msi.path, output_msi.path],
)
return [DefaultInfo(
files = depset([output_msi]),
)]
mongo_msi = rule(
mongo_msi_impl,
attrs = {
"srcs": attr.label_list(allow_files = [".wxs"]),
"deps": attr.string_keyed_label_dict(allow_empty = True),
"custom_action": attr.label(allow_files = [".dll"]),
"extensions": attr.string_list(allow_empty = True),
"mongo_version": attr.string(mandatory = True),
"use_merge_modules": attr.bool(default = False),
"wix_vars": attr.string_dict(allow_empty = True),
"light_sice": attr.string_list(allow_empty = True),
"arch": attr.string(default = "x64"),
"_candle": attr.label(
default = "@wix_toolset//:candle",
allow_single_file = True,
executable = True,
cfg = "host",
),
"_candle_wrapper_script": attr.label(
doc = "The python msi trimming script to use.",
default = "//buildscripts:candle_wrapper.py",
allow_single_file = True,
),
"_light": attr.label(
default = "@wix_toolset//:light",
allow_single_file = True,
executable = True,
cfg = "host",
),
"_msi_trim_script": attr.label(
doc = "The python msi trimming script to use.",
default = "//buildscripts:msitrim.py",
allow_single_file = True,
),
"_merge_modules": attr.label(
default = "@local_windows_msvc//:merge_modules",
),
},
doc = "Create a msi using wix toolset",
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
)

52
bazel/install_rules/pigz.bzl Normal file
View File

@ -0,0 +1,52 @@
def _pigz(ctx):
pigz_bin = ctx.which("pigz")
if pigz_bin:
ctx.symlink(pigz_bin, "pigz")
ctx.file(
"BUILD.bazel",
"""
package(default_visibility = ["//visibility:public"])
config_setting(
name = "pigz_tool_available",
constraint_values = [
"@platforms//os:%s",
]
)
sh_binary(
name = "bin",
srcs = ["pigz"],
)
""" % (ctx.os.name),
)
else:
ctx.file(
"BUILD.bazel",
"""
package(default_visibility = ["//visibility:public"])
constraint_value(
name = "not_set",
constraint_setting = "@platforms//cpu",
)
config_setting(
name = "pigz_tool_available",
constraint_values = [
":not_set",
]
)
sh_binary(
name = "bin",
srcs = [],
)
""",
)
setup_pigz = repository_rule(
implementation = _pigz,
)

76
bazel/install_rules/pretty_printer_test_creator.py Normal file
View File

@ -0,0 +1,76 @@
import argparse
import re
parser = argparse.ArgumentParser()
parser.add_argument("--gdb-test-script")
parser.add_argument("--pip-requirements-script")
parser.add_argument("--pretty-printer-output")
parser.add_argument("--pretty-printer-launcher-infile")
parser.add_argument("--gdb-path")
# Because we 'install' these files to a different location then where they are created
# we want to pass the final location of these files rather than where they currently are
parser.add_argument("--final-binary-path")
parser.add_argument("--final-pretty-printer-path")
parser.add_argument("--pretty-printer-launcher-output")
extra_lines = [
"import os,subprocess,sys,traceback",
"cmd = 'python -c \"import os,sys;print(os.linesep.join(sys.path).strip())\"'",
"paths = subprocess.check_output(cmd,shell=True).decode('utf-8').split()",
"sys.path.extend(paths)",
"symbols_loaded = False",
"try:",
" if gdb.objfiles()[0].lookup_global_symbol('main') is not None:",
" symbols_loaded = True",
"except Exception:",
" pass",
"if not symbols_loaded:",
r" gdb.write('Could not find main symbol, debug info may not be loaded.\n')",
r" gdb.write('TEST FAILED -- No Symbols.\\\n')",
" gdb.execute('quit 1', to_string=True)",
"else:",
r" gdb.write('Symbols loaded.\n')",
"gdb.execute('set confirm off')",
"gdb.execute('source .gdbinit')",
"try:",
" verify_requirements(executable='python3')",
"except MissingRequirements as ex:",
" print(ex)",
" print('continuing testing anyways!')",
"except Exception as exc:",
" print('ERROR: failed while verifying requirements.')",
" traceback.print_exc()",
" sys.exit(1)",
]
args = parser.parse_args()
with open(args.pretty_printer_output, "w") as pretty_printer_output:
with open(args.pip_requirements_script) as pip_requirements_script:
pretty_printer_output.write(pip_requirements_script.read())
pretty_printer_output.write("\n")
pretty_printer_output.write("\n".join(extra_lines))
pretty_printer_output.write("\n")
with open(args.gdb_test_script) as test_script:
pretty_printer_output.write(test_script.read())
# Verbose and test_args are hardcoded because there are no users of those overrides currently
replacements = {
"@VERBOSE@": "True",
"@pretty_printer_test_py@": args.final_pretty_printer_path,
"@gdb_path@": args.gdb_path,
"@pretty_printer_test_program@": args.final_binary_path,
"@test_args@": '[""]',
}
escaped_replacements = dict((re.escape(k), v) for k, v in replacements.items())
pattern = re.compile("|".join(escaped_replacements.keys()))
with open(args.pretty_printer_launcher_output, "w") as pretty_printer_launcher_output:
with open(args.pretty_printer_launcher_infile) as pretty_printer_launcher_infile:
replaced_text = pattern.sub(
lambda match: escaped_replacements[re.escape(match.group(0))],
pretty_printer_launcher_infile.read(),
)
pretty_printer_launcher_output.write(replaced_text)

77
bazel/install_rules/pretty_printer_tests.bzl Normal file
View File

@ -0,0 +1,77 @@
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
load("//bazel/install_rules:providers.bzl", "TestBinaryInfo")
# This will not currently work under bazel test/run until we have a version of gdb to use in bazel
def mongo_pretty_printer_test_impl(ctx):
split_name = ctx.label.name.split("-")
# eg optimizer_gdb_test
short_name = split_name[-1]
final_output_directory = "bazel-bin/install/bin/"
runnable_binary = None
for file in ctx.attr.test_binary.files.to_list():
if file.extension == "" or file.extension == "exe":
runnable_binary = file
pretty_printer_directory = "pretty-printer"
if "stripped" in split_name:
pretty_printer_directory += "-stripped"
pretty_printer_directory += "/"
script_output = ctx.actions.declare_file(pretty_printer_directory + short_name + ".py")
launcher_output = ctx.actions.declare_file(pretty_printer_directory + "pretty_printer_test_launcher_" + short_name + ".py")
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
inputs = depset(transitive = [
ctx.attr._pretty_printer_creation_script.files,
ctx.attr.test_script.files,
ctx.attr._pip_requirements_script.files,
ctx.attr._pretty_printer_launcher_infile.files,
python.files,
])
outputs = [launcher_output, script_output]
ctx.actions.run(
executable = python.interpreter.path,
outputs = outputs,
inputs = inputs,
arguments = [
ctx.file._pretty_printer_creation_script.path,
"--gdb-test-script=" + ctx.file.test_script.path,
"--pip-requirements-script=" + ctx.file._pip_requirements_script.path,
"--pretty-printer-output=" + script_output.path,
"--pretty-printer-launcher-infile=" + ctx.file._pretty_printer_launcher_infile.path,
# TODO have a way to get to gdb from inside bazel
"--gdb-path=" + "/opt/mongodbtoolchain/v4/bin/gdb",
# This is due to us being dependent on the final location of installed binaries - ideally we don't do this and run the tests
# in place and not from another directory
"--final-binary-path=" + final_output_directory + runnable_binary.basename,
"--final-pretty-printer-path=" + final_output_directory + script_output.basename,
"--pretty-printer-launcher-output=" + launcher_output.path,
],
mnemonic = "MongoPrettyPrinterTestCreation",
)
default_provider = DefaultInfo(executable = launcher_output, files = depset(outputs))
test_binary_provider = TestBinaryInfo(test_binaries = depset([launcher_output]))
return [default_provider, test_binary_provider]
mongo_pretty_printer_test = rule(
mongo_pretty_printer_test_impl,
attrs = {
"test_script": attr.label(allow_single_file = True),
"test_binary": attr.label(),
# TODO have a way to get to gdb from inside bazel
#"_gdb": attr.label(allow_single_file = True, default = "//:gdb"),
"_pretty_printer_creation_script": attr.label(allow_single_file = True, default = "//bazel/install_rules:pretty_printer_test_creator.py"),
"_pip_requirements_script": attr.label(allow_single_file = True, default = "//buildscripts:pip_requirements.py"),
"_pretty_printer_launcher_infile": attr.label(allow_single_file = True, default = "//src/mongo/util:pretty_printer_test_launcher.py.in"),
},
doc = "Create pretty printer tests",
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
executable = True,
test = True,
)

6
bazel/install_rules/providers.bzl Normal file
View File

@ -0,0 +1,6 @@
TestBinaryInfo = provider(
doc = "Test binaries returned by this target.",
fields = {
"test_binaries": "group of all test binaries",
},
)

49
bazel/install_rules/windows_msvc.bzl Normal file
View File

@ -0,0 +1,49 @@
load(
"//bazel/toolchains/cc/mongo_windows:windows_cc_configure.bzl",
"find_vc_path",
"get_vc_redist_version",
)
def find_windows_msvc(ctx):
vc_path = find_vc_path(ctx)
if vc_path == None:
fail("Failed to locate Visual Studio. Make sure you are on Windows and have Visual Studio installed.")
redist_version = get_vc_redist_version(ctx)
if redist_version == None:
fail("Failed to locate a redistribution version from Visual Studio")
ctx.symlink(vc_path + "/Redist/MSVC/" + redist_version, "msvc")
ctx.file(
"BUILD.bazel",
"""
package(default_visibility = ["//visibility:public"])
filegroup(
name = "merge_modules",
srcs = select({
"@platforms//os:windows": glob(["**/*.msm"]),
"//conditions:default": [],
}),
)
filegroup(
name = "vc_redist_x64",
srcs = select({
"@platforms//os:windows": glob(["**/vc_redist.x64.exe"]),
"//conditions:default": [],
}),
)
""",
)
return None
windows_msvc = repository_rule(
environ = [
"BAZEL_VC_FULL_VERSION", # Force re-compute if the user changed the version of MS compiler.
"MONGO_VC_REDIST_FULL_VERSION", # Force re-compute if the user changed the VC Redistribution version.
],
implementation = find_windows_msvc,
configure = True,
local = True,
)

15
bazel/lint.sh Executable file
View File

@ -0,0 +1,15 @@
# placeholder for bazel/wrapper_hook/lint.py
RED='\033[0;31m'
GREEN='\033[0;32m'
NO_COLOR='\033[0m'
if [[ $1 == "ALL_PASSING" ]]; then
echo -e "${GREEN}INFO:${NO_COLOR} No linter errors found!"
exit 0
fi
echo -e "${RED}ERROR:${NO_COLOR} Linter run failed, see details above"
echo -e "${GREEN}INFO:${NO_COLOR} Run the following to try to auto-fix the errors:\n\nbazel run lint --fix"
exit 1

12
bazel/mongo_js_rules.bzl Normal file
View File

@ -0,0 +1,12 @@
"""Macros wrapping rules_js's javascript rules."""
load("@aspect_rules_js//js:defs.bzl", "js_library")
def mongo_js_library(*args, **kwargs):
if "target_compatible_with" not in kwargs:
kwargs["target_compatible_with"] = []
kwargs["target_compatible_with"] += select({
"//bazel/config:ppc_or_s390x": ["@platforms//:incompatible"],
"//conditions:default": [],
})
js_library(*args, **kwargs)

88
bazel/mongo_script_rules.bzl Normal file
View File

@ -0,0 +1,88 @@
"""Common mongo-specific bazel build rules intended to be used for buildscripts.
"""
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
MONGO_TOOLCHAIN_V4_PATH = "external/mongo_toolchain_v4/v4"
def _py_cxx_wrapper(*, python_path, toolchain_path, python_interpreter, main_py):
return "\n".join([
"export PYTHONPATH={}".format(python_path),
"export MONGO_TOOLCHAIN_PATH={}".format(toolchain_path),
"{} {}".format(python_interpreter, main_py),
])
def _py_cxx_test_impl(ctx):
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
python_path = []
for dep in ctx.attr.deps:
for path in dep[PyInfo].imports.to_list():
if path not in python_path:
python_path.append(
ctx.expand_make_variables(
"python_library_imports",
"$${RUNFILES_DIR}/" + path,
ctx.var,
),
)
python_path_str = ctx.configuration.host_path_separator.join(python_path)
cc_toolchain = find_cpp_toolchain(ctx)
runfiles = ctx.runfiles(
files = (
ctx.files.srcs +
ctx.files.data +
ctx.files.deps +
ctx.files.main +
cc_toolchain.all_files.to_list()
),
)
transitive_runfiles = []
for runfiles_attr in (
[ctx.attr.main],
ctx.attr.srcs,
ctx.attr.deps,
ctx.attr.data,
):
for target in runfiles_attr:
transitive_runfiles.append(target[DefaultInfo].default_runfiles)
runfiles = runfiles.merge_all(transitive_runfiles)
main_py = ctx.attr.main.files.to_list()[0].path
script = _py_cxx_wrapper(
python_path = python_path_str,
toolchain_path = ctx.attr.toolchain_path,
python_interpreter = python.interpreter.path,
main_py = main_py,
)
ctx.actions.write(
output = ctx.outputs.executable,
content = script,
)
return DefaultInfo(files = depset([ctx.outputs.executable]), runfiles = runfiles)
py_cxx_test = rule(
implementation = _py_cxx_test_impl,
attrs = {
"main": attr.label(allow_single_file = True, mandatory = True),
"srcs": attr.label_list(allow_files = [".py"]),
"deps": attr.label_list(),
"data": attr.label_list(),
"toolchain_path": attr.string(mandatory = True),
},
toolchains = ["@bazel_tools//tools/cpp:toolchain_type", "@bazel_tools//tools/python:toolchain_type"],
executable = True,
test = True,
)
def mongo_toolchain_py_cxx_test(**kwargs):
py_cxx_test(
toolchain_path = select({
"//bazel/config:mongo_toolchain_v4": MONGO_TOOLCHAIN_V4_PATH,
"//conditions:default": MONGO_TOOLCHAIN_V4_PATH,
}),
target_compatible_with = ["@//bazel/platforms:use_mongo_toolchain"],
**kwargs
)

2317
bazel/mongo_src_rules.bzl
View File

File diff suppressed because it is too large Load Diff

127
bazel/platforms/BUILD.bazel
View File

@ -1,8 +1,15 @@
# Define supported build platforms.
load("//bazel/platforms:remote_execution_containers.bzl", "REMOTE_EXECUTION_CONTAINERS")
load("//bazel/platforms:platform_util.bzl", "setup_platform")
package(default_visibility = ["//visibility:public"])
constraint_setting(name = "mongo_toolchain")
constraint_value(
name = "use_mongo_toolchain",
constraint_setting = ":mongo_toolchain",
)
constraint_setting(name = "tcmalloc_kernel_version")
constraint_value(
@ -40,27 +47,7 @@ constraint_setting(name = "distro")
]
[
platform(
name = distro_or_os + "_" + arch + "_" + compiler,
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:arm64" if arch == "arm64" else "@platforms//cpu:x86_64",
"@bazel_tools//tools/cpp:" + compiler,
":" + distro_or_os,
":kernel_version_4_4_or_greater", # remote execution platforms assume a given kernel version
],
exec_properties = {
"container-image": REMOTE_EXECUTION_CONTAINERS[distro_or_os]["container-url"],
"dockerNetwork": "standard",
# EngFlow's "default" pool is ARM64
"Pool": "x86_64" if arch == "amd64" else "default",
},
)
for compiler in [
"clang",
"gcc",
]
setup_platform(arch, distro_or_os, cache_silo)
for arch in [
"arm64",
"amd64",
@ -79,103 +66,101 @@ constraint_setting(name = "distro")
"rhel9",
"suse15",
]
for cache_silo in [
"",
"_cache_silo",
]
]
constraint_setting(name = "mongo_windows_toolchain_config")
constraint_value(
name = "use_mongo_windows_toolchain_config",
constraint_setting = ":mongo_windows_toolchain_config",
)
platform(
name = "windows_amd64_msvc",
name = "windows_amd64",
constraint_values = [
"@platforms//cpu:x86_64",
"@platforms//os:windows",
"@bazel_tools//tools/cpp:msvc",
],
)
platform(
name = "macos_arm64_clang",
name = "macos_arm64",
constraint_values = [
"@platforms//cpu:arm64",
"@platforms//os:macos",
"@bazel_tools//tools/cpp:clang",
],
)
platform(
name = "macos_amd64_clang",
name = "macos_amd64",
constraint_values = [
"@platforms//cpu:x86_64",
"@platforms//os:macos",
"@bazel_tools//tools/cpp:clang",
],
)
platform(
name = "linux_ppc64le_clang",
name = "linux_ppc64le",
constraint_values = [
"@platforms//cpu:ppc64le",
"@platforms//cpu:ppc",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:clang",
],
)
platform(
name = "linux_ppc64le_gcc",
constraint_values = [
"@platforms//cpu:ppc64le",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:gcc",
],
)
platform(
name = "linux_s390x_clang",
name = "linux_s390x",
constraint_values = [
"@platforms//cpu:s390x",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:clang",
],
)
platform(
name = "linux_s390x_gcc",
name = "rhel8_ppc64le",
constraint_values = [
"@platforms//cpu:ppc",
"@platforms//os:linux",
":use_mongo_toolchain",
":rhel8",
],
)
platform(
name = "rhel9_ppc64le",
constraint_values = [
"@platforms//cpu:ppc",
"@platforms//os:linux",
":use_mongo_toolchain",
":rhel9",
],
)
platform(
name = "rhel8_s390x",
constraint_values = [
"@platforms//cpu:s390x",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:gcc",
":use_mongo_toolchain",
":rhel8",
],
)
platform(
name = "rhel8_ppc64le_clang",
constraint_values = [
"@platforms//cpu:ppc64le",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:clang",
],
)
platform(
name = "rhel8_ppc64le_gcc",
constraint_values = [
"@platforms//cpu:ppc64le",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:gcc",
],
)
platform(
name = "rhel8_s390x_clang",
name = "rhel9_s390x",
constraint_values = [
"@platforms//cpu:s390x",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:clang",
":use_mongo_toolchain",
":rhel9",
],
)
platform(
name = "rhel8_s390x_gcc",
constraint_values = [
"@platforms//cpu:s390x",
"@platforms//os:linux",
"@bazel_tools//tools/cpp:gcc",
],
py_binary(
name = "remote_execution_containers_generator",
srcs = ["remote_execution_containers_generator.py"],
visibility = ["//visibility:public"],
)

96
bazel/platforms/local_config_platform.bzl
View File

@ -1,20 +1,8 @@
load("//bazel/platforms:remote_execution_containers.bzl", "REMOTE_EXECUTION_CONTAINERS")
load("//bazel/platforms:normalize.bzl", "ARCH_TO_PLATFORM_MAP", "OS_TO_PLATFORM_MAP")
load("//bazel/toolchains/cc/mongo_linux:mongo_toolchain_version.bzl", "TOOLCHAIN_MAP")
load("//bazel:utils.bzl", "get_host_distro_major_version")
_OS_MAP = {
"macos": "@platforms//os:osx",
"linux": "@platforms//os:linux",
"windows": "@platforms//os:windows",
}
_ARCH_MAP = {
"amd64": "@platforms//cpu:x86_64",
"aarch64": "@platforms//cpu:arm64",
"x86_64": "@platforms//cpu:x86_64",
"ppc64le": "@platforms//cpu:ppc64le",
"s390x": "@platforms//cpu:s390x",
}
def _setup_local_config_platform(ctx):
"""
Generates our own local_config_platform, overriding bazel's built in generation.
@ -32,8 +20,8 @@ def _setup_local_config_platform(ctx):
arch = ctx.os.arch
os_constraint = _OS_MAP[os]
arch_constraint = _ARCH_MAP[arch]
os_constraint = OS_TO_PLATFORM_MAP[os]
arch_constraint = ARCH_TO_PLATFORM_MAP[arch]
constraints = [os_constraint, arch_constraint]
@ -46,10 +34,41 @@ def _setup_local_config_platform(ctx):
elif arch == "aarch64":
arch = "arm64"
kernel_constraints = ""
if os == "linux":
result = ctx.execute([
"uname",
"-r",
])
version_numbers = result.stdout.split(".")
if int(version_numbers[0]) > 4 or (int(version_numbers[0]) == 4 and int(version_numbers[1]) > 3):
kernel_constraints = ',\n "@//bazel/platforms:kernel_version_4_4_or_greater"'
else:
kernel_constraints = ',\n "@//bazel/platforms:kernel_version_less_than_4_4"'
# EngFlow's "default" pool is ARM64
remote_execution_pool = "x86_64" if arch == "amd64" else "default"
result = None
toolchain_key = "{distro}_{arch}".format(distro = distro, arch = arch)
print("Trying to find toolchain for {}".format(toolchain_key))
toolchain_exists = False
for version in TOOLCHAIN_MAP:
if toolchain_key in TOOLCHAIN_MAP[version]:
toolchain_exists = True
break
if distro != None and distro in REMOTE_EXECUTION_CONTAINERS:
cache_silo = '"cache-silo-key": "' + toolchain_key + '",' if ctx.os.environ.get("evergreen_remote_exec") == "off" else ""
if ctx.os.environ.get("USE_NATIVE_TOOLCHAIN"):
exec_props = ""
result = {"USE_NATIVE_TOOLCHAIN": "1"}
constraints_str += kernel_constraints
elif distro != None and distro in REMOTE_EXECUTION_CONTAINERS:
constraints_str += ',\n "@//bazel/platforms:use_mongo_toolchain"'
constraints_str += ',\n "@//bazel/platforms:%s"' % (distro)
# remote execution supported platforms assume valid tcmalloc kernel version even if running locally.
# if running a local build, other flags will auto select the correct tcmalloc.
constraints_str += ',\n "@//bazel/platforms:kernel_version_4_4_or_greater"'
container_url = REMOTE_EXECUTION_CONTAINERS[distro]["container-url"]
web_url = REMOTE_EXECUTION_CONTAINERS[distro]["web-url"]
dockerfile = REMOTE_EXECUTION_CONTAINERS[distro]["dockerfile"]
@ -59,20 +78,26 @@ def _setup_local_config_platform(ctx):
"container-image": "%s",
"dockerNetwork": "standard",
"Pool": "%s",
%s
},
""" % (container_url, remote_execution_pool)
""" % (container_url, remote_execution_pool, cache_silo)
result = {"DISTRO": distro}
elif distro != None and toolchain_exists:
constraints_str += ',\n "@//bazel/platforms:use_mongo_toolchain"'
constraints_str += ',\n "@//bazel/platforms:%s"' % (distro)
result = {"DISTRO": distro}
exec_props = ""
constraints_str += kernel_constraints
elif os == "windows":
constraints_str += ',\n "@//bazel/platforms:use_mongo_windows_toolchain_config"'
result = {"USE_NATIVE_TOOLCHAIN": "1"}
exec_props = ""
else:
constraints_str += kernel_constraints
result = {"USE_NATIVE_TOOLCHAIN": "1"}
exec_props = ""
result = ctx.execute([
"uname",
"-r",
])
version_numbers = result.stdout.split(".")
if int(version_numbers[0]) > 4 or (int(version_numbers[0]) == 4 and int(version_numbers[1]) > 3):
platform_constraints_str = constraints_str + ',\n "@//bazel/platforms:kernel_version_4_4_or_greater"'
else:
platform_constraints_str = constraints_str + ',\n "@//bazel/platforms:kernel_version_less_than_4_4"'
platform_constraints_str = constraints_str
substitutions = {
"{constraints}": constraints_str,
@ -81,18 +106,24 @@ def _setup_local_config_platform(ctx):
}
ctx.template(
"BUILD.bazel",
"host/BUILD.bazel",
ctx.attr.build_tpl,
substitutions = substitutions,
)
ctx.template(
"constraints.bzl",
"host/constraints.bzl",
ctx.attr.constraints_tpl,
substitutions = substitutions,
)
return None
ctx.template(
"host/extension.bzl",
ctx.attr.extension_tpl,
substitutions = substitutions,
)
return result
setup_local_config_platform = repository_rule(
implementation = _setup_local_config_platform,
@ -105,5 +136,10 @@ setup_local_config_platform = repository_rule(
default = "//bazel/platforms:local_config_platform_constraints.bzl",
doc = "Template modeling the builtin local config platform constraints file.",
),
"extension_tpl": attr.label(
default = "//bazel/platforms:local_config_platform_extension.bzl",
doc = "Template modeling the builtin local config platform constraints file.",
),
},
environ = ["USE_NATIVE_TOOLCHAIN"],
)

66
bazel/platforms/local_config_platform_extension.bzl Normal file
View File

@ -0,0 +1,66 @@
def _translate_cpu(arch):
if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]:
return "x86_32"
if arch in ["amd64", "x86_64", "x64"]:
return "x86_64"
if arch in ["ppc", "ppc64", "ppc64le"]:
return "ppc"
if arch in ["arm", "armv7l"]:
return "arm"
if arch in ["aarch64"]:
return "aarch64"
if arch in ["s390x", "s390"]:
return "s390x"
if arch in ["mips64el", "mips64"]:
return "mips64"
if arch in ["riscv64"]:
return "riscv64"
return None
def _translate_os(os):
if os.startswith("mac os"):
return "osx"
if os.startswith("freebsd"):
return "freebsd"
if os.startswith("openbsd"):
return "openbsd"
if os.startswith("linux"):
return "linux"
if os.startswith("windows"):
return "windows"
return None
def _host_platform_repo_impl(rctx):
cpu = _translate_cpu(rctx.os.arch)
os = _translate_os(rctx.os.name)
cpu = "" if cpu == None else " '@platforms//cpu:%s',\n" % cpu
os = "" if os == None else " '@platforms//os:%s',\n" % os
rctx.file("BUILD.bazel", """
# DO NOT EDIT: automatically generated BUILD file
exports_files(["constraints.bzl"])
""")
rctx.file("constraints.bzl", """
# DO NOT EDIT: automatically generated constraints list
HOST_CONSTRAINTS = [
%s%s]
""" % (cpu, os))
host_platform_repo = repository_rule(
implementation = _host_platform_repo_impl,
doc = """Generates constraints for the host platform. The constraints.bzl
file contains a single <code>HOST_CONSTRAINTS</code> variable, which is a
list of strings, each of which is a label to a <code>constraint_value</code>
for the host platform.""",
)
def _host_platform_impl(_mctx):
host_platform_repo(name = "host_platform")
host_platform = module_extension(
implementation = _host_platform_impl,
doc = """Generates a <code>host_platform_repo</code> repo named
<code>host_platform</code>, containing constraints for the host platform.""",
)

38
bazel/platforms/local_host_values.bzl Normal file
View File

@ -0,0 +1,38 @@
def _local_host_values(ctx):
if "win" in ctx.os.name:
result = ctx.execute([
ctx.path(ctx.attr.python_interpreter_target_win),
"-c",
"import os; print(os.cpu_count())",
])
else:
result = ctx.execute([
ctx.path(ctx.attr.python_interpreter_target_default),
"-c",
"import os; print(os.cpu_count())",
])
ctx.file(
"BUILD.bazel",
"",
)
ctx.file(
"local_host_values_set.bzl",
"""
NUM_CPUS = %s
""" % (result.stdout),
)
setup_local_host_values = repository_rule(
implementation = _local_host_values,
attrs = {
"python_interpreter_target_default": attr.label(
default = "@py_host//:dist/bin/python3",
doc = "The target of the Python interpreter used during repository setup, if not windows",
),
"python_interpreter_target_win": attr.label(
default = "@py_host//:dist/python.exe",
doc = "The target of the Python interpreter used during repository setup for windows platforms",
),
},
)

23
bazel/platforms/normalize.bzl Normal file
View File

@ -0,0 +1,23 @@
ARCH_NORMALIZE_MAP = {
"amd64": "x86_64",
"x86_64": "x86_64",
"arm64": "aarch64",
"aarch64": "aarch64",
"ppc64le": "ppc64le",
"s390x": "s390x",
}
ARCH_TO_PLATFORM_MAP = {
"amd64": "@platforms//cpu:x86_64",
"x86_64": "@platforms//cpu:x86_64",
"arm64": "@platforms//cpu:arm64",
"aarch64": "@platforms//cpu:arm64",
"ppc64le": "@platforms//cpu:ppc",
"s390x": "@platforms//cpu:s390x",
}
OS_TO_PLATFORM_MAP = {
"macos": "@platforms//os:osx",
"linux": "@platforms//os:linux",
"windows": "@platforms//os:windows",
}

29
bazel/platforms/platform_util.bzl Normal file
View File

@ -0,0 +1,29 @@
load("//bazel/platforms:remote_execution_containers.bzl", "REMOTE_EXECUTION_CONTAINERS")
def setup_platform(arch, distro_or_os, cache_silo):
exec_properties = {
"container-image": REMOTE_EXECUTION_CONTAINERS[distro_or_os]["container-url"],
"dockerNetwork": "standard",
# EngFlow's "default" pool is ARM64
"Pool": "x86_64" if arch == "amd64" else "default",
}
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:arm64" if arch == "arm64" else "@platforms//cpu:x86_64",
":" + distro_or_os,
":use_mongo_toolchain",
]
if cache_silo:
exec_properties.update({"cache-silo-key": distro_or_os + "_" + arch})
else:
# remote execution platforms assume a given kernel version
constraint_values.append(":kernel_version_4_4_or_greater")
native.platform(
name = distro_or_os + "_" + arch + cache_silo,
constraint_values = constraint_values,
exec_properties = exec_properties,
)

72
bazel/platforms/remote_execution_containers.bzl
View File

@ -2,64 +2,64 @@
REMOTE_EXECUTION_CONTAINERS = {
"amazon_linux_2": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:9c6213c1f59fb2e18e3cf983422cfb89b54776ac654416e18136bf93e1685d78",
"dockerfile": "bazel/remote_execution_container/amazon_linux_2/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:9c6213c1f59fb2e18e3cf983422cfb89b54776ac654416e18136bf93e1685d78",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:57c9b945990dd2288c0dae78155d5478b5b5816a77ea5c632aeaf086b71b583b",
"dockerfile": "bazel/remote_execution_container/amazon_linux_2/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:57c9b945990dd2288c0dae78155d5478b5b5816a77ea5c632aeaf086b71b583b",
},
"amazon_linux_2023": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:6e860a09434327b93c47c11b64d369f7e6b04429d5610a9b063118bcad560e85",
"dockerfile": "bazel/remote_execution_container/amazon_linux_2023/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:6e860a09434327b93c47c11b64d369f7e6b04429d5610a9b063118bcad560e85",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:06fc8103d2e2af9878ee7c5a792f32b3de1c8d68266c193f7ab35b7a136da519",
"dockerfile": "bazel/remote_execution_container/amazon_linux_2023/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:06fc8103d2e2af9878ee7c5a792f32b3de1c8d68266c193f7ab35b7a136da519",
},
"debian10": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:18dbe847c591cf21822aae1ba596da27dcd7a4cb1b5ca1e02449caad11718e4b",
"dockerfile": "bazel/remote_execution_container/debian10/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:18dbe847c591cf21822aae1ba596da27dcd7a4cb1b5ca1e02449caad11718e4b",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:4917278b646f5cd298013cad8a6bf645a258d14a22b93b5389975d2db80baff4",
"dockerfile": "bazel/remote_execution_container/debian10/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:4917278b646f5cd298013cad8a6bf645a258d14a22b93b5389975d2db80baff4",
},
"debian12": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:b6171ae7dba5e5e87df627bcf621ca06642ba1f51de509dc894fa5c2b668215a",
"dockerfile": "bazel/remote_execution_container/debian12/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:b6171ae7dba5e5e87df627bcf621ca06642ba1f51de509dc894fa5c2b668215a",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:d1df0fd76f9fdf28813fbe25dc9b989b5228f94cd37eae11fb194f67d3afdbbb",
"dockerfile": "bazel/remote_execution_container/debian12/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:d1df0fd76f9fdf28813fbe25dc9b989b5228f94cd37eae11fb194f67d3afdbbb",
},
"linux": {
"_COMMENT": "Uses amazon linux 2 container",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:309525376d94d86406ea6e4f6e6ecb0bf9f5b678402516ca0ce286eed27887b9",
"dockerfile": "bazel/remote_execution_container/amazon_linux_2/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:309525376d94d86406ea6e4f6e6ecb0bf9f5b678402516ca0ce286eed27887b9",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:66b4f75708d3f845918b91a7fafb53b9c06612f3bca8eb87a179a206655357ca",
"dockerfile": "bazel/remote_execution_container/amazon_linux_2/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:66b4f75708d3f845918b91a7fafb53b9c06612f3bca8eb87a179a206655357ca",
},
"rhel8": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:d712b0093205bbcb96d6c3dcea615b9fc4af7fd32ae68cf4d0ffbfc1d36533c2",
"dockerfile": "bazel/remote_execution_container/rhel89/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:d712b0093205bbcb96d6c3dcea615b9fc4af7fd32ae68cf4d0ffbfc1d36533c2",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:f80dfd1e9a9d1ea28225b6ecb8049e5789cc365176a0766422771d58dda3708f",
"dockerfile": "bazel/remote_execution_container/rhel89/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:f80dfd1e9a9d1ea28225b6ecb8049e5789cc365176a0766422771d58dda3708f",
},
"rhel9": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:d718a69fcc22e52a66f340c77fc3857e822cf388420a11130993b189dea8b63f",
"dockerfile": "bazel/remote_execution_container/rhel93/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:d718a69fcc22e52a66f340c77fc3857e822cf388420a11130993b189dea8b63f",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:89dbd46e1e9540b8689bf5431a78afe0cc15906b1ba0d6babf00ee59aba574b3",
"dockerfile": "bazel/remote_execution_container/rhel93/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:89dbd46e1e9540b8689bf5431a78afe0cc15906b1ba0d6babf00ee59aba574b3",
},
"suse15": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:dd84ce6e6465417f9d7996bb04c6cd82f5654359004b1f840f8815c5e6164769",
"dockerfile": "bazel/remote_execution_container/suse/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:dd84ce6e6465417f9d7996bb04c6cd82f5654359004b1f840f8815c5e6164769",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:5295903271760e7e555590fb8858c1f7a74d3c0dd109d8ca200130ce07343ea4",
"dockerfile": "bazel/remote_execution_container/suse/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:5295903271760e7e555590fb8858c1f7a74d3c0dd109d8ca200130ce07343ea4",
},
"ubuntu18": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:81def09a0b2f0a3af605e2d30a33d6fb89ac0b2df27f1d4e373aeea91697d05a",
"dockerfile": "bazel/remote_execution_container/ubuntu18/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:81def09a0b2f0a3af605e2d30a33d6fb89ac0b2df27f1d4e373aeea91697d05a",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:fad0b715ba85885f23eb43292014529bd4b9946e1ca1b825c2d8c9443f95a3e1",
"dockerfile": "bazel/remote_execution_container/ubuntu18/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:fad0b715ba85885f23eb43292014529bd4b9946e1ca1b825c2d8c9443f95a3e1",
},
"ubuntu20": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:7d4f25930ac7e15c240c81b99d38fa4c9227e0ee790fc3c7b9950cf0b8e23cd9",
"dockerfile": "bazel/remote_execution_container/ubuntu20/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:7d4f25930ac7e15c240c81b99d38fa4c9227e0ee790fc3c7b9950cf0b8e23cd9",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:ed2d5bd04f0f23dbb57ff4e0f6d73f732e87b5ec976f7283908003a45b98a82e",
"dockerfile": "bazel/remote_execution_container/ubuntu20/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:ed2d5bd04f0f23dbb57ff4e0f6d73f732e87b5ec976f7283908003a45b98a82e",
},
"ubuntu22": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:6c130acbd9016cf56a3c8bac8b6836663da93542dd1eb8fc6a44e33997774cdc",
"dockerfile": "bazel/remote_execution_container/ubuntu22/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:6c130acbd9016cf56a3c8bac8b6836663da93542dd1eb8fc6a44e33997774cdc",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:c1e4291aa0a1054e5c119a6c6a2ac3b43b2903e2f2c68d30148da7b57691ee9d",
"dockerfile": "bazel/remote_execution_container/ubuntu22/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:c1e4291aa0a1054e5c119a6c6a2ac3b43b2903e2f2c68d30148da7b57691ee9d",
},
"ubuntu24": {
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:ed1400f93a93ad7ca5821aa3e1b0a0192532d420379c296ecb93ad504020037b",
"dockerfile": "bazel/remote_execution_container/ubuntu24/dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:ed1400f93a93ad7ca5821aa3e1b0a0192532d420379c296ecb93ad504020037b",
"container-url": "docker://quay.io/mongodb/bazel-remote-execution@sha256:8028f63bd4514e47871c7506aee6280e9f83db1d74e90a525ec61d30375f85bb",
"dockerfile": "bazel/remote_execution_container/ubuntu24/Dockerfile",
"web-url": "https://quay.io/repository/mongodb/bazel-remote-execution/manifest/sha256:8028f63bd4514e47871c7506aee6280e9f83db1d74e90a525ec61d30375f85bb",
},
}

93
bazel/platforms/remote_execution_containers_generator.py
View File

@ -2,13 +2,10 @@
# the dockerfiles listed in bazel/remote_execution_container/.
import argparse
from datetime import datetime
import json
import os
import pathlib
import subprocess
import tempfile
import urllib.request
from datetime import datetime
def log_subprocess_run(*args, **kwargs):
@ -19,10 +16,12 @@ def log_subprocess_run(*args, **kwargs):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--distro", type=str,
help="Restrict to only update a single distro.")
parser.add_argument("--skip-cleanup", action="store_true",
help="Disable cleanup between container builds. This requires a large amount of disk space.")
parser.add_argument("--distro", type=str, help="Restrict to only update a single distro.")
parser.add_argument(
"--skip-cleanup",
action="store_true",
help="Disable cleanup between container builds. This requires a large amount of disk space.",
)
args = parser.parse_args()
if not args.skip_cleanup:
@ -32,15 +31,17 @@ purged after each step to preserve disk space.
Pass --skip-cleanup to disable this step, but be aware that this will require a large amount of disk space.
Your docker images, volumes and containers will be purged if you continue. Enter 'y' to continue or 'n' to exit:""")
if user_input.lower() != 'y':
if user_input.lower() != "y":
return 0
remote_execution_containers = {}
container_file_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "remote_execution_containers.bzl")
container_file_path = os.path.join(
pathlib.Path(__file__).parent.resolve(), "remote_execution_containers.bzl"
)
with open(container_file_path, "r") as f:
code = compile(f.read(), container_file_path, "exec")
exec(code, {}, remote_execution_containers)
for distro, re_container in remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"].items():
if args.distro is not None:
if distro != args.distro:
@ -65,42 +66,64 @@ Your docker images, volumes and containers will be purged if you continue. Enter
print(f"Using tag: {tag}\n")
log_subprocess_run(["docker", "buildx", "create", "--use", "default"], check=True)
log_subprocess_run([
"docker", "buildx", "build", "--push",
"--platform", "linux/arm64/v8,linux/amd64",
"--tag", tag,
str(pathlib.Path(re_container["dockerfile"]).parent.resolve()),
], check=True)
log_subprocess_run(
[
"docker",
"buildx",
"build",
"--push",
"--platform",
"linux/arm64/v8,linux/amd64",
"--tag",
tag,
str(pathlib.Path(re_container["dockerfile"]).parent.resolve()),
],
check=True,
)
log_subprocess_run(["docker", "pull", tag], check=True)
result = log_subprocess_run(["docker", "inspect", "--format='{{.RepoDigests}}'", tag], capture_output=True, text=True, check=True)
result = log_subprocess_run(
["docker", "inspect", "--format='{{.RepoDigests}}'", tag],
capture_output=True,
text=True,
check=True,
)
# The output of this command is a list of strings, ex. ['URL'] so we need to strip off the brackets and quotes.
re_container["container-url"] = "docker://" + result.stdout.strip()[2:-2]
re_container["web-url"] = "https://" + result.stdout.strip()[2:-2].replace("quay.io/", "quay.io/repository/").replace("@sha256", "/manifest/sha256")
re_container["web-url"] = "https://" + result.stdout.strip()[2:-2].replace(
"quay.io/", "quay.io/repository/"
).replace("@sha256", "/manifest/sha256")
print(f"Finished updating {distro}")
print("************************************\n")
with open(container_file_path, "w") as f:
print(f"Writing remote execution container map to {container_file_path}...")
print("# Use bazel/platforms/remote_execution_containers_generator.py to generate this mapping for a given patch build.\n", file=f)
with open(container_file_path, "w") as f:
print(f"Writing remote execution container map to {container_file_path}...")
print(
"# Use bazel/platforms/remote_execution_containers_generator.py to generate this mapping for a given patch build.\n",
file=f,
)
# Manually print out the dict to maintain the trailing comma in each last element to satisfy the buildifier lint rules and
# avoid reformating.
print("REMOTE_EXECUTION_CONTAINERS = {", file=f)
for key, value in sorted(remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"].items(), key=lambda x: x[0]):
print(f" \"{key}\": {{", file=f)
for subkey, subvalue in sorted(value.items(), key=lambda x: x[0]):
print(f" \"{subkey}\": \"{subvalue}\",", file=f)
print(" },", file=f)
print("}", file=f)
# Manually print out the dict to maintain the trailing comma in each last element to satisfy the buildifier lint rules and
# avoid reformating.
print("REMOTE_EXECUTION_CONTAINERS = {", file=f)
for key, value in sorted(
remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"].items(),
key=lambda x: x[0],
):
print(f' "{key}": {{', file=f)
for subkey, subvalue in sorted(value.items(), key=lambda x: x[0]):
print(f' "{subkey}": "{subvalue}",', file=f)
print(" },", file=f)
print("}", file=f)
with open(container_file_path, "r") as f:
print(f"Finished writing to {container_file_path}:")
print(f.read())
with open(container_file_path, "r") as f:
print(f"Finished writing to {container_file_path}:")
print(f.read())
return 0
if __name__ == '__main__':
if __name__ == "__main__":
exit(main())

4
bazel/remote_execution_container/BUILD.bazel Normal file
View File

@ -0,0 +1,4 @@
sh_binary(
name = "repin_dockerfiles",
srcs = ["repin_dockerfiles.sh"],
)

27
bazel/remote_execution_container/amazon_linux_2/dockerfile
View File

@ -1,8 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM amazonlinux:2
FROM amazonlinux:2@sha256:4b425753853906770081d67626d4e8ef63516bb3e056ecbb3c873c57ba74abf1
# glibc-devel is required for headers, libzstd is required by gcc.
RUN yum -y install glibc-devel libzstd
RUN yum check-update || true && \
yum install -y \
cyrus-sasl-devel-2.1.26-24.amzn2 \
cyrus-sasl-gssapi-2.1.26-24.amzn2 \
glibc-devel-2.26-64.amzn2.0.2 \
krb5-devel-1.15.1-55.amzn2.2.8 \
libcurl-devel-8.3.0-1.amzn2.0.7 \
libzstd-1.5.5-1.amzn2.0.1 \
openldap-devel-2.4.44-25.amzn2.0.7 \
openssl-devel-1.0.2k-24.amzn2.0.12 \
systemtap-sdt-devel-4.5-1.amzn2.0.1 \
&& yum clean all && rm -rf /var/cache/yum/*
CMD ["/bin/bash"]

27
bazel/remote_execution_container/amazon_linux_2023/dockerfile
View File

@ -1,8 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM amazonlinux:2023
FROM amazonlinux:2023@sha256:ef1a9c856a0d3b2d581d0d9261540df3c6ab3a7f752546603c93f954def936a3
# glibc-devel is required for headers, libzstd is required by gcc.
RUN yum -y install glibc-devel libzstd
RUN yum check-update || true && \
yum install -y \
cyrus-sasl-devel-2.1.27-18.amzn2023.0.3 \
cyrus-sasl-gssapi-2.1.27-18.amzn2023.0.3 \
glibc-devel-2.34-52.amzn2023.0.10 \
krb5-devel-1.21-3.amzn2023.0.4 \
libcurl-devel-8.5.0-1.amzn2023.0.4 \
libzstd-1.5.5-1.amzn2023.0.1 \
openldap-devel-2.4.57-6.amzn2023.0.6 \
openssl-devel-3.0.8-1.amzn2023.0.12 \
systemtap-sdt-devel-4.8-3.amzn2023.0.5 \
&& yum clean all && rm -rf /var/cache/yum/*
CMD ["/bin/bash"]

28
bazel/remote_execution_container/debian10/dockerfile
View File

@ -1,9 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM debian:10
FROM debian:10@sha256:58ce6f1271ae1c8a2006ff7d3e54e9874d839f573d8009c20154ad0f2fb0a225
RUN apt-get update && apt-get install -y build-essential libfl2
RUN apt-get install -y libxml2-dev
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
build-essential=12.6 \
libcurl4-openssl-dev=7.64.0-4+deb10u9 \
libfl2=2.6.4-6.2 \
libgssapi-krb5-2=1.17-3+deb10u6 \
libkrb5-dev=1.17-3+deb10u6 \
libldap2-dev=2.4.47+dfsg-3+deb10u7 \
libsasl2-dev=2.1.27+dfsg-1+deb10u2 \
libssl-dev=1.1.1n-0+deb10u6 \
libxml2-dev=2.9.4+dfsg1-7+deb10u6 \
&& rm -rf /var/lib/apt/lists/*
CMD ["/bin/bash"]

27
bazel/remote_execution_container/debian12/dockerfile
View File

@ -1,9 +1,26 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM debian:12
FROM debian:12@sha256:45f2e735295654f13e3be10da2a6892c708f71a71be845818f6058982761a6d3
RUN apt-get update && apt-get install -y build-essential
RUN apt-get install -y libxml2-dev
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
build-essential=12.9 \
libcurl4-openssl-dev=7.88.1-10+deb12u6 \
libgssapi-krb5-2=1.20.1-2+deb12u2 \
libkrb5-dev=1.20.1-2+deb12u2 \
libldap2-dev=2.5.13+dfsg-5 \
libsasl2-dev=2.1.28+dfsg-10 \
libssl-dev=3.0.13-1~deb12u1 \
libxml2-dev=2.9.14+dfsg-1.3~deb12u1 \
&& rm -rf /var/lib/apt/lists/*
CMD ["/bin/bash"]

273
bazel/remote_execution_container/repin_dockerfiles.sh Executable file
View File

@ -0,0 +1,273 @@
#!/usr/bin/env bash
set -euo pipefail
# Check if we're running under Bazel
if [[ -n "${BUILD_WORKSPACE_DIRECTORY-}" ]]; then
# We're running under Bazel, use the workspace directory
REPO_DIR="$BUILD_WORKSPACE_DIRECTORY"
else
# We're not running under Bazel, use the Git root
REPO_DIR=$(git rev-parse --show-toplevel 2>/dev/null)
if [[ -z "$REPO_DIR" ]]; then
echo "Error: Not in a Git repository and not running under Bazel."
exit 1
fi
fi
# Use the repository directory as the base for output. This should always be the
# same, regardless of invocation location.
cd "$REPO_DIR/bazel/remote_execution_container"
echo "Working directory: $(pwd)"
# Common packages for different distribution families. Packages you add here
# will end up in all images of this distribution family.
declare -A COMMON_PACKAGES
COMMON_PACKAGES["redhat"]="
cyrus-sasl-devel
cyrus-sasl-gssapi
glibc-devel
krb5-devel
libcurl-devel
openldap-devel
openssl-devel
systemtap-sdt-devel
"
COMMON_PACKAGES["debian"]="
build-essential
libcurl4-openssl-dev
libgssapi-krb5-2
libldap2-dev
libsasl2-dev
libssl-dev
libxml2-dev
libkrb5-dev
"
COMMON_PACKAGES["suse"]="
cyrus-sasl-devel
cyrus-sasl-gssapi
glibc-devel
krb5-devel
libcurl-devel
libopenssl-devel
openldap2-devel
"
# Distribution-specific additional packages. If you have a package that
# shouldn't be installed in all containers, use this section to add custom
# packages.
declare -A ADDITIONAL_PACKAGES
ADDITIONAL_PACKAGES["amazonlinux:2"]="
libzstd
"
ADDITIONAL_PACKAGES["amazonlinux:2023"]="
libzstd
"
ADDITIONAL_PACKAGES["redhat/ubi8:8.9"]="
ncurses-compat-libs
"
ADDITIONAL_PACKAGES["debian:10"]="
libfl2
"
ADDITIONAL_PACKAGES["opensuse/leap:15.2"]="
libfl2
"
ADDITIONAL_PACKAGES["ubuntu:18.04"]="
systemtap-sdt-dev
"
ADDITIONAL_PACKAGES["ubuntu:20.04"]="
systemtap-sdt-dev
"
ADDITIONAL_PACKAGES["ubuntu:22.04"]="
systemtap-sdt-dev
"
ADDITIONAL_PACKAGES["ubuntu:24.04"]="
systemtap-sdt-dev
"
# This maps container images to the output locations for the generated
# Dockerfiles.
declare -A IMAGE_DIRS
IMAGE_DIRS["amazonlinux:2"]="amazon_linux_2"
IMAGE_DIRS["amazonlinux:2023"]="amazon_linux_2023"
IMAGE_DIRS["debian:10"]="debian10"
IMAGE_DIRS["debian:12"]="debian12"
IMAGE_DIRS["redhat/ubi8:8.9"]="rhel89"
IMAGE_DIRS["redhat/ubi9:9.3"]="rhel93"
IMAGE_DIRS["opensuse/leap:15.2"]="suse"
IMAGE_DIRS["ubuntu:18.04"]="ubuntu18"
IMAGE_DIRS["ubuntu:20.04"]="ubuntu20"
IMAGE_DIRS["ubuntu:22.04"]="ubuntu22"
IMAGE_DIRS["ubuntu:24.04"]="ubuntu24"
# Fetch the latest sha256:xxx hash for an image.
get_latest_sha() {
docker pull "$1" >/dev/null
docker inspect --format='{{index .RepoDigests 0}}' "$1" | cut -d@ -f2
}
# Determine the package manager for the distribution.
get_package_manager() {
local image="$1"
if [[ "$image" == *"amazonlinux"* || "$image" == *"redhat"* ]]; then
echo "yum"
elif [[ "$image" == *"debian"* || "$image" == *"ubuntu"* ]]; then
echo "apt"
elif [[ "$image" == *"opensuse"* ]]; then
echo "zypper"
else
echo "unknown"
fi
}
# Expand the global distribution packages.
get_distribution_family() {
local image="$1"
if [[ "$image" == *"amazonlinux"* || "$image" == *"redhat"* ]]; then
echo "redhat"
elif [[ "$image" == *"debian"* || "$image" == *"ubuntu"* ]]; then
echo "debian"
elif [[ "$image" == *"opensuse"* ]]; then
echo "suse"
else
echo "unknown"
fi
}
# Find the pinned package versions.
get_package_versions() {
local image="$1"
shift
local packages=("$@")
local pkg_manager
pkg_manager=$(get_package_manager "$image")
case "$pkg_manager" in
yum)
docker run --rm "$image" bash -c "
yum info ${packages[*]} 2>/dev/null |
awk '/^Name/ {name=\$3} /^Version/ {version=\$3} /^Release/ {release=\$3}
/^Release/ {print name \"-\" version \"-\" release}' |
sort -u"
;;
apt)
docker run --rm "$image" bash -c "
apt-get update >/dev/null 2>&1 &&
apt-cache policy ${packages[*]} |
awk '/^[^ ]/ {pkg=\$1} /Candidate:/ {print pkg \"=\" \$2}' |
sort -u"
;;
zypper)
# TODO(SERVER-93423): Pin suse package versions. At the moment this
# breaks the remote_execution_containers_generator.py script.
printf '%s\n' "${packages[@]}" | sort -u
# docker run --rm "$image" bash -c "
# zypper --non-interactive refresh >/dev/null 2>&1 &&
# zypper --non-interactive info ${packages[*]} |
# awk '/^Name/ {name=\$3} /^Version/ {version=\$3} /^Version/ {print name \"=\" version}' |
# sort -u"
;;
*)
echo "Unsupported package manager for image: $image" >&2
return 1
;;
esac
}
# Write the Dockerfile.
generate_dockerfile() {
local image="$1"
local output_dir="${IMAGE_DIRS[$image]}"
local distribution_family
distribution_family=$(get_distribution_family "$image")
readarray -t packages < <(echo "${COMMON_PACKAGES[$distribution_family]}" | sed '/^\s*$/d')
readarray -t additional < <(echo "${ADDITIONAL_PACKAGES[$image]:-}" | sed '/^\s*$/d')
packages+=("${additional[@]}")
local sha
sha=$(get_latest_sha "$image")
local image_with_sha="${image}@${sha}"
local pkg_manager
pkg_manager=$(get_package_manager "$image")
local install_lines
install_lines=$(get_package_versions "$image" "${packages[@]}" | sed 's/^/ /' | sed 's/$/\ \\/')
case "$pkg_manager" in
yum)
update_cmd="yum check-update || true"
install_cmd="yum install -y"
clean_cmd="&& yum clean all && rm -rf /var/cache/yum/*"
;;
apt)
update_cmd="apt-get update"
install_cmd="DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends"
clean_cmd="&& rm -rf /var/lib/apt/lists/*"
;;
zypper)
update_cmd="zypper refresh"
install_cmd="zypper install -y --no-recommends"
clean_cmd="&& zypper clean --all"
;;
*)
echo "Unsupported package manager for image: $image" >&2
return 1
;;
esac
# Remove colons from package versions for Debian and Ubuntu
if [[ "$pkg_manager" == "apt" ]]; then
install_lines=${install_lines//:=/=}
fi
mkdir -p "$output_dir"
cat << EOF > "$output_dir/dockerfile"
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \\
# //bazel/remote_execution_container:repin_dockerfiles \\
# --config=local
FROM $image_with_sha
RUN $update_cmd && \\
$install_cmd \\
$install_lines
$clean_cmd
CMD ["/bin/bash"]
EOF
echo "Generated Dockerfile for $image in $output_dir"
}
# Generating the Dockerfiles doesn't really put the CPU under load. Generate
# all of them asynchronously to speed up processing.
echo "Generating Dockerfiles..."
for image in "${!IMAGE_DIRS[@]}"; do
(
generate_dockerfile "$image"
) &
done
wait
echo "Done."

28
bazel/remote_execution_container/rhel89/dockerfile
View File

@ -1,9 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM redhat/ubi8:8.9
FROM redhat/ubi8:8.9@sha256:83068ea81dd02717b8e39b55cdeb2c1b2c9a3db260f01381b991755d44b15073
RUN yum install -y glibc-devel
RUN yum install -y ncurses-compat-libs
RUN yum check-update || true && \
yum install -y \
cyrus-sasl-devel-2.1.27-6.el8_5 \
cyrus-sasl-gssapi-2.1.27-6.el8_5 \
glibc-devel-2.28-251.el8_10.2 \
krb5-devel-1.18.2-28.el8_10 \
libcurl-devel-7.61.1-34.el8 \
ncurses-compat-libs-6.1-10.20180224.el8 \
openldap-devel-2.4.46-19.el8_10 \
openssl-devel-1.1.1k-12.el8_9 \
systemtap-sdt-devel-4.9-3.el8 \
&& yum clean all && rm -rf /var/cache/yum/*
CMD ["/bin/bash"]

25
bazel/remote_execution_container/rhel93/dockerfile
View File

@ -1,7 +1,26 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM redhat/ubi9:9.3
FROM redhat/ubi9:9.3@sha256:66233eebd72bb5baa25190d4f55e1dc3fff3a9b77186c1f91a0abdb274452072
RUN yum install -y glibc-devel
RUN yum check-update || true && \
yum install -y \
cyrus-sasl-devel-2.1.27-21.el9 \
cyrus-sasl-gssapi-2.1.27-21.el9 \
glibc-devel-2.34-100.el9_4.2 \
krb5-devel-1.21.1-1.el9 \
libcurl-devel-7.76.1-29.el9_4 \
openldap-devel-2.6.6-3.el9 \
openssl-devel-3.0.7-27.el9 \
systemtap-sdt-devel-5.0-4.el9 \
&& yum clean all && rm -rf /var/cache/yum/*
CMD ["/bin/bash"]

25
bazel/remote_execution_container/suse/dockerfile
View File

@ -1,7 +1,26 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM opensuse/leap:15.2
FROM opensuse/leap:15.2@sha256:dfa464ed7bc25fb77ad652d4e722cb0e78fc230425846be10e51dda1f43aa5c9
RUN zypper install -y glibc-devel libfl2
RUN zypper refresh && \
zypper install -y --no-recommends \
cyrus-sasl-devel \
cyrus-sasl-gssapi \
glibc-devel \
krb5-devel \
libcurl-devel \
libfl2 \
libopenssl-devel \
openldap2-devel \
&& zypper clean --all
CMD ["/bin/bash"]

28
bazel/remote_execution_container/ubuntu18/dockerfile
View File

@ -1,9 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM ubuntu:18.04
FROM ubuntu:18.04@sha256:152dc042452c496007f07ca9127571cb9c29697f42acbfad72324b2bb2e43c98
RUN apt-get update && apt-get install -y build-essential
RUN apt-get install -y libxml2-dev
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
build-essential=12.4ubuntu1 \
libcurl4-openssl-dev=7.58.0-2ubuntu3.24 \
libgssapi-krb5-2=1.16-2ubuntu0.4 \
libkrb5-dev=1.16-2ubuntu0.4 \
libldap2-dev=2.4.45+dfsg-1ubuntu1.11 \
libsasl2-dev=2.1.27~101-g0780600+dfsg-3ubuntu2.4 \
libssl-dev=1.1.1-1ubuntu2.1~18.04.23 \
libxml2-dev=2.9.4+dfsg1-6.1ubuntu1.9 \
systemtap-sdt-dev=3.1-3ubuntu0.1 \
&& rm -rf /var/lib/apt/lists/*
CMD ["/bin/bash"]

28
bazel/remote_execution_container/ubuntu20/dockerfile
View File

@ -1,9 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM ubuntu:20.04
FROM ubuntu:20.04@sha256:0b897358ff6624825fb50d20ffb605ab0eaea77ced0adb8c6a4b756513dec6fc
RUN apt-get update && apt-get install -y build-essential
RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libxml2-dev
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
build-essential=12.8ubuntu1.1 \
libcurl4-openssl-dev=7.68.0-1ubuntu2.23 \
libgssapi-krb5-2=1.17-6ubuntu4.6 \
libkrb5-dev=1.17-6ubuntu4.6 \
libldap2-dev=2.4.49+dfsg-2ubuntu1.10 \
libsasl2-dev=2.1.27+dfsg-2ubuntu0.1 \
libssl-dev=1.1.1f-1ubuntu2.23 \
libxml2-dev=2.9.10+dfsg-5ubuntu0.20.04.7 \
systemtap-sdt-dev=4.2-3ubuntu0.1 \
&& rm -rf /var/lib/apt/lists/*
CMD ["/bin/bash"]

29
bazel/remote_execution_container/ubuntu22/dockerfile
View File

@ -1,10 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM ubuntu:22.04
FROM ubuntu:22.04@sha256:340d9b015b194dc6e2a13938944e0d016e57b9679963fdeb9ce021daac430221
RUN apt-get update && apt-get install -y build-essential
RUN apt-get install -y libxml2-dev
RUN apt-get install -y libffi8
RUN apt-get install -y libffi-dev
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
build-essential=12.9ubuntu3 \
libcurl4-openssl-dev=7.81.0-1ubuntu1.17 \
libgssapi-krb5-2=1.19.2-2ubuntu0.4 \
libkrb5-dev=1.19.2-2ubuntu0.4 \
libldap2-dev=2.5.18+dfsg-0ubuntu0.22.04.2 \
libsasl2-dev=2.1.27+dfsg2-3ubuntu1.2 \
libssl-dev=3.0.2-0ubuntu1.17 \
libxml2-dev=2.9.13+dfsg-1ubuntu0.4 \
systemtap-sdt-dev=4.6-2 \
&& rm -rf /var/lib/apt/lists/*
CMD ["/bin/bash"]

27
bazel/remote_execution_container/ubuntu24/dockerfile
View File

@ -1,8 +1,27 @@
# Image used by bazel remote execution hosts.
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin
# versions or change packages, edit that script instead.
#
# To repin the hashes:
#
# bazel run \
# //bazel/remote_execution_container:repin_dockerfiles \
# --config=local
FROM ubuntu:24.04
FROM ubuntu:24.04@sha256:2e863c44b718727c860746568e1d54afd13b2fa71b160f5cd9058fc436217b30
RUN apt-get update && apt-get install -y build-essential
RUN apt-get install -y libxml2-dev
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
build-essential=12.10ubuntu1 \
libcurl4-openssl-dev=8.5.0-2ubuntu10.2 \
libgssapi-krb5-2=1.20.1-6ubuntu2.1 \
libkrb5-dev=1.20.1-6ubuntu2.1 \
libldap2-dev=2.6.7+dfsg-1~exp1ubuntu8 \
libsasl2-dev=2.1.28+dfsg1-5ubuntu3 \
libssl-dev=3.0.13-0ubuntu3.2 \
libxml2-dev=2.9.14+dfsg-1.3ubuntu3 \
systemtap-sdt-dev=5.0-2ubuntu1 \
&& rm -rf /var/lib/apt/lists/*
CMD ["/bin/bash"]

0
bazel/repository_rules/BUILD.bazel Normal file
View File

20
bazel/repository_rules/evergreen_variables.bzl Normal file
View File

@ -0,0 +1,20 @@
def _setup_evergreen_variables(ctx):
compile_variant = ctx.os.environ.get("compile_variant")
version_id = ctx.os.environ.get("version_id")
ctx.file(
"BUILD.bazel",
"",
)
ctx.file(
"evergreen_variables.bzl",
"""
UNSAFE_COMPILE_VARIANT = "%s"
UNSAFE_VERSION_ID = "%s"
""" % (compile_variant, version_id),
)
setup_evergreen_variables = repository_rule(
implementation = _setup_evergreen_variables,
environ = ["compile_variant", "version_id"],
)

93
bazel/resmoke/BUILD.bazel Normal file
View File

@ -0,0 +1,93 @@
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
load("@poetry//:dependencies.bzl", "dependency")
genrule(
name = "resmoke_mongo_version",
srcs = [],
outs = [".resmoke_mongo_version.yml"],
cmd = """
echo mongo_version: $(MONGO_VERSION) > $@
""",
visibility = ["//visibility:public"],
)
genrule(
name = "on_feature_flags",
srcs = [],
outs = ["on_feature_flags.txt"],
cmd = """
awk '/^on_feature_flags/ { for (i=2; i<=NF; i++) print $$i }' bazel-out/volatile-status.txt > $@
""",
stamp = True,
tags = ["external"], # Consuming the feature flag information from volatile-status, so this should always execute.
visibility = ["//visibility:public"],
)
genrule(
name = "off_feature_flags",
srcs = [],
outs = ["off_feature_flags.txt"],
cmd = """
awk '/^off_feature_flags/ { for (i=2; i<=NF; i++) print $$i }' bazel-out/volatile-status.txt > $@
""",
stamp = True,
tags = ["external"], # Consuming the feature flag information from volatile-status, so this should always execute.
visibility = ["//visibility:public"],
)
genrule(
name = "unreleased_ifr_flags",
srcs = [],
outs = ["unreleased_ifr_flags.txt"],
cmd = """
awk '/^unreleased_ifr_flags/ { for (i=2; i<=NF; i++) print $$i }' bazel-out/volatile-status.txt > $@
""",
stamp = True,
tags = ["external"], # Consuming the feature flag information from volatile-status, so this should always execute.
visibility = ["//visibility:public"],
)
# We want to use keys from volatile-status in the resmoke `py_test`, so this creates a file that
# it can depend on.
genrule(
name = "volatile_status",
srcs = [],
outs = ["volatile-status.txt"],
cmd = """
cat bazel-out/volatile-status.txt > $@
""",
stamp = True,
tags = ["external"], # Consuming information from volatile-status, so this should always execute.
visibility = ["//visibility:public"],
)
bool_flag(
name = "in_evergreen",
build_setting_default = False,
)
config_setting(
name = "in_evergreen_enabled",
flag_values = {
"//bazel/resmoke:in_evergreen": "True",
},
)
py_binary(
name = "resmoke_config_generator",
srcs = ["resmoke_config_generator.py"],
main = "resmoke_config_generator.py",
visibility = ["//visibility:public"],
deps = [
dependency(
"typer",
group = "core",
),
dependency(
"pyyaml",
group = "core",
),
],
)
exports_files(["resmoke_shim.py"])

5
bazel/resmoke/OWNERS.yml Normal file
View File

@ -0,0 +1,5 @@
version: 1.0.0
filters:
- "*":
approvers:
- 10gen/devprod-correctness

128
bazel/resmoke/resmoke.bzl Normal file
View File

@ -0,0 +1,128 @@
def resmoke_config_impl(ctx):
base_name = ctx.label.name.removesuffix("_config")
test_list_file = ctx.actions.declare_file(base_name + ".txt")
generated_config_file = ctx.actions.declare_file(base_name + ".yml")
base_config_file = ctx.files.base_config[0]
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
python_path = []
for path in ctx.attr.generator[PyInfo].imports.to_list():
if path not in python_path:
python_path.append(ctx.expand_make_variables("python_library_imports", "$(BINDIR)/external/" + path, ctx.var))
generator_deps = [ctx.attr.generator[PyInfo].transitive_sources]
test_list = [test.short_path for test in ctx.files.srcs]
for exclude in [test.short_path for test in ctx.files.exclude_files]:
if exclude in test_list:
test_list.remove(exclude)
ctx.actions.write(test_list_file, "\n".join(test_list))
deps = depset([test_list_file, base_config_file] + ctx.files.srcs, transitive = [python.files] + generator_deps)
ctx.actions.run(
executable = python.interpreter.path,
inputs = deps,
outputs = [generated_config_file],
arguments = [
"bazel/resmoke/resmoke_config_generator.py",
"--output",
generated_config_file.path,
"--test-list",
test_list_file.path,
"--base-config",
base_config_file.path,
"--exclude-with-any-tags",
",".join(ctx.attr.exclude_with_any_tags),
"--include-with-any-tags",
",".join(ctx.attr.include_with_any_tags),
],
env = {"PYTHONPATH": ctx.configuration.host_path_separator.join(python_path)},
)
return [DefaultInfo(files = depset([generated_config_file]))]
resmoke_config = rule(
resmoke_config_impl,
attrs = {
"generator": attr.label(
doc = "The config generator to use.",
default = "//bazel/resmoke:resmoke_config_generator",
),
"srcs": attr.label_list(allow_files = True, doc = "Tests to write as the 'roots' of the selector"),
"exclude_files": attr.label_list(allow_files = True),
"exclude_with_any_tags": attr.string_list(),
"include_with_any_tags": attr.string_list(),
"base_config": attr.label(
allow_files = True,
doc = "The base resmoke YAML config for the suite",
),
},
doc = "Generates a resmoke config YAML",
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
)
def resmoke_suite_test(
name,
config,
data = [],
deps = [],
exclude_files = [],
exclude_with_any_tags = [],
include_with_any_tags = [],
resmoke_args = [],
srcs = [],
tags = [],
timeout = "eternal",
**kwargs):
generated_config = name + "_config"
resmoke_config(
name = generated_config,
srcs = srcs,
exclude_files = exclude_files,
base_config = config,
exclude_with_any_tags = exclude_with_any_tags,
include_with_any_tags = include_with_any_tags,
tags = ["resmoke_config"],
)
resmoke_shim = Label("//bazel/resmoke:resmoke_shim.py")
resmoke = Label("//buildscripts:resmoke")
extra_args = select({
"//bazel/resmoke:in_evergreen_enabled": [
"--log=evg",
"--cedarReportFile=cedar_report.json",
"--skipSymbolization", # Symbolization is not yet functional, SERVER-103538
"--installDir=dist-test/bin",
],
"//conditions:default": ["--installDir=install-dist-test/bin"],
})
native.py_test(
name = name,
# To a user of resmoke_suite_test, the `srcs` is the list of tests to select. However, to the py_test rule,
# the `srcs` are expected to be Python files only.
srcs = [resmoke_shim],
data = data + srcs + [
generated_config,
"//bazel/resmoke:resmoke_mongo_version",
"//bazel/resmoke:on_feature_flags",
"//bazel/resmoke:off_feature_flags",
"//bazel/resmoke:unreleased_ifr_flags",
"//bazel/resmoke:volatile_status",
"//buildscripts/resmokeconfig:all_files", # This needs to be reduced, SERVER-103610
"//src/mongo/util/version:releases.yml",
] + select({
"//bazel/resmoke:in_evergreen_enabled": ["//:installed-dist-test"],
"//conditions:default": ["//:install-dist-test"],
}),
deps = deps + [resmoke],
main = resmoke_shim,
args = [
"run",
"--suites=$(location %s)" % native.package_relative_label(generated_config),
"--multiversionDir=multiversion_binaries",
"--continueOnFailure",
] + extra_args + resmoke_args,
tags = tags + ["no-cache", "local"],
timeout = timeout,
**kwargs
)

39
bazel/resmoke/resmoke_config_generator.py Normal file
View File

@ -0,0 +1,39 @@
from pathlib import Path
import typer
import yaml
from typing_extensions import Annotated
def main(
output: Annotated[Path, typer.Option()],
test_list: Annotated[Path, typer.Option()],
base_config: Annotated[Path, typer.Option()],
exclude_with_any_tags: Annotated[str, typer.Option()] = "",
include_with_any_tags: Annotated[str, typer.Option()] = "",
):
with open(test_list, "rt") as fh:
tests = [line.rstrip("\n") for line in fh]
with open(base_config, "rt") as fh:
base_config_content = yaml.safe_load(fh)
if "selector" in base_config_content:
for x in ["roots", "exclude_files", "exclude_with_any_tags", "include_with_any_tags"]:
base_config_content["selector"].pop(x, None)
content = base_config_content
if "selector" not in content:
content["selector"] = {}
content["selector"]["roots"] = tests
if exclude_with_any_tags != "":
content["selector"]["exclude_with_any_tags"] = exclude_with_any_tags.split(",")
if include_with_any_tags != "":
content["selector"]["include_with_any_tags"] = include_with_any_tags.split(",")
with open(output, "wt") as fh:
yaml.dump(content, fh)
if __name__ == "__main__":
typer.run(main)

81
bazel/resmoke/resmoke_shim.py Normal file
View File

@ -0,0 +1,81 @@
import os
import pathlib
import sys
from functools import cache
REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
sys.path.append(str(REPO_ROOT))
from buildscripts.resmokelib import cli
@cache
def get_volatile_status() -> dict:
volatile_status = {}
with open("bazel/resmoke/volatile-status.txt", "rt") as f:
for line in f:
key, val = line.strip().split(" ", 1)[:2]
volatile_status[key] = val
return volatile_status
def get_from_volatile_status(key):
volatile_status = get_volatile_status()
return volatile_status.get(key)
def add_volatile_arg(args, flag, key):
val = get_from_volatile_status(key)
if val:
args.append(flag + val)
def add_evergreen_build_info(args):
add_volatile_arg(args, "--buildId=", "build_id")
add_volatile_arg(args, "--distroId=", "distro_id")
add_volatile_arg(args, "--executionNumber=", "execution")
add_volatile_arg(args, "--projectName=", "project")
add_volatile_arg(args, "--gitRevision=", "revision")
add_volatile_arg(args, "--revisionOrderId=", "revision_order_id")
add_volatile_arg(args, "--taskId=", "task_id")
add_volatile_arg(args, "--taskName=", "task_name")
add_volatile_arg(args, "--variantName=", "build_variant")
add_volatile_arg(args, "--versionId=", "version_id")
add_volatile_arg(args, "--requester=", "requester")
if __name__ == "__main__":
sys.argv[0] = (
"buildscripts/resmoke.py" # Ensure resmoke's local invocation is printed using resmoke.py directly
)
resmoke_args = sys.argv
# If there was an extra --suites argument added as a --test_arg in the bazel invocation, rewrite
# the original as --originSuite. Intentionally 'suite', sinces it is a common partial match for the actual
# argument 'suites'
suite_args = [i for i, arg in enumerate(resmoke_args) if arg.startswith("--suite")]
if len(suite_args) > 1:
resmoke_args[suite_args[0]] = resmoke_args[suite_args[0]].replace(
"--suites", "--originSuite"
)
add_evergreen_build_info(resmoke_args)
if os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR"):
undeclared_output_dir = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR")
resmoke_args.append(f"--dbpathPrefix={os.path.join(undeclared_output_dir,'data')}")
resmoke_args.append(f"--taskWorkDir={undeclared_output_dir}")
resmoke_args.append(f"--reportFile={os.path.join(undeclared_output_dir,'report.json')}")
if os.environ.get("TEST_SRCDIR"):
test_srcdir = os.environ.get("TEST_SRCDIR")
version_file = os.path.join(
test_srcdir, "_main", "bazel", "resmoke", ".resmoke_mongo_version.yml"
)
link_path = os.path.join(test_srcdir, "_main", ".resmoke_mongo_version.yml")
if not os.path.exists(link_path):
os.symlink(
version_file,
link_path,
)
cli.main(resmoke_args)

26
bazel/resmoke/volatile_status.sh Executable file
View File

@ -0,0 +1,26 @@
#! /bin/sh
# This script is used as a workspace status command
# bazel test --workspace_status_command=bazel/resmoke/volatile_status.sh
# to populate key-value pairs in bazel-out/volatile-status.txt.
# This file and the key-values can be consumed by bazel rules, but bazel
# pretends this file never changes when deciding what to rebuild.
# Evergreen expansions used primarily for Resmoke telemetry
echo build_id ${build_id}
echo distro_id ${distro_id}
echo execution ${execution}
echo project ${project}
echo revision ${revision}
echo revision_order_id ${revision_order_id}
echo task_id ${task_id}
echo task_name ${task_name}
echo build_variant ${build_variant}
echo version_id ${version_id}
echo requester ${requester}
# The current sets of enabled, disabled, and unrleased IFR feature flags. It
# would be better to remove this as it risks breaking the contract of
# volatile-status.txt. Changes to feature flag state should invalidate actions
# that consume this. SERVER-103590
python buildscripts/idl/gen_all_feature_flag_list.py feature-flag-status

0
bazel/rules_poetry/BUILD Normal file
View File

12
bazel/rules_poetry/rules_poetry.patch Normal file
View File

@ -0,0 +1,12 @@
diff --git MODULE.bazel MODULE.bazel
new file mode 100644
index 0000000..c09945f
--- /dev/null
+++ MODULE.bazel
@@ -0,0 +1,6 @@
+module(
+ name = "rules_poetry",
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "platforms", version = "0.0.8")

14
bazel/run_under_working_dir.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
set -e
RUNFILES_WORKING_DIRECTORY="$(pwd)"
if [ -z $BUILD_WORKING_DIRECTORY ]; then
echo "ERROR: BUILD_WORKING_DIRECTORY was not set, was this run from bazel?"
exit 1
fi
cd $BUILD_WORKING_DIRECTORY
${RUNFILES_WORKING_DIRECTORY}/${BINARY_PATH} "${@:1}"

377
bazel/separate_debug.bzl
View File

@ -1,8 +1,16 @@
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
TagInfo = provider(
doc = "A rule provider to pass around tags that were passed to rules.",
fields = {
"tags": "Bazel tags that were attached to the rule.",
},
)
WITH_DEBUG_SUFFIX = "_with_debug"
CC_SHARED_LIBRARY_SUFFIX = "_shared"
SHARED_ARCHIVE_SUFFIX = "_shared_archive"
MAC_DEBUG_FOLDER_EXTENSION = ".dSYM"
def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext):
"""
@ -19,6 +27,9 @@ def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext):
"""
shared_lib = None
static_lib = None
input_files = ctx.attr.binary_with_debug.files.to_list()
if len(input_files) == 0:
return None, None, None, None
if ctx.attr.type == "library":
for file in ctx.attr.binary_with_debug.files.to_list():
if file.path.endswith(WITH_DEBUG_SUFFIX + static_ext):
@ -32,7 +43,10 @@ def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext):
if shared_lib:
basename = shared_lib.basename[:-len(WITH_DEBUG_SUFFIX + shared_ext + CC_SHARED_LIBRARY_SUFFIX)]
if ctx.attr.enabled:
debug_info = ctx.actions.declare_file(basename + shared_ext + debug_ext)
if debug_ext == MAC_DEBUG_FOLDER_EXTENSION:
debug_info = ctx.actions.declare_directory(basename + shared_ext + debug_ext)
else:
debug_info = ctx.actions.declare_file(basename + shared_ext + debug_ext)
else:
debug_info = None
output_bin = ctx.actions.declare_file(basename + shared_ext)
@ -46,7 +60,10 @@ def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext):
basename = program_bin.basename[:-len(WITH_DEBUG_SUFFIX)]
if ctx.attr.enabled:
debug_info = ctx.actions.declare_file(basename + debug_ext)
if debug_ext == MAC_DEBUG_FOLDER_EXTENSION:
debug_info = ctx.actions.declare_directory(basename + debug_ext)
else:
debug_info = ctx.actions.declare_file(basename + debug_ext)
else:
debug_info = None
output_bin = ctx.actions.declare_file(basename)
@ -85,6 +102,20 @@ def get_transitive_dyn_libs(deps):
transitive_dyn_libs.append(library.dynamic_library)
return transitive_dyn_libs
def get_transitive_debug_files(deps):
"""
Get a transitive list of all dynamic library files under a set of dependencies.
"""
# TODO(SERVER-85819): Investigate to see if it's possible to merge the depset without looping over all transitive
# dependencies.
transitive_debugs = []
for dep in deps:
for input in dep[DefaultInfo].files.to_list():
if input.basename.endswith(".debug"):
transitive_debugs.append(input)
return transitive_debugs
def symlink_shared_archive(ctx, shared_ext, static_ext):
"""
Shared archives (.so.a/.dll.lib) have different extensions depending on the operating system.
@ -116,25 +147,52 @@ def create_new_ccinfo_library(ctx, cc_toolchain, shared_lib, static_lib, cc_shar
unsupported_features = ctx.disabled_features,
)
linker_input = cc_common.create_linker_input(
owner = ctx.label,
libraries = depset(direct = [
cc_common.create_library_to_link(
actions = ctx.actions,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
dynamic_library = shared_lib,
static_library = static_lib if cc_shared_library == None else None,
),
]),
user_link_flags = ctx.attr.binary_with_debug[CcInfo].linking_context.linker_inputs.to_list()[0].user_link_flags,
)
linker_input_deps = []
for dep in ctx.attr.deps:
linker_input_deps.append(dep[CcInfo].linking_context.linker_inputs)
linking_context = cc_common.create_linking_context(linker_inputs = depset(direct = [linker_input], transitive = linker_input_deps))
if shared_lib or static_lib:
if shared_lib:
so_path = shared_lib.path.replace(ctx.bin_dir.path + "/", "")
else:
so_path = ""
direct_lib = cc_common.create_library_to_link(
actions = ctx.actions,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
dynamic_library = shared_lib,
dynamic_library_symlink_path = so_path,
static_library = static_lib if cc_shared_library == None else None,
alwayslink = True,
)
# For some reason Bazel isn't deduplicating the user link flags, which leads to them accumulating
# with each layer added. Deduplicate them manually.
#
# This routine works by taking the current library's link args and searching for each of its dependency's link
# args present contiguously. If a matching sub list is found, it's removed from the current link line as a duplicate.
# This is to avoid removing positional arguments that may appear more than once.
#
# This solution may break in the case where a base dependency contains only one positional argument,
# but this should never happen since we will always inject at least one non positional argument globally.
cur_flags = ctx.attr.binary_with_debug[CcInfo].linking_context.linker_inputs.to_list()[0].user_link_flags
for dep in ctx.attr.binary_with_debug[CcInfo].linking_context.linker_inputs.to_list()[1:]:
for i in range(len(cur_flags)):
dep_flags = dep.user_link_flags
if dep_flags and cur_flags:
if cur_flags[i] == dep_flags[0] and cur_flags[i:i + len(dep_flags)] == dep_flags:
cur_flags = cur_flags[:i] + cur_flags[i + len(dep_flags):]
break
linker_input = cc_common.create_linker_input(
owner = ctx.label,
libraries = depset(direct = [direct_lib]),
user_link_flags = cur_flags,
)
linking_context = cc_common.create_linking_context(linker_inputs = depset(direct = [linker_input], transitive = linker_input_deps))
else:
linking_context = cc_common.create_linking_context(linker_inputs = depset(transitive = linker_input_deps))
else:
linking_context = ctx.attr.binary_with_debug[CcInfo].linking_context
@ -167,21 +225,44 @@ def create_new_cc_shared_library_info(ctx, cc_toolchain, output_shared_lib, orig
for library in input.libraries:
dep_libraries.append(library)
linker_input = cc_common.create_linker_input(
owner = ctx.label,
libraries = depset(direct = [
cc_common.create_library_to_link(
actions = ctx.actions,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
# Replace reference to dynamic library with final name
dynamic_library = output_shared_lib,
# Omit reference to static library
),
], transitive = [depset(dep_libraries)]),
user_link_flags = original_info.linker_input.user_link_flags,
additional_inputs = depset(original_info.linker_input.additional_inputs),
)
# CcInfo's linkopts are ignored by cc_shared_library by default. To support both transitive and nontransitive
# dynamic linkopts use:
# cc_library's linkopts field for both static and dynamic transitive link opts
# cc_shared_library's user_link_flags field for dynamic non-transitive link opts
all_user_link_flags = dict()
for input in ctx.attr.binary_with_debug[CcInfo].linking_context.linker_inputs.to_list():
for flag in input.user_link_flags:
all_user_link_flags[flag] = True
# We define global linkopts here too, remove duplicates to prevent repeats of the global opts
# from accumulating.
for flag in original_info.linker_input.user_link_flags:
all_user_link_flags[flag] = True
all_user_link_flags = [flag for flag, _ in all_user_link_flags.items()]
if output_shared_lib:
direct_lib = cc_common.create_library_to_link(
actions = ctx.actions,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
# Replace reference to dynamic library with final name
dynamic_library = output_shared_lib,
dynamic_library_symlink_path = output_shared_lib.path.replace(ctx.bin_dir.path + "/", ""),
# Omit reference to static library
)
linker_input = cc_common.create_linker_input(
owner = ctx.label,
libraries = depset(direct = [direct_lib], transitive = [depset(dep_libraries)]),
user_link_flags = all_user_link_flags,
additional_inputs = depset(original_info.linker_input.additional_inputs),
)
else:
linker_input = cc_common.create_linker_input(
owner = ctx.label,
libraries = depset(transitive = [depset(dep_libraries)]),
user_link_flags = all_user_link_flags,
additional_inputs = depset(original_info.linker_input.additional_inputs),
)
return CcSharedLibraryInfo(
dynamic_deps = original_info.dynamic_deps,
@ -190,18 +271,28 @@ def create_new_cc_shared_library_info(ctx, cc_toolchain, output_shared_lib, orig
linker_input = linker_input,
)
def noop_extraction(ctx):
return [
DefaultInfo(
files = depset(transitive = [ctx.attr.binary_with_debug.files]),
executable = ctx.attr.binary_with_debug.files[0] if ctx.attr.type == "program" else None,
),
ctx.attr.binary_with_debug[CcInfo],
]
# TODO(SERVER-101906): We assume the worst-case resource usage to avoid OOMs. Figure out if we can
# generalize numInputs for all link configurations so that this can more intelligently set resource
# expectations.
def linux_extract_resource_set(os, numInputs):
return {
"cpu": 6,
"memory": 20 * 1024, # 20 GB
"local_test": 1,
}
def linux_strip_resource_set(os, numInputs):
return {
"cpu": 3,
"memory": 10 * 1024, # 10 GB
"local_test": 1,
}
def linux_extraction(ctx, cc_toolchain, inputs):
outputs = []
unstripped_static_bin = None
input_bin, output_bin, debug_info, static_lib = get_inputs_and_outputs(ctx, ".so", ".a", ".debug")
input_file = ctx.attr.binary_with_debug.files.to_list()
if input_bin:
if ctx.attr.enabled:
@ -209,6 +300,7 @@ def linux_extraction(ctx, cc_toolchain, inputs):
executable = cc_toolchain.objcopy_executable,
outputs = [debug_info],
inputs = inputs,
resource_set = linux_extract_resource_set if ctx.attr.type == "program" else None,
arguments = [
"--only-keep-debug",
input_bin.path,
@ -221,6 +313,7 @@ def linux_extraction(ctx, cc_toolchain, inputs):
executable = cc_toolchain.objcopy_executable,
outputs = [output_bin],
inputs = depset([debug_info], transitive = [inputs]),
resource_set = linux_strip_resource_set if ctx.attr.type == "program" else None,
arguments = [
"--strip-debug",
"--add-gnu-debuglink",
@ -238,29 +331,42 @@ def linux_extraction(ctx, cc_toolchain, inputs):
)
outputs += [output_bin]
unstripped_static_bin = None
if static_lib:
unstripped_static_bin = propgate_static_lib(ctx, static_lib, ".a", inputs)
outputs.append(unstripped_static_bin)
if len(input_file):
if static_lib:
unstripped_static_bin = propgate_static_lib(ctx, static_lib, ".a", inputs)
outputs.append(unstripped_static_bin)
if ctx.attr.shared_archive:
unstripped_shared_archive = symlink_shared_archive(ctx, ".so", ".a")
outputs.append(unstripped_shared_archive)
if ctx.attr.shared_archive:
unstripped_shared_archive = symlink_shared_archive(ctx, ".so", ".a")
outputs.append(unstripped_shared_archive)
# The final program binary depends on the existence of the dependent dynamic library files. With
# build-without-the-bytes enabled, these aren't downloaded. Manually collect them and add them to the
# output set.
dynamic_deps_runfiles = ctx.runfiles(files = [])
if ctx.attr.type == "program":
outputs.extend(get_transitive_dyn_libs(ctx.attr.deps))
dynamic_deps = get_transitive_dyn_libs(ctx.attr.deps)
dynamic_deps_runfiles = ctx.attr.binary_with_debug[DefaultInfo].data_runfiles.merge(ctx.runfiles(files = get_transitive_dyn_libs(ctx.attr.deps)))
outputs.extend(dynamic_deps)
provided_info = [
DefaultInfo(
files = depset(outputs),
files = depset(outputs, transitive = [depset(get_transitive_debug_files(ctx.attr.deps))]),
runfiles = dynamic_deps_runfiles,
executable = output_bin if ctx.attr.type == "program" else None,
),
create_new_ccinfo_library(ctx, cc_toolchain, output_bin, unstripped_static_bin, ctx.attr.cc_shared_library),
]
if ctx.attr.type == "program":
provided_info += [
RunEnvironmentInfo(
environment = ctx.attr.binary_with_debug[RunEnvironmentInfo].environment,
inherited_environment = ctx.attr.binary_with_debug[RunEnvironmentInfo].inherited_environment,
),
]
provided_info += [ctx.attr.binary_with_debug[DebugPackageInfo]]
if ctx.attr.cc_shared_library != None:
provided_info.append(
create_new_cc_shared_library_info(ctx, cc_toolchain, output_bin, ctx.attr.cc_shared_library[CcSharedLibraryInfo], static_lib),
@ -270,7 +376,9 @@ def linux_extraction(ctx, cc_toolchain, inputs):
def macos_extraction(ctx, cc_toolchain, inputs):
outputs = []
input_bin, output_bin, debug_info, static_lib = get_inputs_and_outputs(ctx, ".dylib", ".a", ".dSYM")
unstripped_static_bin = None
input_bin, output_bin, debug_info, static_lib = get_inputs_and_outputs(ctx, ".dylib", ".a", MAC_DEBUG_FOLDER_EXTENSION)
input_file = ctx.attr.binary_with_debug.files.to_list()
if input_bin:
if ctx.attr.enabled:
@ -308,29 +416,41 @@ def macos_extraction(ctx, cc_toolchain, inputs):
)
outputs += [output_bin]
unstripped_static_bin = None
if static_lib:
unstripped_static_bin = propgate_static_lib(ctx, static_lib, ".a", inputs)
outputs.append(unstripped_static_bin)
if len(input_file):
if static_lib:
unstripped_static_bin = propgate_static_lib(ctx, static_lib, ".a", inputs)
outputs.append(unstripped_static_bin)
if ctx.attr.shared_archive:
unstripped_shared_archive = symlink_shared_archive(ctx, ".dylib", ".a")
outputs.append(unstripped_shared_archive)
if ctx.attr.shared_archive:
unstripped_shared_archive = symlink_shared_archive(ctx, ".dylib", ".a")
outputs.append(unstripped_shared_archive)
# The final program binary depends on the existence of the dependent dynamic library files. With
# build-without-the-bytes enabled, these aren't downloaded. Manually collect them and add them to the
# output set.
dynamic_deps_runfiles = ctx.runfiles(files = [])
if ctx.attr.type == "program":
outputs.extend(get_transitive_dyn_libs(ctx.attr.deps))
dynamic_deps = get_transitive_dyn_libs(ctx.attr.deps)
dynamic_deps_runfiles = ctx.attr.binary_with_debug[DefaultInfo].data_runfiles.merge(ctx.runfiles(files = get_transitive_dyn_libs(ctx.attr.deps)))
outputs.extend(dynamic_deps)
provided_info = [
DefaultInfo(
files = depset(outputs),
executable = output_bin if ctx.attr.type == "program" else None,
runfiles = ctx.attr.binary_with_debug[DefaultInfo].data_runfiles,
),
create_new_ccinfo_library(ctx, cc_toolchain, output_bin, unstripped_static_bin, ctx.attr.cc_shared_library),
]
if ctx.attr.type == "program":
provided_info += [
RunEnvironmentInfo(
environment = ctx.attr.binary_with_debug[RunEnvironmentInfo].environment,
inherited_environment = ctx.attr.binary_with_debug[RunEnvironmentInfo].inherited_environment,
),
]
if ctx.attr.cc_shared_library != None:
provided_info.append(
create_new_cc_shared_library_info(ctx, cc_toolchain, output_bin, ctx.attr.cc_shared_library[CcSharedLibraryInfo]),
@ -339,48 +459,91 @@ def macos_extraction(ctx, cc_toolchain, inputs):
return provided_info
def windows_extraction(ctx, cc_toolchain, inputs):
pdb = None
if ctx.attr.type == "library":
ext = ".lib"
if ctx.attr.cc_shared_library and ctx.attr.enable_pdb:
pdb = ctx.attr.cc_shared_library[OutputGroupInfo].pdb_file
elif ctx.attr.type == "program":
ext = ".exe"
if ctx.attr.enable_pdb:
pdb = ctx.attr.binary_with_debug[OutputGroupInfo].pdb_file
else:
fail("Can't extract debug info from unknown type: " + ctx.attr.type)
basename = ctx.attr.binary_with_debug.files.to_list()[0].basename[:-len(WITH_DEBUG_SUFFIX + ext)]
output = ctx.actions.declare_file(basename + ext)
input_file = ctx.attr.binary_with_debug.files.to_list()
outputs = []
output_library = None
output_dynamic_library = None
for input in ctx.attr.binary_with_debug.files.to_list():
ext = "." + input.extension
basename = input.basename[:-len(WITH_DEBUG_SUFFIX + ext)]
if len(input_file):
basename = ctx.attr.binary_with_debug.files.to_list()[0].basename[:-len(WITH_DEBUG_SUFFIX + ext)]
output = ctx.actions.declare_file(basename + ext)
outputs.append(output)
if ext == ".lib":
output_library = output
if ext == ".dll":
output_dynamic_library = output
for input in ctx.attr.binary_with_debug.files.to_list():
ext = "." + input.extension
ctx.actions.symlink(
output = output,
target_file = input,
)
basename = input.basename[:-len(WITH_DEBUG_SUFFIX + ext)]
output = ctx.actions.declare_file(basename + ext)
outputs.append(output)
if ctx.attr.shared_archive:
unstripped_shared_archive = symlink_shared_archive(ctx, ".dll", ".lib")
outputs.append(unstripped_shared_archive)
if ext == ".lib":
output_library = output
ctx.actions.symlink(
output = output,
target_file = input,
)
if ctx.attr.cc_shared_library != None:
for file in ctx.attr.cc_shared_library.files.to_list():
if file.path.endswith(".dll"):
basename = file.basename[:-len(WITH_DEBUG_SUFFIX + CC_SHARED_LIBRARY_SUFFIX + ".dll")]
output = ctx.actions.declare_file(basename + ".dll")
outputs.append(output)
output_dynamic_library = output
ctx.actions.symlink(
output = output,
target_file = file,
)
if pdb:
if ctx.attr.cc_shared_library != None:
basename = input.basename[:-len(WITH_DEBUG_SUFFIX + ".pdb")]
pdb_output = ctx.actions.declare_file(basename + ".dll.pdb")
else:
basename = input.basename[:-len(WITH_DEBUG_SUFFIX + ext)]
pdb_output = ctx.actions.declare_file(basename + ".pdb")
outputs.append(pdb_output)
ctx.actions.symlink(
output = pdb_output,
target_file = pdb.to_list()[0],
)
if ctx.attr.shared_archive:
unstripped_shared_archive = symlink_shared_archive(ctx, ".dll", ".lib")
outputs.append(unstripped_shared_archive)
provided_info = [
DefaultInfo(
files = depset(outputs),
executable = output if ctx.attr.type == "program" else None,
runfiles = ctx.attr.binary_with_debug[DefaultInfo].data_runfiles,
),
create_new_ccinfo_library(ctx, cc_toolchain, output_dynamic_library, output_library, ctx.attr.cc_shared_library),
]
if ctx.attr.type == "program":
provided_info += [
RunEnvironmentInfo(
environment = ctx.attr.binary_with_debug[RunEnvironmentInfo].environment,
inherited_environment = ctx.attr.binary_with_debug[RunEnvironmentInfo].inherited_environment,
),
]
if ctx.attr.cc_shared_library != None:
provided_info.append(
create_new_cc_shared_library_info(ctx, cc_toolchain, output_dynamic_library, ctx.attr.cc_shared_library[CcSharedLibraryInfo]),
@ -389,11 +552,6 @@ def windows_extraction(ctx, cc_toolchain, inputs):
return provided_info
def extract_debuginfo_impl(ctx):
# some cases (header file groups) there is no input files to do
# anything with, besides forward things along.
if not ctx.attr.binary_with_debug.files.to_list():
return noop_extraction(ctx)
cc_toolchain = find_cpp_toolchain(ctx)
inputs = depset(transitive = [
ctx.attr.binary_with_debug.files,
@ -406,11 +564,24 @@ def extract_debuginfo_impl(ctx):
windows_constraint = ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]
if ctx.target_platform_has_constraint(linux_constraint):
return linux_extraction(ctx, cc_toolchain, inputs)
# When skipping the archives we have to skip modifying debug info
# for the intermediates because we end up taking a dependency
# on the _with_debug .a files
if ctx.attr.skip_archive and ctx.attr.cc_shared_library == None:
return_info = [ctx.attr.binary_with_debug[CcInfo]]
else:
return_info = linux_extraction(ctx, cc_toolchain, inputs)
elif ctx.target_platform_has_constraint(macos_constraint):
return macos_extraction(ctx, cc_toolchain, inputs)
if ctx.attr.skip_archive and ctx.attr.cc_shared_library == None:
return_info = [ctx.attr.binary_with_debug[CcInfo]]
else:
return_info = macos_extraction(ctx, cc_toolchain, inputs)
elif ctx.target_platform_has_constraint(windows_constraint):
return windows_extraction(ctx, cc_toolchain, inputs)
return_info = windows_extraction(ctx, cc_toolchain, inputs)
tag_provider = TagInfo(tags = ctx.attr.tags)
return_info.append(tag_provider)
return return_info
extract_debuginfo = rule(
extract_debuginfo_impl,
@ -423,6 +594,7 @@ extract_debuginfo = rule(
doc = "Set to either 'library' or 'program' to discern how to extract the info.",
),
"enabled": attr.bool(default = False, doc = "Flag to enable/disable separate debug generation."),
"enable_pdb": attr.bool(default = False, doc = "Flag to enable pdb outputs on windows."),
"deps": attr.label_list(providers = [CcInfo]),
"cc_shared_library": attr.label(
doc = "If extracting from a shared library, the target of the cc_shared_library. Otherwise empty.",
@ -432,6 +604,7 @@ extract_debuginfo = rule(
doc = "If generating a shared archive(.so.a/.dll.lib), the shared archive's cc_library. Otherwise empty.",
allow_files = True,
),
"skip_archive": attr.bool(default = False, doc = "Flag to skip generating archives."),
"_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),
"_linux_constraint": attr.label(default = "@platforms//os:linux"),
"_macos_constraint": attr.label(default = "@platforms//os:macos"),
@ -453,6 +626,7 @@ extract_debuginfo_binary = rule(
doc = "Set to either 'library' or 'program' to discern how to extract the info.",
),
"enabled": attr.bool(default = False, doc = "Flag to enable/disable separate debug generation."),
"enable_pdb": attr.bool(default = False, doc = "Flag to enable pdb outputs on windows."),
"deps": attr.label_list(providers = [CcInfo]),
"cc_shared_library": attr.label(
doc = "If extracting from a shared library, the target of the cc_shared_library. Otherwise empty.",
@ -462,6 +636,7 @@ extract_debuginfo_binary = rule(
doc = "If generating a shared archive(.so.a/.dll.lib), the shared archive's cc_library. Otherwise empty.",
allow_files = True,
),
"skip_archive": attr.bool(default = False, doc = "Flag to skip generating archives."),
"_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),
"_linux_constraint": attr.label(default = "@platforms//os:linux"),
"_macos_constraint": attr.label(default = "@platforms//os:macos"),
@ -472,3 +647,37 @@ extract_debuginfo_binary = rule(
fragments = ["cpp"],
executable = True,
)
extract_debuginfo_test = rule(
extract_debuginfo_impl,
attrs = {
"binary_with_debug": attr.label(
doc = "The the binary to extract debuginfo from.",
allow_files = True,
),
"type": attr.string(
doc = "Set to either 'library' or 'program' to discern how to extract the info.",
),
"enabled": attr.bool(default = False, doc = "Flag to enable/disable separate debug generation."),
"enable_pdb": attr.bool(default = False, doc = "Flag to enable pdb outputs on windows."),
"deps": attr.label_list(providers = [CcInfo]),
"cc_shared_library": attr.label(
doc = "If extracting from a shared library, the target of the cc_shared_library. Otherwise empty.",
allow_files = True,
),
"shared_archive": attr.label(
doc = "If generating a shared archive(.so.a/.dll.lib), the shared archive's cc_library. Otherwise empty.",
allow_files = True,
),
"skip_archive": attr.bool(default = False, doc = "Flag to skip generating archives."),
"_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),
"_linux_constraint": attr.label(default = "@platforms//os:linux"),
"_macos_constraint": attr.label(default = "@platforms//os:macos"),
"_windows_constraint": attr.label(default = "@platforms//os:windows"),
},
doc = "Extract debuginfo into a separate file",
toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
fragments = ["cpp"],
executable = True,
test = True,
)

0
bazel/toolchains/cc/BUILD.bazel Normal file
View File

0
bazel/toolchains/cc/mongo_apple/BUILD.bazel Normal file
View File

86
bazel/toolchains/cc/mongo_apple/BUILD.tmpl Normal file
View File

@ -0,0 +1,86 @@
package(default_visibility = ["//visibility:public"])
load("@//bazel/config:configs.bzl", "sdkroot")
load("@//bazel/toolchains/cc/mongo_apple:mongo_apple_toolchain.bzl", "get_supported_apple_archs")
load("@//bazel/toolchains/cc/mongo_apple:mongo_apple_llvm_cc_toolchain_config.bzl", "mongo_apple_llvm_cc_toolchain_config")
load(
"@//bazel/toolchains/cc:mongo_custom_features.bzl",
"FEATURES_ATTR_NAMES",
"get_common_features_attrs")
# Helper target for the toolchain (see below):
filegroup(
name = "all_files",
srcs = glob(["**/*"]),
)
[
cc_toolchain(
name = "cc-compiler-" + arch,
all_files = ":all_files",
ar_files = ":all_files",
as_files = ":all_files",
compiler_files = ":all_files",
dwp_files = ":empty",
linker_files = ":all_files",
objcopy_files = ":empty",
strip_files = ":all_files",
supports_header_parsing = 1,
supports_param_files = 1,
toolchain_config = "llvm_" + arch,
toolchain_identifier = "apple_llvm_clang_toolchain_" + arch,
)
for arch, cpu in get_supported_apple_archs().items()
]
feature_attrs = get_common_features_attrs()
[
mongo_apple_llvm_cc_toolchain_config(
name = "llvm_" + arch,
cpu = "darwin",
compiler = "clang",
toolchain_identifier = "apple_llvm_clang_toolchain_" + arch,
target_libc = "macosx",
abi_version = "darwin_" + arch,
abi_libc_version = "darwin_" + arch,
cxx_builtin_include_directories = [
%{cxx_builtin_include_directories}
],
tool_paths = {
"ar": "%{llvm_path}/bin/llvm-ar",
"cpp": "%{llvm_path}/bin/clang-cpp",
"ld": "%{lld_path}/bin/ld.lld",
"dwp": "%{llvm_path}/bin/llvm-dwp",
"gcc": "%{llvm_path}/bin/clang",
"g++": "%{llvm_path}/bin/clang++",
"gcov": "%{llvm_path}/bin/llvm-profdata",
"llvm-cov": "%{llvm_path}/bin/llvm-cov",
"llvm-profdata": "%{llvm_path}/bin/llvm-profdata",
"nm": "%{llvm_path}/bin/llvm-nm",
"objcopy": "%{llvm_path}/bin/llvm-objcopy",
"objdump": "%{llvm_path}/bin/llvm-objdump",
"strip": "%{llvm_path}/bin/llvm-strip",
},
builtin_sysroot = "@//bazel/config:sdkroot",
optimization_level = feature_attrs[FEATURES_ATTR_NAMES.OPT_LEVEL],
)
for arch, cpu in get_supported_apple_archs().items()
]
[
toolchain(
name = "mongo_apple_" + arch + "_toolchain",
exec_compatible_with = [
"@platforms//os:macos",
"@platforms//cpu:" + cpu,
],
target_compatible_with = [
"@platforms//os:macos",
"@platforms//cpu:" + cpu,
],
toolchain = "@mongo_apple_toolchain//:cc-compiler-" + arch,
toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
)
for arch, cpu in get_supported_apple_archs().items()
]

1725
bazel/toolchains/cc/mongo_apple/mongo_apple_llvm_cc_toolchain_config.bzl Normal file
View File

File diff suppressed because it is too large Load Diff

161
bazel/toolchains/cc/mongo_apple/mongo_apple_toolchain.bzl Normal file
View File

@ -0,0 +1,161 @@
load("@build_bazel_apple_support//configs:platforms.bzl", "APPLE_PLATFORMS_CONSTRAINTS")
def _get_llvm_info(repository_ctx, build_file):
llvm_version = repository_ctx.os.environ.get("LLVM_VERSION") or ""
if llvm_version == "":
error_message = """
The Apple LLVM Clang toolchain has not been defined. Please make sure
that LLVM_VERSION has been defined in //.bazelrc.local or //.bazelrc file."""
return False, "", "", error_message
brew_command = [
"/bin/bash",
"-c",
"brew --prefix llvm@{}".format(llvm_version),
]
result = repository_ctx.execute(brew_command)
if result.return_code != 0:
error_message = """
Unable to find the prefix LLVM path using brew command: {}. Please make
sure that you have installed the LLVM toolchain using Homebrew:
`brew install llvm@{} lld@{}`.
or update the LLVM_VERSION in the //.bazelrc file or //.bazelrc.local.""".format(" ".join(brew_command), llvm_version, llvm_version)
return False, "", "", error_message
llvm_path = result.stdout.strip()
# Find the real path to the LLVM installation as we need to include the LLVM
# lib and headers directories as part of built-in directories.
command = [
"/bin/bash",
"-c",
"readlink -f {}".format(llvm_path),
]
result = repository_ctx.execute(command)
if result.return_code != 0:
return False, "", "", "Failed to find the true LLVM path using command: {}".format(" ".join(command))
llvm_path = result.stdout.strip()
return True, llvm_path, llvm_version, ""
def _get_lld_info(repository_ctx, llvm_version):
error_message = """
Unable to find the lld path. Please make sure that you have installed the lld using Homebrew:
`brew install lld@{}`.""".format(llvm_version)
brew_command = [
"/bin/bash",
"-c",
"brew --prefix lld@{}".format(llvm_version),
]
result = repository_ctx.execute(brew_command)
if result.return_code != 0:
return False, "", error_message
lld_path = result.stdout.strip()
command = [
"/bin/bash",
"-c",
"readlink -f {}".format(lld_path),
]
result = repository_ctx.execute(command)
if result.return_code != 0:
return False, "", error_message
lld_path = result.stdout.strip()
return True, lld_path, ""
def _get_llvm_clang_include_dirs(repository_ctx, llvm_path):
include_dirs = [
"/Applications/",
"/Library",
]
user = repository_ctx.os.environ.get("USER")
if user:
include_dirs.extend([
"/Users/{}/Applications/".format(user),
"/Users/{}/Library/".format(user),
])
for include_dir in ["include", "lib"]:
include_dirs.append(llvm_path + "/" + include_dir)
ret_include_dirs = []
for path in include_dirs:
ret_include_dirs.append((" \"%s\"," % path))
return ret_include_dirs
def _configure_oss_clang_toolchain(repository_ctx):
build_file = "BUILD.bazel"
success, llvm_path, llvm_version, error = _get_llvm_info(repository_ctx, build_file)
if not success:
return False, error
success, lld_path, error = _get_lld_info(repository_ctx, llvm_version)
if not success:
return False, error
include_dirs = _get_llvm_clang_include_dirs(repository_ctx, llvm_path)
repository_ctx.report_progress("Generating Apple OSS LLVM Clang Toolchain build file")
build_template = Label("@//bazel/toolchains/cc/mongo_apple:BUILD.tmpl")
repository_ctx.template(
build_file,
build_template,
{
"%{llvm_path}": llvm_path,
"%{lld_path}": lld_path,
"%{cxx_builtin_include_directories}": "\n".join(include_dirs),
},
)
return True, ""
def _apple_llvm_clang_cc_autoconf_impl(repository_ctx):
"""Configures the Apple LLVM Clang toolchain."""
if repository_ctx.os.name.startswith("mac os"):
# No failure is shown to the user as the toolchain is still being worked on it.
success, error_msg = _configure_oss_clang_toolchain(repository_ctx)
if not success:
fail(error_msg)
else:
repository_ctx.file("BUILD", "# Apple OSS LLVM Clang autoconfiguration was disabled because you're not on macOS")
mongo_apple_brew_llvm_toolchain_config = repository_rule(
environ = [
"LLVM_PATH", # Force re-compute if the user changed the location of the LLVM toolchain
"LLVM_VERSION", # Force re-compute if the user changed the version of the LLVM toolchain
],
implementation = _apple_llvm_clang_cc_autoconf_impl,
configure = True,
local = True,
)
_ARCH_MAP = {
"aarch64": "@platforms//cpu:arm64",
"x86_64": "@platforms//cpu:x86_64",
"ppc64le": "@platforms//cpu:ppc",
"s390x": "@platforms//cpu:s390x",
}
def get_supported_apple_archs():
_APPLE_ARCHS = APPLE_PLATFORMS_CONSTRAINTS.keys()
supported_archs = {}
for arch in APPLE_PLATFORMS_CONSTRAINTS.keys():
if arch.startswith("darwin_"):
cpu = arch.replace("darwin_", "")
if cpu in ["x86_64", "arm64"]:
supported_archs[arch] = cpu
return supported_archs
def setup_mongo_apple_toolchain():
mongo_apple_brew_llvm_toolchain_config(
name = "mongo_apple_toolchain",
)
setup_mongo_apple_toolchain_extension = module_extension(
implementation = lambda ctx: setup_mongo_apple_toolchain(),
)

191
bazel/toolchains/cc/mongo_apple/mongo_custom_features.bzl Normal file
View File

@ -0,0 +1,191 @@
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
load(
"@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl",
"feature",
"flag_group",
"flag_set",
)
load(
"//bazel/toolchains/cc:mongo_custom_features.bzl",
"all_compile_actions",
"get_common_features",
)
load("//bazel/toolchains/cc/mongo_apple:mongo_defines.bzl", "DEFINES")
_OBJCPP_EXECUTABLE_ACTION_NAME = "objc++-executable"
_DYNAMIC_LINK_ACTIONS = [
ACTION_NAMES.cpp_link_dynamic_library,
ACTION_NAMES.cpp_link_executable,
ACTION_NAMES.cpp_link_nodeps_dynamic_library,
ACTION_NAMES.objc_executable,
_OBJCPP_EXECUTABLE_ACTION_NAME,
]
def get_apple_features(ctx):
""" get_features returns a list of toolchain features for apple platform.
The list of features that is returned is a combined list of
common features and apple specific features.
Args:
ctx: The toolchain context.
Returns:
list: The list of features.
"""
return get_common_features(ctx) + [
feature(
name = "mongo_preprocessor_defines",
enabled = True,
flag_sets = [
flag_set(
actions = [
ACTION_NAMES.preprocess_assemble,
ACTION_NAMES.c_compile,
ACTION_NAMES.cpp_compile,
ACTION_NAMES.cpp_header_parsing,
ACTION_NAMES.cpp_module_compile,
ACTION_NAMES.linkstamp_compile,
ACTION_NAMES.objc_compile,
ACTION_NAMES.objcpp_compile,
],
flag_groups = [
flag_group(
flags = [
"-D{}".format(preprocessor_define)
for preprocessor_define in DEFINES
],
),
],
),
],
),
feature(
name = "macos_general_warnings",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
# As of XCode 9, this flag must be present (it is not enabled by -Wall),
# in order to enforce that -mXXX-version-min=YYY will enforce that you
# don't use APIs from ZZZ.
"-Wunguarded-availability",
"-Wno-enum-constexpr-conversion",
],
),
],
),
],
),
# Enable sized deallocation support.
#
# Bazel doesn't allow for defining C++-only flags without a custom toolchain
# config. This is setup in the Linux toolchain, but currently there is no custom
# MacOS toolchain. Enabling warnings-as-errors will fail the build if this flag
# is passed to the compiler when building C code. Define it here on MacOS only
# to allow us to configure warnings-as-errors on Linux.
#
# TODO(SERVER-90183): Remove this once custom toolchain configuration is
# implemented on MacOS.
feature(
name = "macos_fsized_deallocation",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
"-fsized-deallocation",
],
),
],
),
],
),
feature(
name = "macos_general_link_flags",
enabled = True,
flag_sets = [
flag_set(
actions = _DYNAMIC_LINK_ACTIONS,
flag_groups = [
flag_group(
flags = [
"-lresolv",
],
),
],
),
],
),
feature(
name = "macos_mongo_frameworks",
enabled = True,
flag_sets = [
flag_set(
actions = _DYNAMIC_LINK_ACTIONS,
flag_groups = [
flag_group(
flags = [
"-framework",
"CoreFoundation",
"-framework",
"Security",
],
),
],
),
],
),
feature(
name = "macos_no_deduplicate",
enabled = ctx.attr.optimization_level == "O0",
flag_sets = [
flag_set(
actions = _DYNAMIC_LINK_ACTIONS,
flag_groups = [
flag_group(
flags = [
"-Wl,-no_deduplicate",
],
),
],
),
],
),
feature(
name = "macos_standard_c_and_c_plus_plus",
enabled = True,
flag_sets = [
flag_set(
actions = [
ACTION_NAMES.assemble,
ACTION_NAMES.preprocess_assemble,
ACTION_NAMES.linkstamp_compile,
ACTION_NAMES.cpp_compile,
ACTION_NAMES.cpp_header_parsing,
ACTION_NAMES.cpp_module_compile,
ACTION_NAMES.cpp_module_codegen,
ACTION_NAMES.lto_backend,
ACTION_NAMES.clif_match,
ACTION_NAMES.objcpp_compile,
],
flag_groups = [flag_group(flags = ["-std=c++20"])],
),
flag_set(
actions = [
ACTION_NAMES.c_compile,
ACTION_NAMES.objc_compile,
],
flag_groups = [flag_group(flags = ["-std=c17"])],
),
],
),
]

10
bazel/toolchains/cc/mongo_apple/mongo_defines.bzl Normal file
View File

@ -0,0 +1,10 @@
"""This module provides a list of defines that is passed in to compiling.
"""
visibility(["//bazel/toolchains/cc/mongo_apple"])
DEFINES = [
# TODO SERVER-54659 - ASIO depends on std::result_of which was removed
# in C++ 20. xcode15 does not have backwards compatibility.
"ASIO_HAS_STD_INVOKE_RESULT",
]

93
bazel/toolchains/cc/mongo_compiler_flags.bzl Normal file
View File

@ -0,0 +1,93 @@
"""This file contains compiler flags that is specific to C++ compiling and linking."""
load(
"//bazel/toolchains/cc/mongo_linux:mongo_compiler_flags.bzl",
"MONGO_LINUX_CC_COPTS",
"MONGO_LINUX_CC_LINKFLAGS",
)
load(
"//bazel/toolchains/cc/mongo_windows:mongo_compiler_flags.bzl",
"MONGO_WIN_CC_COPTS",
"MONGO_WIN_CC_LINKFLAGS",
"WINDOWS_MULTITHREAD_RUNTIME_COPTS",
)
# Only visible in the build system.
visibility([
"//bazel",
"//src/mongo/util",
])
# Used as both link flags and copts
# Suppress the function sanitizer check for third party libraries, because:
#
# - mongod (a C++ binary) links in WiredTiger (a C library)
# - If/when mongod--built under ubsan--fails, the sanitizer will by
# default analyze the failed execution for undefined behavior related to
# function pointer usage. See:
# https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html#available-checks
# - When this happens, the sanitizer will attempt to dynamically load to perform
# the analysis.
# - However, since WT was built as a C library, is not linked with the function
# sanitizer library symbols despite its C++ dependencies referencing them.
# - This will cause the sanitizer itself to fail, resulting in debug information
# being unavailable.
# - So by suppressing the function ubsan check, we won't reference symbols
# defined in the unavailable ubsan function sanitier library and will get
# useful debugging information.
UBSAN_OPTS_THIRD_PARTY = select({
"//bazel/config:sanitize_undefined_dynamic_link_settings": [
"-fno-sanitize=function",
],
"//conditions:default": [],
})
def force_includes_copt(package_name, name):
if package_name.startswith("src/mongo"):
basic_h = "mongo/platform/basic.h"
return select({
"@platforms//os:windows": ["/FI" + basic_h],
"//conditions:default": ["-include", basic_h],
})
if name in ["scripting", "scripting_mozjs_test", "encrypted_dbclient"]:
return select({
"//bazel/config:linux_aarch64": ["-include", "third_party/mozjs/platform/aarch64/linux/build/js-config.h"],
"//bazel/config:linux_ppc64le": ["-include", "third_party/mozjs/platform/ppc64le/linux/build/js-config.h"],
"//bazel/config:linux_s390x": ["-include", "third_party/mozjs/platform/s390x/linux/build/js-config.h"],
"//bazel/config:linux_x86_64": ["-include", "third_party/mozjs/platform/x86_64/linux/build/js-config.h"],
"//bazel/config:macos_aarch64": ["-include", "third_party/mozjs/platform/aarch64/macOS/build/js-config.h"],
"//bazel/config:macos_x86_64": ["-include", "third_party/mozjs/platform/x86_64/macOS/build/js-config.h"],
"//bazel/config:windows_x86_64": ["/FI" + "third_party/mozjs/platform/x86_64/windows/build/js-config.h"],
})
return []
# TODO(SERVER-103006): Stop including this flag when ASP is able to upgrade mongoc and mongocxx
STREAMS_THIRD_PARTY_DIR = "src/mongo/db/modules/enterprise/src/streams/third_party"
def package_specific_copt(package_name):
if package_name.startswith("src/third_party") or package_name.startswith(STREAMS_THIRD_PARTY_DIR):
return UBSAN_OPTS_THIRD_PARTY
return []
def package_specific_linkflag(package_name):
if package_name.startswith("src/third_party") or package_name.startswith(STREAMS_THIRD_PARTY_DIR):
return UBSAN_OPTS_THIRD_PARTY
return []
MONGO_GLOBAL_COPTS = MONGO_LINUX_CC_COPTS + MONGO_WIN_CC_COPTS
def get_copts(name, package_name, copts = [], skip_windows_crt_flags = False):
copts = MONGO_GLOBAL_COPTS + \
package_specific_copt(package_name) + \
copts + \
force_includes_copt(package_name, name)
if not skip_windows_crt_flags:
copts = copts + WINDOWS_MULTITHREAD_RUNTIME_COPTS
return copts
MONGO_GLOBAL_LINKFLAGS = MONGO_LINUX_CC_LINKFLAGS + MONGO_WIN_CC_LINKFLAGS
def get_linkopts(package_name, linkopts = []):
return MONGO_GLOBAL_LINKFLAGS + package_specific_linkflag(package_name) + linkopts

363
bazel/toolchains/cc/mongo_custom_features.bzl Normal file
View File

@ -0,0 +1,363 @@
""" This file contains customized features that is common to most platforms"""
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
load(
"@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl",
"feature",
"flag_group",
"flag_set",
)
# Features listed in this file is only visible to the toolchain cc.
#visibility([
# "//",
# "//bazel/toolchains/cc/mongo_linux",
# "//bazel/toolchains/cc/mongo_apple",
#])
COMPILERS = struct(
CLANG = "clang",
GCC = "gcc",
)
LINKERS = struct(
GOLD = "gold",
LLD = "lld",
MOLD = "mold",
)
all_c_compile_actions = [
ACTION_NAMES.assemble,
ACTION_NAMES.c_compile,
ACTION_NAMES.objc_compile,
ACTION_NAMES.preprocess_assemble,
]
all_cpp_compile_actions = [
ACTION_NAMES.clif_match,
ACTION_NAMES.cpp_compile,
ACTION_NAMES.cpp_header_parsing,
ACTION_NAMES.cpp_module_compile,
ACTION_NAMES.cpp_module_codegen,
ACTION_NAMES.linkstamp_compile,
ACTION_NAMES.objcpp_compile,
]
all_compile_actions = \
all_c_compile_actions + \
all_cpp_compile_actions + \
[
ACTION_NAMES.lto_backend,
]
FEATURES_ATTR_NAMES = struct(
OPT_LEVEL = "optimization_level",
)
def get_common_features_attrs():
""" get_common_features_attrs returns a map of attributes and their values.
The map of attributes settings and their values is returned which
is used by the toolchain config. Please make sure to refer the key
by the FEATURE_ATTR_NAMES.
Returns:
list: The map of attributes and their values.
"""
return {
FEATURES_ATTR_NAMES.OPT_LEVEL: select({
# This is opt=debug, not to be confused with (opt=on && dbg=on)
"@//bazel/config:gcc_or_clang_opt_debug": "Og",
"@//bazel/config:gcc_or_clang_opt_off": "O0",
"@//bazel/config:gcc_or_clang_opt_on": "O2",
"@//bazel/config:gcc_or_clang_opt_size": "Os",
"@//conditions:default": None,
}),
}
def get_common_features(ctx):
""" get_features returns a list of toolchain features.
The list of features that is returned is a common list
to most of the platforms that we currently support.
Args:
ctx: The toolchain context.
Returns:
list: The list of features.
"""
return [
feature(
name = "og",
enabled = ctx.attr.optimization_level == "Og",
provides = [
"O0",
"O2",
"Os",
],
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
"-Og",
],
),
],
),
],
),
feature(
name = "o0",
enabled = ctx.attr.optimization_level == "O0",
provides = [
"Og",
"O2",
"Os",
],
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
"-O0",
],
),
],
),
],
),
feature(
name = "o2",
enabled = ctx.attr.optimization_level == "O2",
provides = [
"Og",
"O1",
"Os",
],
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
"-O2",
],
),
],
),
],
),
feature(
name = "os",
enabled = ctx.attr.optimization_level == "Os",
provides = [
"Og",
"O1",
"O2",
],
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
"-Os",
],
),
],
),
],
),
feature(
name = "no_unused_function",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
# Clang likes to warn about unused functions, which seems a tad
# aggressive and breaks -Werror, which we want to be able to use.
"-Wno-unused-function",
],
),
],
),
],
),
feature(
name = "no_defaulted_function_deleted",
enabled = ctx.attr.compiler == COMPILERS.CLANG,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
# This warning was added in Apple clang version 11 and flags many
# explicitly defaulted move constructors and assignment operators for
# being implicitly deleted, which is not useful.
"-Wno-defaulted-function-deleted",
],
),
],
),
],
),
# -Wno-invalid-offsetof is only valid for C++ but not for C
feature(
name = "no_invalid_offsetof_warning",
enabled = False,
flag_sets = [
flag_set(
actions = all_cpp_compile_actions,
flag_groups = [flag_group(flags = ["-Wno-invalid-offsetof"])],
),
],
),
feature(
name = "no_unknown_pragmas",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# Do not warn on unknown pragmas.
"-Wno-unknown-pragmas",
])],
),
],
),
feature(
name = "no_builtin_macro_redefined",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# Replace compile timestamp-related macros for reproducible binaries with consistent hashes.
"-Wno-builtin-macro-redefined",
])],
),
],
),
feature(
name = "no_unused_local_typedefs",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# This warning was added in g++-4.8.
"-Wno-unused-local-typedefs",
])],
),
],
),
feature(
name = "no_unused_lambda_capture",
enabled = ctx.attr.compiler == COMPILERS.CLANG,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# This warning was added in clang-5 and flags many of our lambdas. Since
# it isn't actively harmful to capture unused variables we are
# suppressing for now with a plan to fix later.
"-Wno-unused-lambda-capture",
])],
),
],
),
feature(
name = "no_deprecated_declarations",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# Prevents warning about using deprecated features (such as auto_ptr in
# c++11) Using -Wno-error=deprecated-declarations does not seem to work
# on some compilers, including at least g++-4.6.
"-Wno-deprecated-declarations",
])],
),
],
),
feature(
name = "no_unused_const_variable",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# New in clang-3.4, trips up things mostly in third_party, but in a few
# places in the primary mongo sources as well.
"-Wno-unused-const-variable",
])],
),
],
),
feature(
name = "no_deprecate_this_capture",
enabled = ctx.attr.compiler == COMPILERS.CLANG,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
"-Wno-deprecated-this-capture",
])],
),
],
),
feature(
name = "no_undefined_var_template",
enabled = ctx.attr.compiler == COMPILERS.CLANG,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# Disable warning about templates that can't be implicitly instantiated.
# It is an attempt to make a link error into an easier-to-debug compiler
# failure, but it triggers false positives if explicit instantiation is
# used in a TU that can see the full definition. This is a problem at
# least for the S2 headers.
"-Wno-undefined-var-template",
])],
),
],
),
feature(
name = "no_unused_private_field",
enabled = ctx.attr.compiler == COMPILERS.CLANG,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# Clang likes to warn about unused private fields, but some of our
# third_party libraries have such things.
"-Wno-unused-private-field",
])],
),
],
),
feature(
name = "no_potentially_evaluated_expression",
enabled = ctx.attr.compiler == COMPILERS.CLANG,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [flag_group(flags = [
# Don't issue warnings about potentially evaluated expressions
"-Wno-potentially-evaluated-expression",
])],
),
],
),
]

114
bazel/toolchains/cc/mongo_defines.bzl Normal file
View File

@ -0,0 +1,114 @@
"""This module provides a list of defines that is passed in to compiling that is agnostic to the OS.
"""
load(
"//bazel/toolchains/cc:mongo_errors.bzl",
"GLIBCXX_DEBUG_ERROR_MESSAGE",
"LIBCXX_ERROR_MESSAGE",
"SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE",
"THREAD_SANITIZER_ERROR_MESSAGE",
)
load("//bazel/toolchains/cc/mongo_windows:mongo_defines.bzl", "WINDOWS_DEFINES")
# Defines are only visible to within //bazel directory where
# toolchains and rules are defined.
# TODO: define mongo_generate_config_header rule to hide
# all the compiler options.
visibility([
"//src/mongo/util",
"//bazel",
])
ABSEIL_DEFINES = [
"ABSL_FORCE_ALIGNED_ACCESS",
]
BOOST_DEFINES = [
"BOOST_ENABLE_ASSERT_DEBUG_HANDLER",
# TODO: Ideally, we could not set this define in C++20 builds, but at least
# our current Xcode 12 doesn't offer std::atomic_ref, so we cannot.
"BOOST_FILESYSTEM_NO_CXX20_ATOMIC_REF",
"BOOST_LOG_NO_SHORTHAND_NAMES",
"BOOST_LOG_USE_NATIVE_SYSLOG",
"BOOST_LOG_WITHOUT_THREAD_ATTR",
"BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS",
"BOOST_SYSTEM_NO_DEPRECATED",
"BOOST_THREAD_USES_DATETIME",
"BOOST_THREAD_VERSION=5",
] + select({
"//bazel/config:linkdynamic_not_shared_archive": ["BOOST_LOG_DYN_LINK"],
"//conditions:default": [],
})
ENTERPRISE_DEFINES = select({
"//bazel/config:build_enterprise_enabled": ["MONGO_ENTERPRISE_VERSION=1"],
"//conditions:default": [],
}) + select({
"//bazel/config:enterprise_feature_audit_enabled": ["MONGO_ENTERPRISE_AUDIT=1"],
"//conditions:default": [],
}) + select({
"//bazel/config:enterprise_feature_encryptdb_enabled": ["MONGO_ENTERPRISE_ENCRYPTDB=1"],
"//conditions:default": [],
})
# Fortify only possibly makes sense on POSIX systems, and we know that clang is
# not a valid combination:
# http://lists.llvm.org/pipermail/cfe-dev/2015-November/045852.html
GCC_OPT_DEFINES = select({
"//bazel/config:gcc_opt": ["_FORTIFY_SOURCE=2"],
"//conditions:default": [],
})
DEBUG_DEFINES = select({
"//bazel/config:dbg_enabled": [],
"//conditions:default": ["NDEBUG"],
})
PCRE2_DEFINES = ["PCRE2_STATIC"]
SAFEINT_DEFINES = ["SAFEINT_USE_INTRINSICS=0"]
# Unfortunately, abseil requires that we make these macros (this, and THREAD_
# and UNDEFINED_BEHAVIOR_ below) set, because apparently it is too hard to query
# the running compiler. We do this unconditionally because abseil is basically
# pervasive via the 'base' library.
ADDRESS_SANITIZER_DEFINES = select({
"//bazel/config:sanitize_address_required_settings": ["ADDRESS_SANITIZER"],
"//bazel/config:asan_disabled": [],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE)
THREAD_SANITIZER_DEFINES = select({
"//bazel/config:sanitize_thread_required_settings": ["THREAD_SANITIZER"],
"//bazel/config:tsan_disabled": [],
}, no_match_error = THREAD_SANITIZER_ERROR_MESSAGE)
UNDEFINED_SANITIZER_DEFINES = select({
"//bazel/config:ubsan_enabled": ["UNDEFINED_BEHAVIOR_SANITIZER"],
"//bazel/config:ubsan_disabled": [],
})
GLIBCXX_DEBUG_DEFINES = select({
("//bazel/config:use_glibcxx_debug_required_settings"): ["_GLIBCXX_DEBUG"],
("//bazel/config:use_glibcxx_debug_disabled"): [],
}, no_match_error = GLIBCXX_DEBUG_ERROR_MESSAGE)
TCMALLOC_DEFINES = select({
"//bazel/config:tcmalloc_google_enabled": ["ABSL_ALLOCATOR_NOTHROW"],
"//conditions:default": [],
})
MONGO_GLOBAL_DEFINES = (
WINDOWS_DEFINES +
DEBUG_DEFINES +
ADDRESS_SANITIZER_DEFINES +
THREAD_SANITIZER_DEFINES +
UNDEFINED_SANITIZER_DEFINES +
GLIBCXX_DEBUG_DEFINES +
TCMALLOC_DEFINES +
GCC_OPT_DEFINES +
BOOST_DEFINES +
ABSEIL_DEFINES +
PCRE2_DEFINES +
SAFEINT_DEFINES +
ENTERPRISE_DEFINES
)

77
bazel/toolchains/cc/mongo_errors.bzl Normal file
View File

@ -0,0 +1,77 @@
"""This file contains a list of error strings that is related to
all functionality under toolchains/cc.
"""
THREAD_SANITIZER_ERROR_MESSAGE = """
Error:
Build failed due to either -
- Cannot use libunwind with TSAN, please add
--use_libunwind=False to your compile flags or
- TSAN is only supported with dynamic link models, please add
--linkstatic=False to your compile flags.
"""
# TODO(SERVER-85340): Fix this error message when libc++ is re-added to the
# toolchain.
LIBCXX_ERROR_MESSAGE = """
Error:
libc++ is not currently supported in the mongo toolchain. Follow this ticket
to see when support is being added SERVER-85340 We currently only support
passing the libcxx config on macos for compatibility reasons.
libc++ requires these configuration: --compiler_type=clang
"""
#TODO SERVER-84714 add message about using the toolchain version of C++ libs
GLIBCXX_DEBUG_ERROR_MESSAGE = """
Error:
glibcxx_debug requires these configurations:
--dbg=True
--use_libcxx=False
"""
REQUIRED_SETTINGS_LIBUNWIND_ERROR_MESSAGE = """
Error:
libunwind=on is only supported on linux"
"""
SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE = """
Error:
fuzzer, address, and memory sanitizers require these configurations:
--allocator=system
"""
REQUIRED_SETTINGS_SANITIZER_ERROR_MESSAGE = """
Error:
any sanitizer requires these configurations:
--compiler_type=clang
--opt=on [OR] --opt=debug
"""
DETECT_ODR_VIOLATIONS_ERROR_MESSAGE = """
Error:
detect_odr_violations requires these configurations:
--opt=off
--linker=gold
"""
BAZELISK_CHECK_ERROR_MESSAGE = """
Error:
This repository must be built through bazelisk, please uninstall your current bazel
installation and then run:
python buildscripts/install_bazel.py
"""
DWP_ERROR_MESSAGE = """
Error:
to create dwp files you must use one of these configurations:
--compiler_type=clang
--fission=yes
--create_dwp=True
[OR]
--compiler_type=gcc
--fission=yes
--dwarf_version=4
--create_dwp=True
"""

5
bazel/toolchains/cc/mongo_linux/BUILD.bazel Normal file
View File

@ -0,0 +1,5 @@
py_binary(
name = "mongo_toolchain_version_generator",
srcs = ["mongo_toolchain_version_generator.py"],
visibility = ["//visibility:public"],
)

397
bazel/toolchains/cc/mongo_linux/mongo_compiler_flags.bzl Normal file
View File

@ -0,0 +1,397 @@
"""This file contains compiler flags that are specific to linux cc compiling and linking."""
load(
"//bazel/toolchains/cc:mongo_errors.bzl",
"BAZELISK_CHECK_ERROR_MESSAGE",
"DETECT_ODR_VIOLATIONS_ERROR_MESSAGE",
"LIBCXX_ERROR_MESSAGE",
"REQUIRED_SETTINGS_SANITIZER_ERROR_MESSAGE",
"SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE",
"THREAD_SANITIZER_ERROR_MESSAGE",
)
# Flags listed in this file is only visible to the bazel build system.
visibility("//bazel/toolchains/cc")
# SERVER-9761: Ensure early detection of missing symbols in dependent libraries
# at program startup. For non-release dynamic builds we disable this behavior in
# the interest of improved mongod startup times. Xcode15 removed bind_at_load
# functionality so we cannot have a selection for macosx here
# ld: warning: -bind_at_load is deprecated on macOS
# TODO: SERVER-90596 reenable loading at startup
BIND_AT_LOAD_LINKFLAGS = select({
"//bazel/config:linkstatic_enabled_linux": [
"-Wl,-z,now",
],
"//conditions:default": [],
})
IMPLICIT_FALLTHROUGH_COPTS = select({
"//bazel/config:compiler_type_clang": ["-Wimplicit-fallthrough"],
"//bazel/config:compiler_type_gcc": ["-Wimplicit-fallthrough=5"],
"//conditions:default": [],
})
# -fno-omit-frame-pointer should be added if any sanitizer flag is used by user
ANY_SANITIZER_AVAILABLE_COPTS = select({
"//bazel/config:any_sanitizer_required_setting": [
"-fno-omit-frame-pointer",
],
"//bazel/config:no_enabled_sanitizer": [],
}, no_match_error = REQUIRED_SETTINGS_SANITIZER_ERROR_MESSAGE)
ANY_SANITIZER_AVAILABLE_LINKFLAGS = select({
# Sanitizer libs may inject undefined refs (for hooks) at link time, but the
# symbols will be available at runtime via the compiler runtime lib.
"//bazel/config:any_sanitizer_required_setting": [
"-Wl,--allow-shlib-undefined",
],
"//bazel/config:no_enabled_sanitizer": [],
}, no_match_error = REQUIRED_SETTINGS_SANITIZER_ERROR_MESSAGE)
ANY_SANITIZER_GCC_LINKFLAGS = select({
# GCC's implementation of ASAN depends on libdl.
"//bazel/config:any_sanitizer_gcc": ["-ldl"],
"//conditions:default": [],
})
ADDRESS_SANITIZER_COPTS = select({
"//bazel/config:asan_disabled": [],
"//bazel/config:sanitize_address_required_settings": [
"-fsanitize=address",
"-fsanitize-blacklist=$(location //etc:asan_denylist_h)",
],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE)
ADDRESS_SANITIZER_LINKFLAGS = select({
"//bazel/config:asan_disabled": [],
"//bazel/config:sanitize_address_required_settings": ["-fsanitize=address"],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE)
# Makes it easier to debug memory failures at the cost of some perf:
# -fsanitize-memory-track-origins
MEMORY_SANITIZER_COPTS = select({
"//bazel/config:msan_disabled": [],
"//bazel/config:sanitize_memory_required_settings": [
"-fsanitize=memory",
"-fsanitize-memory-track-origins",
"-fsanitize-blacklist=$(location //etc:msan_denylist_h)",
],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE)
SANITIZE_WITHOUT_TSAN_LINKFLAGS = select({
"//bazel/config:sanitize_without_tsan": [
"-rtlib=compiler-rt",
"-unwindlib=libgcc",
],
"//conditions:default": [],
})
# Makes it easier to debug memory failures at the cost of some perf:
# -fsanitize-memory-track-origins
MEMORY_SANITIZER_LINKFLAGS = select({
"//bazel/config:sanitize_memory_required_settings": ["-fsanitize=memory"],
"//bazel/config:msan_disabled": [],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE)
# We can't include the fuzzer flag with the other sanitize flags. The libfuzzer
# library already has a main function, which will cause the dependencies check
# to fail
FUZZER_SANITIZER_COPTS = select({
"//bazel/config:sanitize_fuzzer_required_settings": [
"-fsanitize=fuzzer-no-link",
"-fprofile-instr-generate",
"-fcoverage-mapping",
],
"//bazel/config:fsan_disabled": [],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE + "fuzzer")
# These flags are needed to generate a coverage report
FUZZER_SANITIZER_LINKFLAGS = select({
"//bazel/config:sanitize_fuzzer_required_settings": [
"-fsanitize=fuzzer-no-link",
"-fprofile-instr-generate",
"-fcoverage-mapping",
"-nostdlib++",
"-lstdc++",
],
"//bazel/config:fsan_disabled": [],
}, no_match_error = SYSTEM_ALLOCATOR_SANITIZER_ERROR_MESSAGE + "fuzzer")
# Combines following two conditions -
# 1.
# TODO: SERVER-48622
#
# See https://github.com/google/sanitizers/issues/943 for why we disallow
# combining TSAN with libunwind. We could, aternatively, have added logic to
# automate the decision about whether to enable libunwind based on whether TSAN
# is enabled, but that logic is already complex, and it feels better to make it
# explicit that using TSAN means you won't get the benefits of libunwind.
#
# 2.
# We add suppressions based on the library file in etc/tsan.suppressions so the
# link-model needs to be dynamic.
THREAD_SANITIZER_COPTS = select({
"//bazel/config:sanitize_thread_required_settings": [
"-fsanitize=thread",
"-fsanitize-blacklist=$(location //etc:tsan_denylist_h)",
],
"//bazel/config:tsan_disabled": [],
}, no_match_error = THREAD_SANITIZER_ERROR_MESSAGE)
THREAD_SANITIZER_LINKFLAGS = select({
"//bazel/config:sanitize_thread_required_settings": ["-fsanitize=thread"],
"//bazel/config:tsan_disabled": [],
}, no_match_error = THREAD_SANITIZER_ERROR_MESSAGE)
# By default, undefined behavior sanitizer doesn't stop on the first error. Make
# it so. Newer versions of clang have renamed the flag. However, this flag
# cannot be included when using the fuzzer sanitizer if we want to suppress
# errors to uncover new ones.
# In dynamic builds, the `vptr` sanitizer check can require additional
# dependency edges. That is very inconvenient, because such builds can't use
# z,defs. The result is a very fragile link graph, where refactoring the link
# graph in one place can have surprising effects in others. Instead, we just
# disable the `vptr` sanitizer for dynamic builds. We tried some other
# approaches in SERVER-49798 of adding a new descriptor type, but that didn't
# address the fundamental issue that the correct link graph for a dynamic+ubsan
# build isn't the same as the correct link graph for a regular dynamic build.
UNDEFINED_SANITIZER_COPTS = select({
"//bazel/config:ubsan_enabled": ["-fsanitize=undefined"],
"//conditions:default": [],
}) + select({
"//bazel/config:sanitize_undefined_dynamic_link_settings": [
"-fno-sanitize=vptr",
],
"//conditions:default": [],
}) + select({
"//bazel/config:sanitize_undefined_without_fuzzer_settings": [
"-fno-sanitize-recover",
],
"//conditions:default": [],
}) + select({
"//bazel/config:ubsan_enabled": [
"-fsanitize-blacklist=$(location //etc:ubsan_denylist_h)",
],
"//conditions:default": [],
})
UNDEFINED_SANITIZER_LINKFLAGS = select({
"//bazel/config:ubsan_enabled": ["-fsanitize=undefined"],
"//conditions:default": [],
}) + select({
"//bazel/config:sanitize_undefined_dynamic_link_settings": [
"-fno-sanitize=vptr",
],
"//conditions:default": [],
})
DETECT_ODR_VIOLATIONS_LINKFLAGS = select({
"//bazel/config:detect_odr_violations_required_settings": [
"-Wl,--detect-odr-violations",
],
"//bazel/config:detect_odr_violations_disabled": [],
}, no_match_error = DETECT_ODR_VIOLATIONS_ERROR_MESSAGE)
# TODO(SERVER-101099): Remove this once builds are containerized and system libraries inside the containers
# no longer contain debug symbols.
#
# In RHEL8 and RHEL9 the debug symbols for libgcc aren't stripped and are instead split, which still leaves behind
# debug symbols in the libgcc shared object file. These debug symbols are created with gdwarf32, so they're limited to
# a 32 bit address space. Even if the mongodb binaries are compiled with gdwarf64, there's a chance that the gdwarf32
# libgcc debug symbols will be placed after the gdwarf64 debug symbols. This started happening in the RHEL9 ppc64le
# build.
#
# The workaround for this is stripping the debug symbols from libgcc and statically compiling the libgcc from the
# toolchain into the mongodb binaries. The longer term solution for this is to containerize the non-remote-execution
# build and strip the debug symbols inside the container, or patch the compilers to properly order gdwarf32 symbols
# before gdwarf64 symbols. See https://reviews.llvm.org/D96144
LIBGCC_LINKFLAGS = select({
"//bazel/config:rhel9_ppc64le_gcc_linkstatic": ["-static-libgcc"],
"//conditions:default": [],
})
DEBUG_TYPES_SECTION_FLAGS = select({
"//bazel/config:linux_clang_linkstatic": [
"-fdebug-types-section",
],
# SUSE15 gcc builds system libraries with dwarf32 and needs -fdebug-types-section to keep
# the size of the debug information under the 4GB limit.
"//bazel/config:suse15_gcc_linkstatic": [
"-fdebug-types-section",
],
"//conditions:default": [],
})
COMPRESS_DEBUG_COPTS = select({
# Debug compression significantly reduces .o, .dwo, and .a sizes
"//bazel/config:compress_debug_compile_enabled": [
"-Wa,--compress-debug-sections",
],
# explicitly disable compression if its not enabled or else not passing the flag
# by default still compresses on x86/x86_64 - nocompress is only a flag in gcc not clang
"//bazel/config:compress_debug_compile_disabled_linux_gcc": [
"-Wa,--nocompress-debug-sections",
],
"//conditions:default": [],
})
DISABLE_SOURCE_WARNING_AS_ERRORS_COPTS = select({
"//bazel/config:disable_warnings_as_errors_linux": ["-Werror"],
# TODO(SERVER-90183): Enable once MacOS has a custom Bazel toolchain config.
# "//bazel/config:disable_warnings_as_errors_macos": ["-Werror"],
"//bazel/config:disable_warnings_as_errors_windows": ["/WX"],
"//bazel/config:warnings_as_errors_disabled": [],
"//conditions:default": [],
})
DISABLE_SOURCE_WARNING_AS_ERRORS_LINKFLAGS = select({
"//bazel/config:disable_warnings_as_errors_linux": ["-Wl,--fatal-warnings"],
"//bazel/config:warnings_as_errors_disabled": [],
"//conditions:default": [],
})
MTUNE_MARCH_COPTS = select({
"//bazel/config:linux_aarch64": [
"-march=armv8.2-a",
"-mtune=generic",
],
"//bazel/config:linux_ppc64le": [
"-mcpu=power8",
"-mtune=power8",
"-mcmodel=medium",
],
"//bazel/config:linux_s390x": [
"-march=z196",
"-mtune=zEC12",
],
# If we are enabling vectorization in sandybridge mode, we'd rather not hit
# the 256 wide vector instructions because the heavy versions can cause
# clock speed reductions.
"//bazel/config:linux_x86_64": [
"-march=sandybridge",
"-mtune=generic",
"-mprefer-vector-width=128",
],
"//conditions:default": [],
})
THIN_LTO_FLAGS = select({
"//bazel/config:thin_lto_enabled": ["-flto=thin"],
"//conditions:default": [],
})
SYMBOL_ORDER_COPTS = select({
"//bazel/config:symbol_ordering_file_enabled": ["-ffunction-sections"],
"//bazel/config:symbol_ordering_file_enabled_al2023": ["-ffunction-sections"],
"//conditions:default": [],
})
SYMBOL_ORDER_LINKFLAGS = select({
"//bazel/config:symbol_ordering_file_enabled": [
"-Wl,--symbol-ordering-file=$(location //buildscripts:symbols.orderfile)",
"-Wl,--no-warn-symbol-ordering",
],
"//bazel/config:symbol_ordering_file_enabled_al2023": [
"-Wl,--symbol-ordering-file=$(location //buildscripts:symbols-al2023.orderfile)",
"-Wl,--no-warn-symbol-ordering",
],
"//conditions:default": [],
})
SHARED_ARCHIVE_COPTS = select({
"//bazel/config:shared_archive_enabled_gcc": [
"-fno-gnu-unique",
],
"//conditions:default": [],
})
SHARED_ARCHIVE_LINKFLAGS_GNU_UNIQUE = select({
"//bazel/config:shared_archive_enabled_gcc_not_mold": [
"-Wl,--no-gnu-unique",
],
"//conditions:default": [],
})
SHARED_ARCHIVE_LINKFLAGS_B_SYMBOLIC = select({
"//bazel/config:shared_archive_enabled_gcc": [
"-Wl,-Bsymbolic",
],
"//conditions:default": [],
})
# Passed to both the compiler and linker
COVERAGE_FLAGS = select({
"//bazel/config:gcov_enabled": ["--coverage", "-fprofile-update=single"],
"//conditions:default": [],
})
# Passed to both the compiler and linker
PGO_PROFILE_FLAGS = select({
"//bazel/config:pgo_profile_enabled": [
"-fprofile-instr-generate",
],
"//conditions:default": [],
})
# Hack to throw an error if the user isn't running bazel through bazelisk,
# since we want to make sure the hook inside of tools/bazel gets run.
RUNNING_THROUGH_BAZELISK_CHECK = select({
"//bazel/config:running_through_bazelisk_x86_64_or_arm64": [],
"@platforms//cpu:s390x": [],
"@platforms//cpu:ppc": [],
}, no_match_error = BAZELISK_CHECK_ERROR_MESSAGE)
MONGO_GLOBAL_INCLUDE_DIRECTORIES = [
"-Isrc",
"-I$(GENDIR)/src",
"-I$(GENDIR)/src/mongo/db/modules/enterprise/src",
"-Isrc/mongo/db/modules/enterprise/src",
"-Isrc/third_party/valgrind/include",
]
MONGO_LINUX_CC_COPTS = (
MONGO_GLOBAL_INCLUDE_DIRECTORIES +
ADDRESS_SANITIZER_COPTS +
MEMORY_SANITIZER_COPTS +
FUZZER_SANITIZER_COPTS +
UNDEFINED_SANITIZER_COPTS +
THREAD_SANITIZER_COPTS +
ANY_SANITIZER_AVAILABLE_COPTS +
COMPRESS_DEBUG_COPTS +
DEBUG_TYPES_SECTION_FLAGS +
IMPLICIT_FALLTHROUGH_COPTS +
MTUNE_MARCH_COPTS +
DISABLE_SOURCE_WARNING_AS_ERRORS_COPTS +
THIN_LTO_FLAGS +
SYMBOL_ORDER_COPTS +
COVERAGE_FLAGS +
PGO_PROFILE_FLAGS +
SHARED_ARCHIVE_COPTS +
RUNNING_THROUGH_BAZELISK_CHECK
)
MONGO_LINUX_CC_LINKFLAGS = (
MEMORY_SANITIZER_LINKFLAGS +
ADDRESS_SANITIZER_LINKFLAGS +
FUZZER_SANITIZER_LINKFLAGS +
UNDEFINED_SANITIZER_LINKFLAGS +
THREAD_SANITIZER_LINKFLAGS +
DETECT_ODR_VIOLATIONS_LINKFLAGS +
BIND_AT_LOAD_LINKFLAGS +
ANY_SANITIZER_AVAILABLE_LINKFLAGS +
ANY_SANITIZER_GCC_LINKFLAGS +
DEBUG_TYPES_SECTION_FLAGS +
DISABLE_SOURCE_WARNING_AS_ERRORS_LINKFLAGS +
THIN_LTO_FLAGS +
SYMBOL_ORDER_LINKFLAGS +
COVERAGE_FLAGS +
PGO_PROFILE_FLAGS +
SANITIZE_WITHOUT_TSAN_LINKFLAGS +
SHARED_ARCHIVE_LINKFLAGS_GNU_UNIQUE +
SHARED_ARCHIVE_LINKFLAGS_B_SYMBOLIC +
LIBGCC_LINKFLAGS
)

24
bazel/toolchains/cc/mongo_linux/mongo_defines.bzl Normal file
View File

@ -0,0 +1,24 @@
"""This module provides a list of defines that is passed in to compiling specifically to linux builds.
"""
visibility(["//bazel/toolchains/cc/mongo_linux"])
DEFINES = [
# On linux, C code compiled with gcc/clang -std=c11 causes
# __STRICT_ANSI__ to be set, and that drops out all of the feature test
# definitions, resulting in confusing errors when we run C language
# configure checks and expect to be able to find newer POSIX things.
# Explicitly enabling _XOPEN_SOURCE fixes that, and should be mostly
# harmless as on Linux, these macros are cumulative. The C++ compiler
# already sets _XOPEN_SOURCE, and, notably, setting it again does not
# disable any other feature test macros, so this is safe to do. Other
# platforms like macOS and BSD have crazy rules, so don't try this
# there.
#
# Furthermore, as both C++ compilers appear to define _GNU_SOURCE
# unconditionally (because libstdc++ requires it), it seems prudent to
# explicitly add that too, so that C language checks see a consistent
# set of definitions.
"_XOPEN_SOURCE=700",
"_GNU_SOURCE",
]

Some files were not shown because too many files have changed in this diff Show More