From 309551f4eb237e22ae4f1a1423b2e6974e2af391 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Mon, 20 Apr 2026 10:37:21 -0400 Subject: [PATCH] SERVER-124473: Add bazel rules for running db-contrib-tool (#52092) GitOrigin-RevId: 2bfaaca0973363a1e8164de5ed9b33fdbc9db864 --- BUILD.bazel | 5 + WORKSPACE.bazel | 4 + bazel/db_contrib_tool/BUILD.bazel | 16 +++ bazel/db_contrib_tool/OWNERS.yml | 5 + bazel/db_contrib_tool/db_contrib_tool.bzl | 120 ++++++++++++++++++ bazel/db_contrib_tool/db_contrib_tool.sh | 13 ++ .../resmokelib/generate_fuzz_config/README.md | 2 +- .../e2e/mongot_testing_instructions.md | 32 ++--- 8 files changed, 176 insertions(+), 21 deletions(-) create mode 100644 bazel/db_contrib_tool/BUILD.bazel create mode 100644 bazel/db_contrib_tool/OWNERS.yml create mode 100644 bazel/db_contrib_tool/db_contrib_tool.bzl create mode 100755 bazel/db_contrib_tool/db_contrib_tool.sh diff --git a/BUILD.bazel b/BUILD.bazel index dc5214aa796..1351dfc0bb4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -553,6 +553,11 @@ filegroup( ]), ) +alias( + name = "db-contrib-tool", + actual = "//bazel/db_contrib_tool:db-contrib-tool", +) + # Resmoke config files generated by mongo-task-generator filegroup( name = "generated_resmoke_config", diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index dc19a4b4844..ac5d03a78e5 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -218,3 +218,7 @@ gpg() load("//bazel/mongot_extension_signing_key:mongot_extension_signing_key.bzl", "mongot_extension_signing_key") mongot_extension_signing_key() + +load("//bazel/db_contrib_tool:db_contrib_tool.bzl", "db_contrib_tool") + +db_contrib_tool() diff --git a/bazel/db_contrib_tool/BUILD.bazel b/bazel/db_contrib_tool/BUILD.bazel new file mode 100644 index 00000000000..32ac037b7da --- /dev/null +++ b/bazel/db_contrib_tool/BUILD.bazel @@ -0,0 +1,16 @@ +# Visibility is restricted to because db-contrib-tool downloads binaries +# from external sources at build time. +# Non-hermetic external downloads should not influence the core build graph. +package(default_visibility = [ + "//:__pkg__", +]) + +sh_binary( + name = "db-contrib-tool", + srcs = ["db_contrib_tool.sh"], + data = [ + "//bazel/resmoke:resmoke_mongo_version", + "//buildscripts:resmoke", + "@db_contrib_tool//:db-contrib-tool", + ], +) diff --git a/bazel/db_contrib_tool/OWNERS.yml b/bazel/db_contrib_tool/OWNERS.yml new file mode 100644 index 00000000000..889c04951ed --- /dev/null +++ b/bazel/db_contrib_tool/OWNERS.yml @@ -0,0 +1,5 @@ +version: 1.0.0 +filters: + - "*": + approvers: + - 10gen/devprod-correctness diff --git a/bazel/db_contrib_tool/db_contrib_tool.bzl b/bazel/db_contrib_tool/db_contrib_tool.bzl new file mode 100644 index 00000000000..f28605b1744 --- /dev/null +++ b/bazel/db_contrib_tool/db_contrib_tool.bzl @@ -0,0 +1,120 @@ +"""Repository rules for db-contrib-tool""" + +load("//bazel:utils.bzl", "retry_download") +load("@bazel_rules_mongo//utils:platforms_normalize.bzl", "ARCH_NORMALIZE_MAP", "OS_NORMALIZE_MAP") + +URLS_MAP = { + "linux_aarch64": { + "sha": "000189ea41fc498a9090d39cb1d0fbf426dfe6ba119dcd60cab802bcb261bd4d", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_linux_arm64.gz", + }, + "linux_x86_64": { + "sha": "c7dd52b4dc706f6ee6a2f553271f4f57a6013cf3914363802427aaf38732d2ec", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_linux_x64.gz", + }, + "linux_s390x": { + "sha": "de5e149c041b4f982b72579e499f47ac16bac6c7df6c5326d7d225e00c8a5a40", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_linux_s390x.gz", + }, + "rhel8_ppc64le": { + "sha": "0f6a380bd881d2423195d1338389d4ed305b6ad2fff6e773776f40896e4d58a8", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_rhel8_ppc64le.gz", + }, + "rhel9_ppc64le": { + "sha": "92ae51c9ee0b343fc6723e0f6b9d529a3f93e1f4f54042193b77c762b8911c4e", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_rhel9_ppc64le.gz", + }, + "macos_x86_64": { + "sha": "42dcc92c2914214783ddec659a157dcef0aadc1a03bd29730c69511d8ad84912", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_darwin_x64.gz", + }, + "windows_x86_64": { + "sha": "da881cf80ab10ae98ade5fd7ea43337b26b5f674fabdbd11c5d1644804a8f089", + "url": "https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.2.3/db-contrib-tool_v2.2.3_windows_x64.exe.gz", + }, +} + +def _get_python(ctx): + os_constraint = OS_NORMALIZE_MAP[ctx.os.name] + if os_constraint == "windows": + return ctx.path(Label("@py_host//:dist/python.exe")) + return ctx.path(Label("@py_host//:dist/bin/python3")) + +def _extract_gz_executable(ctx, src, dst): + """Extract a gzip-compressed file using the toolchain Python's gzip module, and mark the output as executable.""" + python = _get_python(ctx) + result = ctx.execute([ + python, + "-c", + "import gzip,shutil,sys,os; shutil.copyfileobj(gzip.open(sys.argv[1],'rb'),open(sys.argv[2],'wb')); os.chmod(sys.argv[2], 0o755)", + src, + dst, + ]) + if result.return_code != 0: + fail("Failed to extract {}: {}".format(src, result.stderr)) + +def _detect_rhel_major(ctx): + """Detect RHEL major version from the kernel release string (e.g. el8, el9).""" + result = ctx.execute(["uname", "-r"]) + if result.return_code != 0: + fail("db_contrib_tool: failed to detect RHEL major version: `uname -r` exited with {}: {}".format(result.return_code, result.stderr)) + for part in result.stdout.strip().replace("-", ".").split("."): + if part.startswith("el") and part[2:].isdigit(): + return str(min(int(part[2:]), 9)) + fail("db_contrib_tool: failed to detect RHEL major version from kernel release: {}".format(result.stdout.strip())) + +def _db_contrib_tool_download(ctx): + os = ctx.os.name + arch = ctx.os.arch + os_constraint = OS_NORMALIZE_MAP[os] + arch_constraint = ARCH_NORMALIZE_MAP[arch] + if arch_constraint == "ppc64le": + platform_key = "rhel{}_ppc64le".format(_detect_rhel_major(ctx)) + else: + platform_key = "{os}_{arch}".format(os = os_constraint, arch = arch_constraint) + if platform_key not in URLS_MAP: + fail("db_contrib_tool: unsupported platform: " + platform_key) + platform_info = URLS_MAP[platform_key] + ctx.report_progress("downloading db-contrib-tool") + retry_download( + ctx = ctx, + output = "db-contrib-tool.gz", + tries = 3, + url = platform_info["url"], + sha256 = platform_info["sha"], + ) + + _extract_gz_executable(ctx, "db-contrib-tool.gz", "db-contrib-tool-bin") + + ctx.file( + "BUILD.bazel", + """ +# Visibility restricted: db-contrib-tool downloads binaries from external sources. +# Non-hermetic external downloads should not influence the core build graph. +package(default_visibility = [ + "@//:__pkg__", + "@//bazel/db_contrib_tool:__pkg__", +]) +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") + +native_binary( + name = "db-contrib-tool", + src = "db-contrib-tool-bin", + data = [ + + ], + out = "db-contrib-tool", + env = {} +) +""", + ) + + return None + +_db_contrib_tool = repository_rule( + implementation = _db_contrib_tool_download, + attrs = {}, +) + +def db_contrib_tool(): + _db_contrib_tool(name = "db_contrib_tool") diff --git a/bazel/db_contrib_tool/db_contrib_tool.sh b/bazel/db_contrib_tool/db_contrib_tool.sh new file mode 100755 index 00000000000..fef28e5bd2a --- /dev/null +++ b/bazel/db_contrib_tool/db_contrib_tool.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -e + +runfiles_root="$(pwd)" +db_contrib_tool="${runfiles_root}/../db_contrib_tool/db-contrib-tool" + +# Change to the workspace root so that db-contrib-tool's relative-path defaults +# land inside the repo rather than the bazel runfiles directory. +if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then + cd "${BUILD_WORKSPACE_DIRECTORY}" +fi + +exec "${db_contrib_tool}" "$@" diff --git a/buildscripts/resmokelib/generate_fuzz_config/README.md b/buildscripts/resmokelib/generate_fuzz_config/README.md index 19454eada56..f1c600bbce1 100644 --- a/buildscripts/resmokelib/generate_fuzz_config/README.md +++ b/buildscripts/resmokelib/generate_fuzz_config/README.md @@ -116,7 +116,7 @@ The log line starting with "resmoke.py invocation for local usage" and the one w ## Running the config fuzzer locally -Before running the Resmoke config fuzzer command, you need to obtain the necessary binaries. You can download them from the "Files" section of the `archive_dist_test` task in Evergreen (e.g., binaries from the `amazon2-arm64-compile` variant). Alternatively, if you don't require those specific binaries, you can use `db-contrib-tool` to download the binaries (e.g., by running `db-contrib-tool setup-repro-env master`). +Before running the Resmoke config fuzzer command, you need to obtain the necessary binaries. You can download them from the "Files" section of the `archive_dist_test` task in Evergreen (e.g., binaries from the `amazon2-arm64-compile` variant). Alternatively, if you don't require those specific binaries, you can use `db-contrib-tool` to download the binaries (e.g., by running `bazel run db-contrib-tool -- setup-repro-env master`). To re-run a command locally that failed through the config fuzzer, you can navigate to the specific test that failed, and under files you can find a name titled "Resmoke.py Invocation for Local Usage". If you are replicating an older config fuzzer invocation, remove the command line argument "`--installDir=dist-test/bin`". A simple example command is shown below: diff --git a/jstests/with_mongot/e2e/mongot_testing_instructions.md b/jstests/with_mongot/e2e/mongot_testing_instructions.md index 5fb4c9c50c4..ac9aaaad5db 100644 --- a/jstests/with_mongot/e2e/mongot_testing_instructions.md +++ b/jstests/with_mongot/e2e/mongot_testing_instructions.md @@ -4,63 +4,55 @@ To run aggregation pipelines containing $search or $vectorSearch stages, you wil ## Using release or latest mongot -In order to acquire a release or latest mongot binary, from your ~/mongo directory you will need to: - -1. Make sure your [db-contrib-tool](https://github.com/10gen/db-contrib-tool/tree/main) is up-to-date. In order to do this, you will need to run: - -###### - - python3 -m pipx upgrade db-contrib-tool - -2. Know your virtual workstations OS and architecture. Assuming your VM is on ubuntu (the default), run `lscpu` in your terminal and inspect the first line of the response to confirm your VM's architecture. +In order to acquire a release or latest mongot binary, from your ~/mongo directory you will need to know your virtual workstations OS and architecture. Assuming your VM is on ubuntu (the default), run `lscpu` in your terminal and inspect the first line of the response to confirm your VM's architecture. The default behavior of setup-mongot-repro assume you want to download the latest version of mongot binary compatible with linux x86_64. In which case, if this works for your VM/testing needs, you can run: ###### - db-contrib-tool setup-mongot-repro-env --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env --installDir build/install/bin However, you can be more verbose and get the same result via: ###### - db-contrib-tool setup-mongot-repro-env --architecture x86_64 --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env --architecture x86_64 --installDir build/install/bin and ###### - db-contrib-tool setup-mongot-repro-env --architecture x86_64 --platform linux --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env --architecture x86_64 --platform linux --installDir build/install/bin and even ###### - db-contrib-tool setup-mongot-repro-env latest --architecture x86_64 --platform linux --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env latest --architecture x86_64 --platform linux --installDir build/install/bin To install the production mongot linux x86_64 binary, you should run: ###### - db-contrib-tool setup-mongot-repro-env release --architecture x86_64 --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env release --architecture x86_64 --installDir build/install/bin If your architecture is of type aarch64, to install the latest mongot binary, you should run: ###### - db-contrib-tool setup-mongot-repro-env --architecture aarch64 --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env --architecture aarch64 --installDir build/install/bin If your VM is running macos, you can install the latest macos compatible mongot binary via: ###### - db-contrib-tool setup-mongot-repro-env --platform macos --installDir build/install/bin + bazel run db-contrib-tool -- setup-mongot-repro-env --platform macos --installDir build/install/bin Clearly, many options to play around with! To learn more about setup-mongot-repro-env command line options, use ###### - db-contrib-tool setup-mongot-repro-env --help + bazel run db-contrib-tool -- setup-mongot-repro-env --help ## Compiling mongot from source @@ -138,19 +130,19 @@ The general format of the command is: ###### - db-contrib-tool setup-repro-env --variant + bazel run db-contrib-tool -- setup-repro-env --variant Specifically, to download from the `AL2023 x86 mongot integration tasks cron only` build variant, you could run: ###### - db-contrib-tool setup-repro-env --variant amazon-linux-2023-x86-mongot-integration-cron-only 23b790a2a81767b8edbbc266043a205029867b74 + bazel run db-contrib-tool -- setup-repro-env --variant amazon-linux-2023-x86-mongot-integration-cron-only 23b790a2a81767b8edbbc266043a205029867b74 By default, the download will be placed in `build/multiversion_bin//dist_test/`, but you can also specify a location via the `--installDir` option. For example: ###### - db-contrib-tool setup-repro-env --variant amazon-linux2023-arm64-static-compile 23b790a2a81767b8edbbc266043a205029867b74 --installDir=build/multiversion_bin/my_variant + bazel run db-contrib-tool -- setup-repro-env --variant amazon-linux2023-arm64-static-compile 23b790a2a81767b8edbbc266043a205029867b74 --installDir=build/multiversion_bin/my_variant Will place the mongot binary in `build/multiversion_bin/my_variant/23b790a2a81767b8edbbc266043a205029867b74/dist_test/bin/mongot-localdev`