mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-117897 Add mongot-extension to dist and dist-test (#47736)

Browse Source 
GitOrigin-RevId: 1304427ba869f2626adaddf419b4f8ce9cbe7b76
This commit is contained in:
Zac Codiamat 2026-02-06 16:24:27 -06:00 committed by MongoDB Bot
parent 98ff0c9b7f
commit 8e0ca8de69
12 changed files with 362 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
BUILD.bazel
View File

@ -226,6 +226,25 @@ mongo_install(
"//bazel/config:bolt_profile_use_enabled": ["//:bolt_optimized_mongod"],
"//conditions:default": ["//src/mongo/db:mongod"],
}),
include_files = select({
"//bazel/config:al2023_x86_64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-x86_64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-x86_64.so.sig": "lib/libmongot_extension.so.sig",
},
"//bazel/config:al2023_aarch64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-aarch64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-aarch64.so.sig": "lib/libmongot_extension.so.sig",
},
"//bazel/config:al2_x86_64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-x86_64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-x86_64.so.sig": "lib/libmongot_extension.so.sig",
},
"//bazel/config:al2_aarch64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-aarch64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-aarch64.so.sig": "lib/libmongot_extension.so.sig",
},
"//conditions:default": {},
}),
package_extract_name = select({
"//bazel/config:build_enterprise_linux_enabled": "mongodb-linux-{MONGO_ARCH}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
"//bazel/config:build_enterprise_linux_disabled": "mongodb-linux-{MONGO_ARCH}-{MONGO_DISTMOD}-{MONGO_VERSION}",
@ -304,6 +323,33 @@ mongo_install(
"//bazel/config:bolt_profile_use_enabled": ["//:bolt_optimized_mongod"],
"//conditions:default": ["//src/mongo/db:mongod"],
}),
include_files = select({
"//bazel/config:al2023_x86_64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-x86_64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-x86_64.so.sig": "lib/libmongot_extension.so.sig",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2023-x86_64.so": "lib/librerank_extension.so",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2023-x86_64.so.sig": "lib/librerank_extension.so.sig",
},
"//bazel/config:al2023_aarch64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-aarch64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2023-aarch64.so.sig": "lib/libmongot_extension.so.sig",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2023-aarch64.so": "lib/librerank_extension.so",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2023-aarch64.so.sig": "lib/librerank_extension.so.sig",
},
"//bazel/config:al2_x86_64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-x86_64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-x86_64.so.sig": "lib/libmongot_extension.so.sig",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2-x86_64.so": "lib/librerank_extension.so",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2-x86_64.so.sig": "lib/librerank_extension.so.sig",
},
"//bazel/config:al2_aarch64": {
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-aarch64.so": "lib/libmongot_extension.so",
"//src/mongo/db/extension/extensions:mongot_extension_amazon2-aarch64.so.sig": "lib/libmongot_extension.so.sig",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2-aarch64.so": "lib/librerank_extension.so",
"//src/mongo/db/extension/extensions:rerank_extension_amazon2-aarch64.so.sig": "lib/librerank_extension.so.sig",
},
"//conditions:default": {},
}),
pretty_printer_tests = {
"//src/mongo/util:pretty_printer_test.py": "//src/mongo/util:pretty_printer_test_program",
"//src/mongo/db/shard_role/lock_manager:lock_gdb_test.py": "//src/mongo/db:mongod",

40
bazel/config/BUILD.bazel
View File

@ -3122,3 +3122,43 @@ config_setting(
"//bazel/config:coverage": "True",
},
)
# ==========
# Extension variants
# ==========
selects.config_setting_group(
name = "al2023_x86_64",
match_all = [
"@platforms//os:linux",
"//bazel/platforms:amazon_linux_2023",
"@platforms//cpu:x86_64",
],
)
selects.config_setting_group(
name = "al2023_aarch64",
match_all = [
"@platforms//os:linux",
"//bazel/platforms:amazon_linux_2023",
"@platforms//cpu:aarch64",
],
)
selects.config_setting_group(
name = "al2_x86_64",
match_all = [
"@platforms//os:linux",
"//bazel/platforms:amazon_linux_2",
"@platforms//cpu:x86_64",
],
)
selects.config_setting_group(
name = "al2_aarch64",
match_all = [
"@platforms//os:linux",
"//bazel/platforms:amazon_linux_2",
"@platforms//cpu:aarch64",
],
)

57
bazel/config/python_genrule.bzl Normal file
View File

@ -0,0 +1,57 @@
load("//bazel:utils.bzl", "write_target")
def python_genrule_impl(ctx):
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
python_libs = [py_dep[PyInfo].transitive_sources for py_dep in ctx.attr.python_libs]
python_path = []
for py_dep in ctx.attr.python_libs:
for path in py_dep[PyInfo].imports.to_list():
if path not in python_path:
python_path.append(ctx.expand_make_variables("python_library_imports", "$(BINDIR)/external/" + path, ctx.var))
expanded_args = [
ctx.expand_make_variables("render_template_expand", ctx.expand_location(arg, ctx.attr.srcs), ctx.var)
for arg in ctx.attr.cmd
]
ctx.actions.run(
executable = python.interpreter.path,
outputs = ctx.outputs.outputs,
inputs = depset(transitive = [python.files, depset([arg.files.to_list()[0] for arg in ctx.attr.srcs])] + python_libs),
arguments = expanded_args,
env = {"PYTHONPATH": ctx.configuration.host_path_separator.join(python_path)},
mnemonic = "TemplateRenderer",
)
return [DefaultInfo(files = depset(ctx.outputs.outputs))]
python_genrule_rule = rule(
python_genrule_impl,
attrs = {
"srcs": attr.label_list(
doc = "The input files of this rule.",
allow_files = True,
),
"outputs": attr.output_list(
doc = "The outputs of this rule.",
mandatory = True,
),
"cmd": attr.string_list(
doc = "The command line arguments to pass to python",
),
"python_libs": attr.label_list(
providers = [PyInfo],
default = [],
),
},
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
output_to_genfiles = True,
)
def python_genrule(name, tags = [], **kwargs):
python_genrule_rule(
name = name,
tags = tags + ["gen_source"],
**kwargs
)

56
bazel/config/render_template.bzl
View File

@ -55,3 +55,59 @@ def render_template(name, tags = [], **kwargs):
tags = tags + ["gen_source"],
**kwargs
)
def render_templates_impl(ctx):
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
python_libs = [py_dep[PyInfo].transitive_sources for py_dep in ctx.attr.python_libs]
python_path = []
for py_dep in ctx.attr.python_libs:
for path in py_dep[PyInfo].imports.to_list():
if path not in python_path:
python_path.append(ctx.expand_make_variables("python_library_imports", "$(BINDIR)/external/" + path, ctx.var))
expanded_args = [
ctx.expand_make_variables("render_template_expand", ctx.expand_location(arg, ctx.attr.srcs), ctx.var)
for arg in ctx.attr.cmd
]
ctx.actions.run(
executable = python.interpreter.path,
outputs = ctx.outputs.outputs,
inputs = depset(transitive = [python.files, depset([arg.files.to_list()[0] for arg in ctx.attr.srcs])] + python_libs),
arguments = expanded_args,
env = {"PYTHONPATH": ctx.configuration.host_path_separator.join(python_path)},
mnemonic = "TemplateRenderer",
)
return [DefaultInfo(files = depset(ctx.outputs.outputs))]
render_templates_rule = rule(
render_templates_impl,
attrs = {
"srcs": attr.label_list(
doc = "The input files of this rule.",
allow_files = True,
),
"outputs": attr.output_list(
doc = "The outputs of this rule.",
mandatory = True,
),
"cmd": attr.string_list(
doc = "The command line arguments to pass to python",
),
"python_libs": attr.label_list(
providers = [PyInfo],
default = [],
),
},
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
output_to_genfiles = True,
)
def render_templates(name, tags = [], **kwargs):
render_templates_rule(
name = name,
tags = tags + ["gen_source"],
**kwargs
)

22
bazel/install_rules/install_rules.bzl
View File

@ -219,6 +219,7 @@ def mongo_install_rule_impl(ctx):
"dynamic_libs_debug": {},
"dynamic_libs": {},
"root_files": {},
"include_files": {},
}
test_files = []
outputs = []
@ -239,6 +240,10 @@ def mongo_install_rule_impl(ctx):
for file in input_label.files.to_list():
file_map["root_files"][file.path] = declare_output(ctx, install_dir + "/" + output_folder + "/" + file.basename, file.is_directory)
for input_label, output_path in ctx.attr.include_files.items():
file = input_label.files.to_list()[0]
file_map["include_files"][file.path] = declare_output(ctx, install_dir + "/" + output_path, False)
# sort dependency install files
for dep in ctx.attr.deps:
test_files.extend(dep[TestBinaryInfo].test_binaries.to_list())
@ -272,6 +277,7 @@ def mongo_install_rule_impl(ctx):
bins = [bin for bin in file_map["binaries"]] + [bin for bin in file_map["binaries_debug"]]
libs = [lib for lib in file_map["dynamic_libs"]] + [lib for lib in file_map["dynamic_libs_debug"]]
root_files = [root_file for root_file in file_map["root_files"]]
include_files = [include_file for include_file in file_map["include_files"]]
unittest_bin = None
if len(bins) == 1 and ctx.attr.debug != "debug" and file_map["binaries"][bins[0]].basename.endswith("_test") and len(root_files) == 0:
@ -312,8 +318,16 @@ def mongo_install_rule_impl(ctx):
folder_index_start = path.find(install_dir) + len(install_dir) + 1
folder_index_end = path.rfind("/")
roots[file] = path[folder_index_start:folder_index_end]
includes = {}
for file in include_files:
path = file_map["include_files"][file].short_path
folder_index_start = path.find(install_dir) + len(install_dir) + 1
includes[file] = path[folder_index_start:]
json_out = struct(
roots = roots,
includes = includes,
bins = bins,
libs = libs,
)
@ -337,6 +351,9 @@ def mongo_install_rule_impl(ctx):
for root_file in root_files:
pkg_dict[flat_map[root_file].basename] = flat_map[root_file]
outputs.append(flat_map[root_file])
for include_file in include_files:
pkg_dict[flat_map[include_file].basename] = flat_map[include_file]
outputs.append(flat_map[include_file])
if len(installed_tests) > 0:
real_test_list_output_location = ctx.actions.declare_file(install_dir + "/" + installed_test_list_file.basename)
pkg_dict[real_test_list_output_location.basename] = real_test_list_output_location
@ -353,7 +370,7 @@ def mongo_install_rule_impl(ctx):
inputs = depset(direct = input_deps, transitive = [
ctx.attr._install_script.files,
python.files,
] + [f.files for f in ctx.attr.srcs] + [r.files for r in ctx.attr.root_files.keys()] + [dep[MongoInstallInfo].deps_files for dep in ctx.attr.deps] + [dep[DefaultInfo].files for dep in ctx.attr.deps] + [depset(dwps)])
] + [f.files for f in ctx.attr.srcs] + [r.files for r in ctx.attr.root_files.keys()] + [i.files for i in ctx.attr.include_files.keys()] + [dep[MongoInstallInfo].deps_files for dep in ctx.attr.deps] + [dep[DefaultInfo].files for dep in ctx.attr.deps] + [depset(dwps)])
if outputs:
ctx.actions.run(
@ -406,6 +423,7 @@ mongo_install_rule = rule(
"deps": attr.label_list(providers = [PackageFilesInfo], aspects = [test_binary_aspect]),
"debug": attr.string(),
"root_files": attr.label_keyed_string_dict(allow_files = True),
"include_files": attr.label_keyed_string_dict(allow_files = True),
"publish_debug_in_stripped": attr.bool(),
"create_dwp": attr.bool(),
"_install_script": attr.label(allow_single_file = True, default = "//bazel/install_rules:install_rules.py"),
@ -422,6 +440,7 @@ def mongo_install(
srcs,
deps = [],
root_files = {},
include_files = {},
target_compatible_with = [],
testonly = False,
pretty_printer_tests = {},
@ -511,6 +530,7 @@ def mongo_install(
name = install_target,
srcs = modified_srcs,
root_files = root_files,
include_files = include_files,
debug = debug,
create_dwp = select({
"//bazel/config:dwp_supported": True,

13
bazel/install_rules/install_rules.py
View File

@ -35,9 +35,14 @@ install_link = args.install_dir + "/../install"
os.makedirs(install_link, exist_ok=True)
def install(src, install_type):
install_dst = os.path.join(args.install_dir, install_type, os.path.basename(src))
link_dst = os.path.join(install_link, install_type, os.path.basename(src))
def install(src, install_type, is_rename=False):
if is_rename:
install_dst = os.path.join(args.install_dir, install_type)
link_dst = os.path.join(install_link, install_type)
else:
install_dst = os.path.join(args.install_dir, install_type, os.path.basename(src))
link_dst = os.path.join(install_link, install_type, os.path.basename(src))
if src.endswith(".dwp"):
# Due to us creating our binaries using the _with_debug name
# the dwp files also contain it. Strip the _with_debug from the name
@ -122,3 +127,5 @@ for depfile in args.depfile:
install(lib, "lib")
for file, folder in content["roots"].items():
install(file, folder)
for file, folder in content["includes"].items():
install(file, folder, is_rename=True)

2
buildscripts/packager.py
View File

@ -591,7 +591,7 @@ def unpack_binaries_into(build_os, arch, spec, where):
for releasefile in "bin", "LICENSE-Community.txt", "README", "THIRD-PARTY-NOTICES", "MPL-2":
print("moving file: %s/%s" % (release_dir, releasefile))
os.rename("%s/%s" % (release_dir, releasefile), releasefile)
os.rmdir(release_dir)
shutil.rmtree(release_dir)
except Exception:
exc = sys.exc_info()[1]
os.chdir(rootdir)

2
buildscripts/packager_enterprise.py
View File

@ -75,7 +75,7 @@ class EnterpriseSpec(packager.Spec):
"MPL-2",
):
os.rename("%s/%s" % (release_dir, release_file), release_file)
os.rmdir(release_dir)
shutil.rmtree(release_dir)
class EnterpriseCryptSpec(EnterpriseSpec):

1
etc/BUILD.bazel
View File

@ -6,6 +6,7 @@ exports_files([
"lsan.suppressions",
"tsan.suppressions",
"burn_in_tests.yml",
"extensions.yml",
])
# This is a hack to work around the fact that the cc_library flag additional_compiler_inputs doesn't

24
etc/extensions.yml
View File

@ -12,10 +12,10 @@
# name: mongot-extension
# version: 0.0.0
# variants:
# amazon2-x86_64: amazon2-x86_64
# amazon2-aarch64: amazon2-aarch64
# amazon2023-x86_64: amazon2023-x86_64
# amazon2023-aarch64: amazon2023-aarch64
# amazon2-x86_64: checksum1
# amazon2-aarch64: checksum2
# amazon2023-x86_64: checksum3
# amazon2023-aarch64: checksum4
extensions:
mongot_extension:
@ -23,16 +23,16 @@ extensions:
name: mongot-extension
version: 0.0.0
variants:
amazon2-x86_64: amazon2-x86_64
amazon2-aarch64: amazon2-aarch64
amazon2023-x86_64: amazon2023-x86_64
amazon2023-aarch64: amazon2023-aarch64
amazon2-x86_64: 6abaef106a4cddb14023e74b6806f50962042e48e4540f96a1df37eaf62ca2a3
amazon2-aarch64: ee8a40f84b96d75af7304817ac221bf672d18ab427a5a9f9374da0556fc56b0a
amazon2023-x86_64: 0755418443d1a069ff328c63ca0fd22aee52ef3b7757ee2fa9a84f55ad98ab25
amazon2023-aarch64: 41364b9c87b087f55b9272bf124fe172fd51e1fa806c415d1162f350e0eb15a9
rerank_extension:
base_url: https://mongot-extension.s3.amazonaws.com/release/
name: rerank-extension
version: 0.0.0
variants:
amazon2-x86_64: amazon2-x86_64
amazon2-aarch64: amazon2-aarch64
amazon2023-x86_64: amazon2023-x86_64
amazon2023-aarch64: amazon2023-aarch64
amazon2-x86_64: 8103cede35bca027eb53059efae65f79e10fdf9a441658cb0866904efdcf594d
amazon2-aarch64: 54365e10e23995a864e12e35dbcebcd9e4cfbee285ad1ddc4f2caadaa353ceae
amazon2023-x86_64: f2b87efaab686ab19f631385e3044616f22cc90101be55ac0f03cb40d8a3f82b
amazon2023-aarch64: e7d923ca8b231ff03a45684a70eeeacda4a23fa2622c88e16a5a48f550daa53b

60
src/mongo/db/extension/extensions/BUILD.bazel Normal file
View File

@ -0,0 +1,60 @@
load("//bazel/install_rules:install_rules.bzl", "extensions_with_config")
load("//bazel:mongo_src_rules.bzl", "mongo_cc_extension_shared_library")
load("@poetry//:dependencies.bzl", "dependency")
load("//bazel/config:python_genrule.bzl", "python_genrule")
package(default_visibility = ["//visibility:public"])
# mongot-extension
[
python_genrule(
name = "download_" + extension_name + "_" + variant,
srcs = [
":download_external_extension.py",
"//etc:extensions.yml",
],
cmd = [
"$(location :download_external_extension.py)",
"--extension-name",
extension_name,
"--conf-file",
"$(location //etc:extensions.yml)",
"--variant",
variant,
"--archive-files",
extension_name.replace("_", "-") + ".so",
extension_name.replace("_", "-") + ".so.sig",
"--outputs",
"$(location " + extension_name + "_" + variant + ".so)",
"$(location " + extension_name + "_" + variant + ".so.sig)",
],
outputs = [
extension_name + "_" + variant + ".so",
extension_name + "_" + variant + ".so.sig",
],
python_libs = [
dependency(
"pyyaml",
group = "core",
),
dependency(
"requests",
group = "core",
),
dependency(
"retry",
group = "testing",
),
],
)
for extension_name in [
"mongot_extension",
"rerank_extension",
]
for variant in [
"amazon2-x86_64",
"amazon2-aarch64",
"amazon2023-x86_64",
"amazon2023-aarch64",
]
]

57
src/mongo/db/extension/extensions/download_external_extension.py Normal file
View File

@ -0,0 +1,57 @@
import argparse
import hashlib
import io
import os
import shutil
import sys
import tarfile
import requests
from retry import retry
import yaml
def get_extension_conf(file_path, extension_name):
with open(file_path, "r") as file:
data = yaml.safe_load(file)
return data["extensions"][extension_name]
@retry(tries=3)
def get_tarball(base_url, name, version, variant, checksum):
url = f"{base_url}{name.replace('_', '-')}-{version}-{variant}.tgz"
response = requests.get(url)
assert hashlib.sha256(response.content).hexdigest() == checksum
tarball = tarfile.open(fileobj=io.BytesIO(response.content), mode="r:gz")
return tarball
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--extension-name", required=True)
parser.add_argument("--conf-file", required=True)
parser.add_argument("--variant", required=True)
parser.add_argument("--archive-files", required=True, nargs="+")
parser.add_argument("--outputs", required=True, nargs="+")
args = parser.parse_args(sys.argv[1:])
conf_data = get_extension_conf(args.conf_file, args.extension_name)
checksum = conf_data["variants"][args.variant]
tarball = get_tarball(
conf_data["base_url"], args.extension_name, conf_data["version"], args.variant, checksum
)
basename_to_output = dict(zip(args.archive_files, args.outputs))
for tf in tarball.getmembers():
base = os.path.basename(tf.name)
if base in basename_to_output:
output = basename_to_output[base]
with open(output, "wb") as stream:
archive_file_stream = tarball.extractfile(tf)
shutil.copyfileobj(archive_file_stream, stream)
archive_file_stream.close()