mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-110326 Modify load_all_extensions() to use .conf files (#41278)

Browse Source 
GitOrigin-RevId: b5ae8a1c21483fc1afb3b8f6a5c010f833e61bd6
This commit is contained in:
Joshua Siegel 2025-09-15 15:25:10 -04:00 committed by MongoDB Bot
parent d1d1e58c0f
commit 40d59d9823
8 changed files with 152 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
BUILD.bazel
View File

@ -445,9 +445,8 @@ extensions_with_config(
# TODO SERVER-109108: Remove this entry when the bar extension is no longer needed.
"//src/mongo/db/extension/test_examples:bar_mongo_extension",
"//src/mongo/db/extension/test_examples:foo_mongo_extension",
# TODO SERVER-110326: Add these tests when possible.
# "//src/mongo/db/extension/test_examples:test_options_mongo_extension",
# "//src/mongo/db/extension/test_examples:parse_options_mongo_extension",
"//src/mongo/db/extension/test_examples:test_options_mongo_extension",
"//src/mongo/db/extension/test_examples:parse_options_mongo_extension",
# Any extension that is just loaded in a no-passthrough test MUST NOT have the
# "_mongo_extension" suffix.

87
buildscripts/resmokelib/testing/fixtures/fixturelib.py
View File

@ -2,9 +2,12 @@
import glob
import os
import shutil
from logging import Handler, Logger
from typing import Dict, Optional
import yaml
from buildscripts.resmokelib import config, core, errors, logging, utils
from buildscripts.resmokelib.core import network
from buildscripts.resmokelib.testing.fixtures import _builder
@ -137,9 +140,87 @@ class FixtureLib:
original[self.SET_PARAMETERS_KEY] = merged_set_parameters
return original
def load_all_extensions(self, is_evergreen: bool, mongod_options: dict, logger: logging.Logger, mongos_options: Optional[dict] = None):
conf_out_dir = os.path.join(os.path.abspath(os.sep), "tmp", "mongo", "extensions")
def delete_extension_conf_dir(self):
"""Delete the directory containing extension configuration files."""
try:
# Ignore errors if the directory does not exist.
shutil.rmtree(os.path.dirname(self.conf_out_dir), ignore_errors=True)
except Exception as e:
raise RuntimeError(
f"Failed to delete configuration directory {os.path.dirname(self.conf_out_dir)}"
) from e
def generate_extension_configuration_files(
self,
so_files: list[str],
logger: logging.Logger,
) -> list[str]:
"""Return a list of extension names with configuration files generated."""
conf_in_path = os.path.join(
os.getcwd(), "src", "mongo", "db", "extension", "test_examples", "configurations.yml"
)
try:
with open(conf_in_path, "r") as fstream:
yml = yaml.safe_load(fstream)
logger.debug("Loaded test extensions' configuration file %s", conf_in_path)
except FileNotFoundError as e:
raise RuntimeError(
f"Cannot find test extensions' configuration file {conf_in_path}"
) from e
extensions = yml.get("extensions") or {}
extension_names = []
for so_file in so_files:
# path/to/libfoo_mongo_extension.so -> libfoo_mongo_extension
file_name = os.path.basename(so_file)
extension_name = os.path.splitext(file_name)[0]
# TODO SERVER-110634: Remove 'lib' prefix and '_mongo_extension' suffix from extension names.
# Add the parsed extension name to the list.
extension_names.append(extension_name)
conf_file_path = os.path.join(self.conf_out_dir, f"{extension_name}.conf")
try:
os.makedirs(os.path.dirname(conf_file_path), exist_ok=True)
# Create the configuration file for the extension.
with open(conf_file_path, "w+") as conf_file:
# All extension configuration files will have a sharedLibrary path.
conf_file.write(f"sharedLibraryPath: {so_file}\n")
# Copy over extensionOptions if they exist.
if ext_config := extensions.get(extension_name):
yaml.dump(ext_config, conf_file)
logger.debug(
"Created configuration file for extension %s at %s",
extension_name,
conf_file_path,
)
except (IOError, OSError) as e:
# Clean up created directories on failure.
self.delete_extension_conf_dir()
raise RuntimeError(
f"Failed to create configuration file for extension {extension_name} at {conf_file_path}"
) from e
return extension_names
def load_all_extensions(
self,
is_evergreen: bool,
mongod_options: dict,
logger: logging.Logger,
mongos_options: Optional[dict] = None,
):
"""Find extensions, generate configuration files, and add them to mongod/mongos startup parameters."""
cwd = os.getcwd()
if is_evergreen:
search_dirs = [os.path.join(cwd, "dist-test", "lib")]
else:
@ -161,6 +242,8 @@ class FixtureLib:
if so_files:
logger.debug("Found extension files: %s", so_files)
# TODO SERVER-110634: Pass through extension names instead of paths to mongod/mongos.
self.generate_extension_configuration_files(so_files, logger)
joined_files = ",".join(so_files)
mongod_options["loadExtensions"] = joined_files
if mongos_options is not None:

20
buildscripts/resmokelib/testing/fixtures/replicaset.py
View File

@ -86,10 +86,13 @@ class ReplicaSetFixture(interface.ReplFixture, interface._DockerComposeInterface
self.mongod_options = self.fixturelib.make_historic(
self.fixturelib.default_if_none(mongod_options, {})
)
if load_all_extensions:
self.fixturelib.load_all_extensions(self.config.EVERGREEN_TASK_ID, self.mongod_options, self.logger)
self.load_all_extensions = load_all_extensions
if self.load_all_extensions:
self.fixturelib.load_all_extensions(
self.config.EVERGREEN_TASK_ID, self.mongod_options, self.logger
)
self.preserve_dbpath = preserve_dbpath
self.start_initial_sync_node = start_initial_sync_node
self.electable_initial_sync_node = electable_initial_sync_node
@ -465,14 +468,12 @@ class ReplicaSetFixture(interface.ReplFixture, interface._DockerComposeInterface
primary = self.nodes[0]
client = primary.mongo_client()
while True:
self.logger.info(
"Waiting for primary on port %d to be elected.", primary.port)
self.logger.info("Waiting for primary on port %d to be elected.", primary.port)
cmd_result = client.admin.command("isMaster")
if cmd_result["ismaster"]:
break
time.sleep(0.1) # Wait a little bit before trying again.
self.logger.info(
"Primary on port %d successfully elected.", primary.port)
self.logger.info("Primary on port %d successfully elected.", primary.port)
def _await_secondaries(self):
# Wait for the secondaries to become available.
@ -682,6 +683,9 @@ class ReplicaSetFixture(interface.ReplFixture, interface._DockerComposeInterface
def _do_teardown(self, mode=None):
self.logger.info("Stopping all members of the replica set '%s'...", self.replset_name)
if self.load_all_extensions:
self.fixturelib.delete_extension_conf_dir()
running_at_start = self.is_running()
if not running_at_start:
self.logger.info(

18
buildscripts/resmokelib/testing/fixtures/shardedcluster.py
View File

@ -46,8 +46,8 @@ class ShardedClusterFixture(interface.Fixture, interface._DockerComposeInterface
load_all_extensions=False,
set_cluster_parameter=None,
inject_catalog_metadata=None,
shard_replset_name_prefix = "shard-rs",
configsvr_replset_name = "config-rs",
shard_replset_name_prefix="shard-rs",
configsvr_replset_name="config-rs",
):
"""
Initialize ShardedClusterFixture with different options for the cluster processes.
@ -68,10 +68,13 @@ class ShardedClusterFixture(interface.Fixture, interface._DockerComposeInterface
self.mongod_options = self.fixturelib.make_historic(
self.fixturelib.default_if_none(mongod_options, {})
)
if load_all_extensions:
self.fixturelib.load_all_extensions(self.config.EVERGREEN_TASK_ID, self.mongod_options, self.logger, self.mongos_options)
self.load_all_extensions = load_all_extensions
if self.load_all_extensions:
self.fixturelib.load_all_extensions(
self.config.EVERGREEN_TASK_ID, self.mongod_options, self.logger, self.mongos_options
)
self.mongod_executable = mongod_executable
self.mongod_options["set_parameters"] = self.fixturelib.make_historic(
mongod_options.get("set_parameters", {})
@ -385,6 +388,9 @@ class ShardedClusterFixture(interface.Fixture, interface._DockerComposeInterface
"""Shut down the sharded cluster."""
self.logger.info("Stopping all members of the sharded cluster...")
if self.load_all_extensions:
self.fixturelib.delete_extension_conf_dir()
running_at_start = self.is_running()
if not running_at_start:
self.logger.warning(

11
buildscripts/resmokelib/testing/fixtures/standalone.py
View File

@ -47,6 +47,7 @@ class MongoDFixture(interface.Fixture, interface._DockerComposeInterface):
preserve_dbpath (bool, optional): preserve_dbpath. Defaults to False.
port (Optional[int], optional): Port to use for mongod. Defaults to None.
launch_mongot (bool, optional): Should mongot be launched as well. Defaults to False.
load_all_extensions (bool, optional): Whether to load all test extensions upon startup. Defaults to False.
Raises
ValueError: _description_
@ -56,8 +57,11 @@ class MongoDFixture(interface.Fixture, interface._DockerComposeInterface):
self.fixturelib.default_if_none(mongod_options, {})
)
if load_all_extensions:
self.fixturelib.load_all_extensions(self.config.EVERGREEN_TASK_ID, self.mongod_options, self.logger)
self.load_all_extensions = load_all_extensions
if self.load_all_extensions:
self.fixturelib.load_all_extensions(
self.config.EVERGREEN_TASK_ID, self.mongod_options, self.logger
)
if "set_parameters" not in self.mongod_options:
self.mongod_options["set_parameters"] = {}
@ -263,6 +267,9 @@ class MongoDFixture(interface.Fixture, interface._DockerComposeInterface):
self.logger.info("Successfully contacted the mongod on port %d.", self.port)
def _do_teardown(self, mode=None):
if self.load_all_extensions:
self.fixturelib.delete_extension_conf_dir()
if self.config.NOOP_MONGO_D_S_PROCESSES:
self.logger.info(
"This is running against an External System Under Test setup with `docker-compose.yml` -- skipping teardown."

3
evergreen/functions/find_extensions.sh
View File

@ -32,8 +32,9 @@ if [[ -n "${EXTENSION_NAME}" ]]; then
echo "extension_paths: \"${EXTENSION_PATH}\"" >"${TOP_LEVEL_DIR}/extension_paths.yml"
else
echo "EXTENSION_NAME not provided. Finding all unpacked extensions."
# TODO SERVER-110634: Remove exclusions once parse_options and test_options can generate .conf files.
# Find all *_mongo_extension.so files and create a comma-separated list.
EXTENSIONS_LIST=$(find "${LIB_SRC}" -name "*_mongo_extension.so" | paste -sd, -)
EXTENSIONS_LIST=$(find "${LIB_SRC}" -name "*_mongo_extension.so" -not -name "libparse_options_mongo_extension.so" -not -name "libtest_options_mongo_extension.so" | paste -sd, -)
if [[ -z "${EXTENSIONS_LIST}" ]]; then
echo "Error: Could not find any extracted extension files."

20
src/mongo/db/extension/host/load_extension.cpp
View File

@ -29,6 +29,7 @@
#include "mongo/db/extension/host/load_extension.h"
#include "mongo/db/commands/test_commands_enabled.h"
#include "mongo/db/extension/host_adapter/extension_handle.h"
#include "mongo/db/extension/public/api.h"
#include "mongo/db/extension/sdk/extension_status.h"
@ -50,8 +51,14 @@
namespace mongo::extension::host {
namespace {
// TODO SERVER-110326: Check if we are in a test environment. If so, use /tmp/mongo/extensions.
static const std::filesystem::path kExtensionConfDir{"/etc/mongo/extensions"};
const std::filesystem::path& getExtensionConfDir() {
// Use /tmp/mongo/extensions in test environments, otherwise use /etc/mongo/extensions.
static const std::filesystem::path kExtensionConfDir =
getTestCommandsEnabled() ? "/tmp/mongo/extensions" : "/etc/mongo/extensions";
return kExtensionConfDir;
}
void assertVersionCompatibility(const ::MongoExtensionAPIVersionVector* hostVersions,
const ::MongoExtensionAPIVersion& extensionVersion) {
@ -147,13 +154,18 @@ bool loadExtensions(const std::vector<std::string>& extensionPaths) {
ExtensionConfig ExtensionLoader::loadExtensionConfig(const std::string& extensionPath) {
// TODO SERVER-110317: 'extensionPath' shouldn't represent a path anymore. We can omit the
// truncation.
const auto confPath = kExtensionConfDir /
const auto confPath = getExtensionConfDir() /
std::filesystem::path(extensionPath).filename().replace_extension(".conf");
// TODO SERVER-110326: Remove this once we have proper loading for tests.
// TODO SERVER-110634: Remove this once we have proper loading for tests in Evergreen.
if (!std::filesystem::exists(confPath)) {
LOGV2(11032600,
"Could not find configuration file for extension, using default configuration",
"confPath"_attr = confPath.string());
return ExtensionConfig{.sharedLibraryPath = extensionPath,
.extOptions = YAML::Node(YAML::NodeType::Map)};
}
uassert(11042900,
str::stream() << "Loading extension '" << extensionPath
<< "' failed: Expected configuration file not found at '"

14
src/mongo/db/extension/test_examples/configurations.yml Normal file
View File

@ -0,0 +1,14 @@
# Holds extension-specific configuration information necessary to generate configuration files for passthrough tests.
#
# If a test extension does not require any unique configuration, no changes to this file are necessary. Notably,
# "sharedLibraryPath" should _not_ be included in this file. load_all_extensions() will generate correct configuration
# files for all loadable test extensions, including "sharedLibraryPath"s for both local and Evergreen runs.
extensions:
# TODO SERVER-110634: Change extension names to test_options and parse_options, respectively.
libtest_options_mongo_extension:
extensionOptions:
optionA: true
libparse_options_mongo_extension:
extensionOptions:
checkMax: true
max: 10