mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-121737 Add fixture and hook support to external resmoke modules (#49689)

Browse Source 
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
GitOrigin-RevId: ed84084a285fbfdb493480f5a5a6b633c703017f
This commit is contained in:
Trevor Guidry 2026-03-17 09:24:58 -05:00 committed by MongoDB Bot
parent d1ffae38ad
commit 6fa6d77e0b
33 changed files with 586 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
buildscripts/resmoke.py
View File

@ -6,7 +6,19 @@ import sys
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
mongo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# use the mongo root directory to resolve depencies first
# This is so when resmoke is used as a module in an external project,
# we use the resmoke code that is bundled with resmoke,
# not a local checkout of resmoke that might be for a different version
sys.path.insert(0, mongo_root)
if os.path.normpath(mongo_root) != os.path.normpath(os.getcwd()):
# If the current working directory is not the mongo root that means
# we are running as an external module.
# We need to add that path so we can import fixtures from the external location.
# This needs to be a lower priority than the mongo root.
sys.path.insert(1, os.getcwd())
try:
from buildscripts.resmokelib import cli

30
buildscripts/resmokelib/bazel_suite_parser.py
View File

@ -12,6 +12,8 @@ import functools
import os
import re
from buildscripts.resmokelib import config
class BazelParseError(Exception):
"""Exception raised when parsing BUILD.bazel files fails."""
@ -99,7 +101,8 @@ def _parse_label(target_label: str) -> tuple[str, str]:
Args:
target_label: A Bazel target label like "//package/path:target_name"
Returns:
Tuple of (package_path, target_name)
Tuple of (absolute_package_path, target_name) where absolute_package_path
is the full path to the package directory (for finding BUILD.bazel files).
Raises:
BazelParseError: If the label format is invalid
"""
@ -117,7 +120,8 @@ def _parse_label(target_label: str) -> tuple[str, str]:
)
package, target_name = label_without_prefix.split(":", 1)
return package, target_name
# Return absolute path for finding BUILD.bazel files
return os.path.join(config.RESMOKE_ROOT, package), target_name
def _parse_load_statements(content: str, package: str) -> dict[str, str]:
@ -149,13 +153,16 @@ def _parse_load_statements(content: str, package: str) -> dict[str, str]:
# Convert the .bzl label to a file path
# Example: "//jstests/suites:selectors.bzl"
# -> "jstests/suites/selectors.bzl"
# -> "/absolute/path/to/jstests/suites/selectors.bzl"
if bzl_label.startswith("//"):
bzl_path = bzl_label[2:].replace(":", "/")
# Absolute Bazel label - resolve relative to RESMOKE_ROOT
relative_path = bzl_label[2:].replace(":", "/")
bzl_path = os.path.normpath(os.path.join(config.RESMOKE_ROOT, relative_path))
else:
# Relative path - resolve relative to current package
bzl_path = os.path.join(package, bzl_label.replace(":", ""))
bzl_path = os.path.join(*bzl_path.split("/"))
# Strip leading ':' if present (package-relative reference)
relative_part = bzl_label[1:] if bzl_label.startswith(":") else bzl_label
bzl_path = os.path.normpath(os.path.join(package, relative_part.replace(":", "/")))
# Extract all identifiers from the load statement
identifier_pattern = r'["\']([^"\']+)["\']'
@ -336,19 +343,22 @@ def resolve_target_to_files(target_label: str) -> str:
Raises:
BazelParseError: If target type is unsupported
"""
package, target_name = _parse_label(target_label)
absolute_package, target_name = _parse_label(target_label)
# Convert absolute package path to relative (for suite configuration)
relative_package = os.path.relpath(absolute_package, config.RESMOKE_ROOT)
if target_name.endswith(".js"):
# Direct file reference
return os.path.join(package, target_name)
return os.path.join(relative_package, target_name)
elif target_name == "all_javascript_files":
# Return glob pattern for *.js in package directory
return os.path.join(package, "*.js")
return os.path.join(relative_package, "*.js")
elif target_name == "all_subpackage_javascript_files":
# Return glob pattern for recursive **/*.js
return os.path.join(package, "**/*.js")
return os.path.join(relative_package, "**/*.js")
else:
raise BazelParseError(

3
buildscripts/resmokelib/config.py
View File

@ -17,6 +17,9 @@ RESMOKE_ROOT = str(Path(__file__).parent.parent.parent)
# This is used when external projects import and use resmoke as a module
EXTERNAL_MODULE_ROOT = os.getcwd()
# Whether or not resmoke is being run from an external module
IN_EXTERNAL_MODULE = os.path.normpath(EXTERNAL_MODULE_ROOT) != os.path.normpath(RESMOKE_ROOT)
# Subdirectory under the dbpath prefix that contains directories with data files of mongod's started
# by resmoke.py.
FIXTURE_SUBDIR = "resmoke"

18
buildscripts/resmokelib/configure_resmoke.py
View File

@ -125,6 +125,8 @@ def _load_external_module_config(config_path: str):
Expected YAML format:
suite_directories: [list of suite directories relative to EXTERNAL_MODULE_ROOT]
matrix_suite_directories: [list of matrix suite directories relative to EXTERNAL_MODULE_ROOT]
fixture_directories: [list of fixture directories relative to EXTERNAL_MODULE_ROOT]
hook_directories: [list of hook directories relative to EXTERNAL_MODULE_ROOT]
"""
if not os.path.isabs(config_path):
# If relative path provided, resolve relative to EXTERNAL_MODULE_ROOT
@ -167,6 +169,22 @@ def _load_external_module_config(config_path: str):
else:
print(f"Warning: External matrix suite directory not found: {full_path}")
for config_key, dir_type in [("fixture_directories", "fixture"), ("hook_directories", "hook")]:
dirs = external_config.get(config_key, [])
if not isinstance(dirs, list):
raise RuntimeError(f"'{config_key}' must be a list")
for dir_path in dirs:
full_path = os.path.join(_config.EXTERNAL_MODULE_ROOT, dir_path)
if not os.path.exists(full_path):
print(f"Warning: External {dir_type} directory not found: {full_path}")
continue
# Convert directory path to Python package name (replace slashes with dots)
package_name = dir_path.replace("/", ".")
# Use autoloader to load all modules in the directory
autoloader.load_all_modules(name=package_name, path=[full_path])
def _validate_options(parser: argparse.ArgumentParser, args: dict):
"""Do preliminary validation on the options and error on any invalid options."""

46
buildscripts/resmokelib/suitesconfig.py
View File

@ -226,6 +226,10 @@ def _get_suite_config(suite_name_or_path):
def generate():
MatrixSuiteConfig.generate_all_matrix_suite_files()
# don't try to use bazel run format in external modules since bazel may not be available
if _config.IN_EXTERNAL_MODULE:
return
print("\nRunning 'bazel run format' to format generated files...")
print("Note: This may take a while to complete.")
try:
@ -441,9 +445,11 @@ class MatrixSuiteConfig(SuiteConfigInterface):
Returns:
List of absolute paths to matrix suite directories.
"""
return [
os.path.join(_config.CONFIG_DIR, "matrix_suites")
] + _config.MODULE_MATRIX_SUITE_DIRS
return (
[os.path.join(_config.CONFIG_DIR, "matrix_suites")]
+ _config.MODULE_MATRIX_SUITE_DIRS
+ _config.EXTERNAL_MODULE_MATRIX_SUITE_DIRS
)
@staticmethod
def get_suites_dirs_with_roots() -> list[tuple[str, str]]:
@ -553,6 +559,31 @@ class MatrixSuiteConfig(SuiteConfigInterface):
res["matrix_suite"] = True
overrides = copy.deepcopy(overrides)
# If this matrix suite is from an external module but the base suite is built-in,
# prefix selector paths with "builtin:" so they resolve correctly
matrix_suite_root = cls.get_suite_root(suite_name)
base_suite_root = ExplicitSuiteConfig.get_suite_root(base_suite_name)
if (
_config.IN_EXTERNAL_MODULE
and matrix_suite_root == _config.EXTERNAL_MODULE_ROOT
and base_suite_root == _config.RESMOKE_ROOT
):
# Prefix all selector paths (roots, include_files, exclude_files) with "builtin:"
if "selector" in res:
selector = res["selector"]
for key in ["roots", "include_files", "exclude_files"]:
if key in selector:
paths = selector[key]
if isinstance(paths, list):
new_paths = []
for path in paths:
if path.startswith("builtin:"):
# Already has prefix
new_paths.append(path)
else:
new_paths.append(f"builtin:{path}")
selector[key] = new_paths
if description:
res["description"] = description
@ -801,13 +832,18 @@ class MatrixSuiteConfig(SuiteConfigInterface):
print(f"Could not find mappings file for {suite_name}")
return None
suite_root = cls.get_suite_root(suite_name)
if not suite_root:
print(f"Could not determine suite root for {suite_name}")
raise RuntimeError(f"Could not determine suite root for {suite_name}")
# Convert absolute path to relative path from RESMOKE_ROOT
# This path needs to output the same text on both windows and linux/mac
mapping_path = pathlib.PurePath(mapping_path)
try:
mapping_path = mapping_path.relative_to(_config.RESMOKE_ROOT)
mapping_path = mapping_path.relative_to(suite_root)
except ValueError:
# If mapping_path is not under RESMOKE_ROOT, keep it as-is
# If mapping_path is not under suite_root, keep it as-is
pass
yml = yaml.safe_dump(matrix_suite)
comments = [

9
buildscripts/resmokelib/testing/testcases/benchmark_test.py
View File

@ -18,11 +18,14 @@ class BenchmarkTestCase(interface.ProcessTestCase):
logger: logging.Logger,
program_executables: list[str],
program_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the BenchmarkTestCase with the executable to run."""
assert len(program_executables) == 1
interface.ProcessTestCase.__init__(self, logger, "Benchmark test", program_executables[0])
interface.ProcessTestCase.__init__(
self, logger, "Benchmark test", program_executables[0], **kwargs
)
self.validate_benchmark_options()
self.bm_executable = program_executables[0]
@ -80,8 +83,8 @@ class BenchmarkTestCase(interface.ProcessTestCase):
process_kwargs = copy.deepcopy(bm_options.get("process_kwargs", {}))
interface.append_process_tracking_options(process_kwargs, self._id)
# Merge fixture environment variables into process_kwargs
self._merge_fixture_environment_variables(process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(process_kwargs)
bm_options["process_kwargs"] = process_kwargs
self.bm_options = bm_options

7
buildscripts/resmokelib/testing/testcases/cpp_integration_test.py
View File

@ -18,12 +18,13 @@ class CPPIntegrationTestCase(interface.ProcessTestCase):
logger: logging.Logger,
program_executables: list[str],
program_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the CPPIntegrationTestCase with the executable to run."""
assert len(program_executables) == 1
interface.ProcessTestCase.__init__(
self, logger, "C++ integration test", program_executables[0]
self, logger, "C++ integration test", program_executables[0], **kwargs
)
self.program_executable = program_executables[0]
@ -39,8 +40,8 @@ class CPPIntegrationTestCase(interface.ProcessTestCase):
process_kwargs = copy.deepcopy(self.program_options.get("process_kwargs", {}))
interface.append_process_tracking_options(process_kwargs, self._id)
# Merge fixture environment variables into process_kwargs
self._merge_fixture_environment_variables(process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(process_kwargs)
self.program_options["process_kwargs"] = process_kwargs
self.program_options = certs.expand_x509_paths(self.program_options)

7
buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py
View File

@ -20,12 +20,13 @@ class CPPLibfuzzerTestCase(interface.ProcessTestCase):
program_executables: list[str],
program_options: Optional[dict] = None,
corpus_directory_stem="corpora",
**kwargs,
):
"""Initialize the CPPLibfuzzerTestCase with the executable to run."""
assert len(program_executables) == 1
interface.ProcessTestCase.__init__(
self, logger, "C++ libfuzzer test", program_executables[0]
self, logger, "C++ libfuzzer test", program_executables[0], **kwargs
)
self.program_executable = program_executables[0]
@ -59,8 +60,8 @@ class CPPLibfuzzerTestCase(interface.ProcessTestCase):
f"--corpus_database={self.corpus_directory}",
f"--llvm_fuzzer_wrapper_corpus_dir={self.seed_directory}",
]
# Merge fixture environment variables into program_options
# Merge test and fixture environment variables into program_options
program_options = self.program_options.copy()
self._merge_fixture_environment_variables(program_options)
self._merge_environment_variables(program_options)
return core.programs.make_process(self.logger, default_args, **program_options)

9
buildscripts/resmokelib/testing/testcases/cpp_unittest.py
View File

@ -16,11 +16,14 @@ class CPPUnitTestCase(interface.ProcessTestCase):
logger: logging.Logger,
program_executables: list[str],
program_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the CPPUnitTestCase with the executable to run."""
assert len(program_executables) == 1
interface.ProcessTestCase.__init__(self, logger, "C++ unit test", program_executables[0])
interface.ProcessTestCase.__init__(
self, logger, "C++ unit test", program_executables[0], **kwargs
)
self.program_executable = program_executables[0]
self.program_options = utils.default_if_none(program_options, {}).copy()
@ -28,9 +31,9 @@ class CPPUnitTestCase(interface.ProcessTestCase):
interface.append_process_tracking_options(self.program_options, self._id)
def _make_process(self):
# Merge fixture environment variables into program_options
# Merge test and fixture environment variables into program_options
program_options = self.program_options.copy()
self._merge_fixture_environment_variables(program_options)
self._merge_environment_variables(program_options)
return core.programs.make_process(
self.logger,

7
buildscripts/resmokelib/testing/testcases/dbtest.py
View File

@ -21,11 +21,12 @@ class DBTestCase(interface.ProcessTestCase):
dbtest_suites: list[str],
dbtest_executable: Optional[str] = None,
dbtest_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the DBTestCase with the dbtest suite to run."""
assert len(dbtest_suites) == 1
interface.ProcessTestCase.__init__(self, logger, "dbtest suite", dbtest_suites[0])
interface.ProcessTestCase.__init__(self, logger, "dbtest suite", dbtest_suites[0], **kwargs)
# Command line options override the YAML configuration.
self.dbtest_executable = utils.default_if_none(config.DBTEST_EXECUTABLE, dbtest_executable)
@ -52,8 +53,8 @@ class DBTestCase(interface.ProcessTestCase):
process_kwargs = copy.deepcopy(self.dbtest_options.get("process_kwargs", {}))
interface.append_process_tracking_options(process_kwargs, self._id)
# Merge fixture environment variables into process_kwargs
self._merge_fixture_environment_variables(process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(process_kwargs)
self.dbtest_options["process_kwargs"] = process_kwargs
def _execute(self, process):

58
buildscripts/resmokelib/testing/testcases/interface.py
View File

@ -151,6 +151,19 @@ class TestCase(unittest.TestCase, metaclass=registry.make_registry_metaclass(_TE
class ProcessTestCase(TestCase):
"""Base class for TestCases that executes an external process."""
def __init__(
self,
logger: logging.Logger,
test_kind: str,
test_name: str,
dynamic: bool = False,
**kwargs,
):
"""Initialize ProcessTestCase."""
TestCase.__init__(self, logger, test_kind, test_name, dynamic)
# Extract environment variables from test configuration
self._test_env_vars = self._extract_env_vars_from_config(**kwargs)
def run_test(self):
"""Run the test."""
try:
@ -231,13 +244,30 @@ class ProcessTestCase(TestCase):
return {}
return self.fixture.get_environment_variables()
def _merge_fixture_environment_variables(self, process_kwargs):
@staticmethod
def _extract_env_vars_from_config(**kwargs):
"""
Merge fixture environment variables into process_kwargs.
Extract environment variables from test configuration.
This method updates process_kwargs in-place by merging fixture environment
variables with any existing env_vars. Fixture environment variables will not
override existing values in env_vars.
This is a helper to extract env_vars from the test_config passed to test cases.
Args:
**kwargs: Test configuration (typically from executor.config in suite YAML)
Returns:
dict: Environment variables to pass to the test, or empty dict if none found.
"""
return kwargs.get("env_vars", {})
def _merge_environment_variables(self, process_kwargs):
"""
Merge test and fixture environment variables into process_kwargs.
This method updates process_kwargs in-place by merging environment variables
in the following priority order (highest to lowest):
1. Existing values in process_kwargs['env_vars'] (e.g., process-specific)
2. Test environment variables from suite configuration
3. Fixture environment variables
Args:
process_kwargs (dict): Process kwargs dictionary that may contain 'env_vars'.
@ -245,18 +275,22 @@ class ProcessTestCase(TestCase):
Returns:
dict: The updated process_kwargs dictionary (same object, modified in-place).
"""
fixture_env_vars = self._get_fixture_environment_variables()
if not fixture_env_vars:
return process_kwargs
# Get or create env_vars in process_kwargs
if "env_vars" not in process_kwargs:
process_kwargs["env_vars"] = {}
# Merge test env vars, but don't override existing values
if self._test_env_vars:
for key, value in self._test_env_vars.items():
if key not in process_kwargs["env_vars"]:
process_kwargs["env_vars"][key] = value
# Merge fixture env vars, but don't override existing values
for key, value in fixture_env_vars.items():
if key not in process_kwargs["env_vars"]:
process_kwargs["env_vars"][key] = value
fixture_env_vars = self._get_fixture_environment_variables()
if fixture_env_vars:
for key, value in fixture_env_vars.items():
if key not in process_kwargs["env_vars"]:
process_kwargs["env_vars"][key] = value
return process_kwargs

7
buildscripts/resmokelib/testing/testcases/jsrunnerfile.py
View File

@ -22,10 +22,11 @@ class JSRunnerFileTestCase(interface.ProcessTestCase):
test_runner_file: str,
shell_executable: Optional[str] = None,
shell_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the JSRunnerFileTestCase with the 'test_name' file."""
interface.ProcessTestCase.__init__(self, logger, test_kind, test_name)
interface.ProcessTestCase.__init__(self, logger, test_kind, test_name, **kwargs)
# Command line options override the YAML configuration.
self.shell_executable = utils.default_if_none(config.MONGO_EXECUTABLE, shell_executable)
@ -47,8 +48,8 @@ class JSRunnerFileTestCase(interface.ProcessTestCase):
process_kwargs = copy.deepcopy(self.shell_options.get("process_kwargs", {}))
interface.append_process_tracking_options(process_kwargs, self._id)
# Merge fixture environment variables into process_kwargs
self._merge_fixture_environment_variables(process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(process_kwargs)
self.shell_options["process_kwargs"] = process_kwargs
def _populate_test_data(self, test_data):

14
buildscripts/resmokelib/testing/testcases/jstest.py
View File

@ -31,9 +31,10 @@ class _SingleJSTestCase(interface.ProcessTestCase):
_id: uuid.UUID,
shell_executable: Optional[str] = None,
shell_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the _SingleJSTestCase with the JS file to run."""
interface.ProcessTestCase.__init__(self, logger, "JSTest", test_name)
interface.ProcessTestCase.__init__(self, logger, "JSTest", test_name, **kwargs)
# Command line options override the YAML configuration.
self.shell_executable: Optional[str] = utils.default_if_none(
@ -124,8 +125,8 @@ class _SingleJSTestCase(interface.ProcessTestCase):
interface.append_process_tracking_options(process_kwargs, self._id)
# Add fixture environment variables to process_kwargs
self._merge_fixture_environment_variables(process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(process_kwargs)
self.shell_options["process_kwargs"] = process_kwargs
self.shell_options = certs.expand_x509_paths(self.shell_options)
@ -165,11 +166,13 @@ class JSTestCaseBuilder(interface.TestCaseFactory):
test_id: uuid.UUID,
shell_executable: Optional[str] = None,
shell_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the JSTestCase with the JS file to run."""
self.test_case_template = _SingleJSTestCase(
logger, js_filenames, test_name, test_id, shell_executable, shell_options
logger, js_filenames, test_name, test_id, shell_executable, shell_options, **kwargs
)
self._test_kwargs = kwargs
interface.TestCaseFactory.__init__(self, _SingleJSTestCase, shell_options)
def configure(self, fixture: "interface.Fixture", *args, **kwargs):
@ -190,6 +193,7 @@ class JSTestCaseBuilder(interface.TestCaseFactory):
self.test_case_template._id,
self.test_case_template.shell_executable,
shell_options,
**self._test_kwargs,
)
test_case.configure(self.test_case_template.fixture)
return test_case
@ -349,6 +353,7 @@ class JSTestCase(MultiClientsTestCase):
js_filenames: list[str],
shell_executable: Optional[str] = None,
shell_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the TestCase for running JS files."""
assert len(js_filenames) >= 1
@ -364,6 +369,7 @@ class JSTestCase(MultiClientsTestCase):
test_id,
shell_executable,
shell_options,
**kwargs,
)
MultiClientsTestCase.__init__(self, logger, self.TEST_KIND, test_name, test_id, factory)

10
buildscripts/resmokelib/testing/testcases/mongos_test.py
View File

@ -9,7 +9,7 @@ class MongosTestCase(interface.ProcessTestCase):
REGISTERED_NAME = "mongos_test"
def __init__(self, logger: logging.Logger, mongos_options: list[dict]):
def __init__(self, logger: logging.Logger, mongos_options: list[dict], **kwargs):
"""Initialize the mongos test and saves the options."""
assert len(mongos_options) == 1
@ -18,7 +18,9 @@ class MongosTestCase(interface.ProcessTestCase):
config.MONGOS_EXECUTABLE, config.DEFAULT_MONGOS_EXECUTABLE
)
# Use the executable as the test name.
interface.ProcessTestCase.__init__(self, logger, "mongos test", self.mongos_executable)
interface.ProcessTestCase.__init__(
self, logger, "mongos test", self.mongos_executable, **kwargs
)
self.options = mongos_options[0].copy()
self.process_kwargs = {}
@ -32,8 +34,8 @@ class MongosTestCase(interface.ProcessTestCase):
self.options["test"] = ""
interface.append_process_tracking_options(self.process_kwargs, self._id)
# Merge fixture environment variables into process_kwargs
self._merge_fixture_environment_variables(self.process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(self.process_kwargs)
def _make_process(self):
return core.programs.mongos_program(

7
buildscripts/resmokelib/testing/testcases/pretty_printer_testcase.py
View File

@ -16,12 +16,13 @@ class PrettyPrinterTestCase(interface.ProcessTestCase):
logger: logging.Logger,
program_executables: list[str],
program_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the PrettyPrinterTestCase with the executable to run."""
assert len(program_executables) == 1
interface.ProcessTestCase.__init__(
self, logger, "pretty printer test", program_executables[0]
self, logger, "pretty printer test", program_executables[0], **kwargs
)
self.program_executable = program_executables[0]
@ -30,8 +31,8 @@ class PrettyPrinterTestCase(interface.ProcessTestCase):
interface.append_process_tracking_options(self.program_options, self._id)
def _make_process(self):
# Merge fixture environment variables into program_options
# Merge test and fixture environment variables into program_options
program_options = self.program_options.copy()
self._merge_fixture_environment_variables(program_options)
self._merge_environment_variables(program_options)
return core.programs.make_process(self.logger, [self.program_executable], **program_options)

8
buildscripts/resmokelib/testing/testcases/pytest.py
View File

@ -11,16 +11,16 @@ class PyTestCase(interface.ProcessTestCase):
REGISTERED_NAME = "py_test"
def __init__(self, logger: logging.Logger, py_filenames: list[str]):
def __init__(self, logger: logging.Logger, py_filenames: list[str], **kwargs):
"""Initialize PyTestCase."""
assert len(py_filenames) == 1
interface.ProcessTestCase.__init__(self, logger, "PyTest", py_filenames[0])
interface.ProcessTestCase.__init__(self, logger, "PyTest", py_filenames[0], **kwargs)
def _make_process(self):
program_options = {}
interface.append_process_tracking_options(program_options, self._id)
# Merge fixture environment variables into program_options
self._merge_fixture_environment_variables(program_options)
# Merge test and fixture environment variables into program_options
self._merge_environment_variables(program_options)
return core.programs.generic_program(
self.logger, [sys.executable, "-m", "unittest", self.test_name], program_options
)

10
buildscripts/resmokelib/testing/testcases/query_tester_self_test.py
View File

@ -12,21 +12,23 @@ class QueryTesterSelfTestCase(interface.ProcessTestCase):
REGISTERED_NAME = "query_tester_self_test"
def __init__(self, logger: logging.Logger, test_filenames: list[str]):
def __init__(self, logger: logging.Logger, test_filenames: list[str], **kwargs):
"""Initialize QueryTesterSelfTestCase.
test_filenames must contain one test_file - a python file that takes one argument: the uri of the mongod.
To run multiple test files, you would create an instance of QueryTesterSelfTestCase for each one.
"""
assert len(test_filenames) == 1
interface.ProcessTestCase.__init__(self, logger, "QueryTesterSelfTest", test_filenames[0])
interface.ProcessTestCase.__init__(
self, logger, "QueryTesterSelfTest", test_filenames[0], **kwargs
)
self.test_file = test_filenames[0]
def _make_process(self):
program_options = {}
interface.append_process_tracking_options(program_options, self._id)
# Merge fixture environment variables into program_options
self._merge_fixture_environment_variables(program_options)
# Merge test and fixture environment variables into program_options
self._merge_environment_variables(program_options)
return core.programs.generic_program(
self.logger,
[

9
buildscripts/resmokelib/testing/testcases/query_tester_server_test.py
View File

@ -30,6 +30,7 @@ class QueryTesterServerTestCase(interface.ProcessTestCase):
test_dir: list[str],
wait_for_files: bool = True,
override: str = None,
**kwargs,
):
"""Initialize QueryTesterServerTestCase.
test_dir: file path to a dir that contains .test files, their corresponding .results and a .coll file
@ -38,7 +39,9 @@ class QueryTesterServerTestCase(interface.ProcessTestCase):
if we are pulling from a remote git repo.
"""
assert len(test_dir) == 1
interface.ProcessTestCase.__init__(self, logger, "QueryTesterServerTest", test_dir[0])
interface.ProcessTestCase.__init__(
self, logger, "QueryTesterServerTest", test_dir[0], **kwargs
)
self.test_dir = test_dir[0]
self.wait_for_files = wait_for_files
@ -73,7 +76,7 @@ class QueryTesterServerTestCase(interface.ProcessTestCase):
program_options = {}
interface.append_process_tracking_options(program_options, self._id)
# Merge fixture environment variables into program_options
self._merge_fixture_environment_variables(program_options)
# Merge test and fixture environment variables into program_options
self._merge_environment_variables(program_options)
return core.programs.generic_program(self.logger, command, program_options)

9
buildscripts/resmokelib/testing/testcases/sdam_json_test.py
View File

@ -20,10 +20,13 @@ class SDAMJsonTestCase(interface.ProcessTestCase):
json_test_files: list[str],
program_executable: Optional[str] = None,
program_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the TestCase with the executable to run."""
assert len(json_test_files) == 1
interface.ProcessTestCase.__init__(self, logger, "SDAM Json Test", json_test_files[0])
interface.ProcessTestCase.__init__(
self, logger, "SDAM Json Test", json_test_files[0], **kwargs
)
if program_executable:
self.program_executable = program_executable
@ -47,7 +50,7 @@ class SDAMJsonTestCase(interface.ProcessTestCase):
command_line = [self.program_executable]
command_line += ["--source-dir", self.TEST_DIR]
command_line += ["-f", self.json_test_file]
# Merge fixture environment variables into program_options
# Merge test and fixture environment variables into program_options
program_options = self.program_options.copy()
self._merge_fixture_environment_variables(program_options)
self._merge_environment_variables(program_options)
return core.programs.make_process(self.logger, command_line, **program_options)

7
buildscripts/resmokelib/testing/testcases/server_selection_json_test.py
View File

@ -20,11 +20,12 @@ class ServerSelectionJsonTestCase(interface.ProcessTestCase):
json_test_files: list[str],
program_executable: Optional[str] = None,
program_options: Optional[dict] = None,
**kwargs,
):
"""Initialize the TestCase with the executable to run."""
assert len(json_test_files) == 1
interface.ProcessTestCase.__init__(
self, logger, "Server Selection Json Test", json_test_files[0]
self, logger, "Server Selection Json Test", json_test_files[0], **kwargs
)
if program_executable:
@ -51,7 +52,7 @@ class ServerSelectionJsonTestCase(interface.ProcessTestCase):
command_line = [self.program_executable]
command_line += ["--source-dir", self.TEST_DIR]
command_line += ["-f", self.json_test_file]
# Merge fixture environment variables into program_options
# Merge test and fixture environment variables into program_options
program_options = self.program_options.copy()
self._merge_fixture_environment_variables(program_options)
self._merge_environment_variables(program_options)
return core.programs.make_process(self.logger, command_line, **program_options)

7
buildscripts/resmokelib/testing/testcases/tla_plus_test.py
View File

@ -19,6 +19,7 @@ class TLAPlusTestCase(interface.ProcessTestCase):
model_config_files: list[str],
java_binary: Optional[str] = None,
model_check_command: Optional[str] = "sh model-check.sh",
**kwargs,
):
"""Initialize the TLAPlusTestCase with a TLA+ model config file.
@ -46,7 +47,7 @@ class TLAPlusTestCase(interface.ProcessTestCase):
self.model_check_command = model_check_command
interface.ProcessTestCase.__init__(self, logger, "TLA+ test", spec_dir)
interface.ProcessTestCase.__init__(self, logger, "TLA+ test", spec_dir, **kwargs)
def _make_process(self):
process_kwargs = {"cwd": self.working_dir}
@ -54,8 +55,8 @@ class TLAPlusTestCase(interface.ProcessTestCase):
process_kwargs["env_vars"] = {"JAVA_BINARY": self.java_binary}
interface.append_process_tracking_options(process_kwargs, self._id)
# Merge fixture environment variables into process_kwargs
self._merge_fixture_environment_variables(process_kwargs)
# Merge test and fixture environment variables into process_kwargs
self._merge_environment_variables(process_kwargs)
return core.programs.generic_program(
self.logger,

34
buildscripts/resmokelib/utils/autoloader.py
View File

@ -1,7 +1,10 @@
"""Utility for loading all modules within a package."""
import importlib
import os
import pkgutil
import sys
import types
def load_all_modules(name: str, path: list[str]) -> None:
@ -18,5 +21,36 @@ def load_all_modules(name: str, path: list[str]) -> None:
_autoloader.load_all_modules(name=__name__, path=__path__)
"""
# If package doesn't exist yet, ensure all parent packages are created
if name not in sys.modules:
parts = name.split(".")
for i in range(len(parts)):
parent = ".".join(parts[: i + 1])
if parent not in sys.modules:
# Create a synthetic module for the parent package
parent_module = types.ModuleType(parent)
parent_module.__package__ = parent
# Set __path__ to make it a proper package that can contain submodules
# Derive parent path by going up the directory tree
if parent == name:
# Target package gets the provided path
parent_module.__path__ = path
else:
# Parent packages get paths derived from the target path
# Calculate how many levels up we need to go
levels_up = len(parts) - (i + 1)
parent_paths = []
for p in path:
# Go up 'levels_up' directories from the target path
parent_path = p
for _ in range(levels_up):
parent_path = os.path.dirname(parent_path)
parent_paths.append(parent_path)
parent_module.__path__ = parent_paths
sys.modules[parent] = parent_module
for _, module, _ in pkgutil.walk_packages(path=path):
importlib.import_module("." + module, package=name)

178
buildscripts/tests/resmoke_end2end/test_external_fixtures_hooks.py Normal file
View File

@ -0,0 +1,178 @@
"""Test external module fixtures and hooks loading."""
import logging
import os
import sys
import unittest
import uuid
import yaml
# Add the repo root to the path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
from buildscripts.resmokelib import config as _config
from buildscripts.resmokelib import configure_resmoke, suitesconfig
from buildscripts.resmokelib.testing import fixtures, hooks
from buildscripts.resmokelib.testing.testcases import jstest, pytest
class TestExternalFixturesHooks(unittest.TestCase):
"""Test that external fixtures and hooks are properly loaded."""
@classmethod
def setUpClass(cls):
"""Set up test environment once for all tests."""
# Set EXTERNAL_MODULE_ROOT to the test module directory
test_module_dir = os.path.join(
os.path.dirname(__file__), "test_external_module_fixtures_hooks"
)
_config.EXTERNAL_MODULE_ROOT = os.path.abspath(test_module_dir)
# Recalculate IN_EXTERNAL_MODULE after changing EXTERNAL_MODULE_ROOT
_config.IN_EXTERNAL_MODULE = os.path.normpath(
_config.EXTERNAL_MODULE_ROOT
) != os.path.normpath(_config.RESMOKE_ROOT)
# Set CONFIG_DIR to avoid errors
_config.CONFIG_DIR = os.path.join(_config.RESMOKE_ROOT, "buildscripts", "resmokeconfig")
# Load the external module config once for all tests
config_path = os.path.join(_config.EXTERNAL_MODULE_ROOT, "external_module.yml")
configure_resmoke._load_external_module_config(config_path)
def test_load_external_fixtures(self):
"""Test that external fixtures are loaded and registered."""
# Try to instantiate the external fixture by name
# The fixture should be registered in the fixtures registry
logger = logging.getLogger("test")
fixture = fixtures.make_fixture("TestExternalFixture", logger, job_num=0)
self.assertIsNotNone(fixture)
self.assertEqual(fixture.REGISTERED_NAME, "TestExternalFixture")
def test_load_external_hooks(self):
"""Test that external hooks are loaded and registered."""
# Try to instantiate the external hook by name
# The hook should be registered in the hooks registry
logger = logging.getLogger("test")
hook = hooks.make_hook("TestExternalHook", logger, None, "Test hook")
self.assertIsNotNone(hook)
self.assertEqual(hook.REGISTERED_NAME, "TestExternalHook")
def test_test_env_vars_jstest(self):
"""Test that test environment variables are properly passed to JS test cases."""
# Create a _SingleJSTestCase with env_vars in config (not shell_options)
logger = logging.getLogger("test")
test_env_vars = {"TEST_VAR_1": "value1", "TEST_VAR_2": "value2"}
shell_options = {
"global_vars": {"TestData": {}},
}
test_case = jstest._SingleJSTestCase(
logger,
["jstests/core/query/find/find1.js"],
"test.js",
uuid.uuid4(),
shell_options=shell_options,
env_vars=test_env_vars,
)
# Verify that env_vars were extracted and stored
self.assertIsNotNone(test_case._test_env_vars)
self.assertEqual(test_case._test_env_vars, test_env_vars)
def test_test_env_vars_pytest(self):
"""Test that test environment variables are properly passed to Python test cases."""
# Create a PyTestCase with env_vars in config
logger = logging.getLogger("test")
test_env_vars = {"TEST_VAR_1": "value1", "TEST_VAR_2": "value2"}
test_case = pytest.PyTestCase(
logger,
["buildscripts/tests/test_example.py"],
env_vars=test_env_vars,
)
# Verify that env_vars were extracted and stored
self.assertIsNotNone(test_case._test_env_vars)
self.assertEqual(test_case._test_env_vars, test_env_vars)
def test_generate_matrix_suites_with_external_module(self):
"""Test that matrix suite generation works with external modules."""
# Create a MatrixSuiteConfig
matrix_suite_config = suitesconfig.MatrixSuiteConfig()
# The suite name is the filename without extension
suite_name = "test_external_matrix"
# Generate the matrix suite file
matrix_suite_config.generate_matrix_suite_file(suite_name)
# Get the generated suite path
generated_suite_path = matrix_suite_config.get_generated_suite_path(suite_name)
# Verify the file exists
self.assertTrue(
os.path.exists(generated_suite_path),
msg=f"Generated suite file not found at {generated_suite_path}",
)
# Load the generated suite and verify it contains builtin: prefix
with open(generated_suite_path, "r", encoding="utf8") as file:
generated_suite = yaml.safe_load(file)
# The generated suite should have selector paths with builtin: prefix
# since the base suite (core) is from RESMOKE_ROOT but the matrix suite
# is from EXTERNAL_MODULE_ROOT
self.assertIn("selector", generated_suite)
selector = generated_suite["selector"]
# Check roots have builtin: prefix
self.assertIn("roots", selector)
roots = selector["roots"]
self.assertIsInstance(roots, list)
self.assertGreater(len(roots), 0, "Generated suite should have selector roots")
for root in roots:
self.assertTrue(
root.startswith("builtin:"),
f"Root '{root}' should have 'builtin:' prefix since it comes from a built-in base suite",
)
# Check exclude_files have builtin: prefix (if present)
if "exclude_files" in selector:
exclude_files = selector["exclude_files"]
self.assertIsInstance(exclude_files, list)
for exclude_file in exclude_files:
self.assertTrue(
exclude_file.startswith("builtin:"),
f"Exclude file '{exclude_file}' should have 'builtin:' prefix since it comes from a built-in base suite",
)
# Check include_files have builtin: prefix (if present)
if "include_files" in selector:
include_files = selector["include_files"]
self.assertIsInstance(include_files, list)
for include_file in include_files:
self.assertTrue(
include_file.startswith("builtin:"),
f"Include file '{include_file}' should have 'builtin:' prefix since it comes from a built-in base suite",
)
# Verify the suite can be loaded and verified
try:
suite = matrix_suite_config.get_config_obj_and_verify(suite_name)
self.assertIsNotNone(suite, msg=f"Suite {suite_name} could not be loaded")
except Exception as ex:
self.fail(f"Failed to load generated suite: {repr(ex)}")
# Clean up: remove the generated file
if os.path.exists(generated_suite_path):
os.remove(generated_suite_path)
if __name__ == "__main__":
unittest.main()

17
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/external_module.yml Normal file
View File

@ -0,0 +1,17 @@
# Example external module configuration for resmoke
# Directories containing test suite YAML files
suite_directories:
- suites
# Directories containing matrix suite configurations
matrix_suite_directories:
- matrix_suites
# Directories containing custom fixture classes
fixture_directories:
- fixtures
# Directories containing custom hook classes
hook_directories:
- hooks

8
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/fixtures/BUILD.bazel Normal file
View File

@ -0,0 +1,8 @@
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
# TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets
py_library(
name = "all_python_files",
srcs = glob(["*.py"]),
visibility = ["//visibility:public"],
)

28
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/fixtures/test_fixture.py Normal file
View File

@ -0,0 +1,28 @@
"""Test fixture for external module."""
from buildscripts.resmokelib.testing.fixtures import interface
class TestExternalFixture(interface.Fixture):
"""A simple test fixture for external module testing."""
REGISTERED_NAME = "TestExternalFixture"
def __init__(self, logger, job_num, fixturelib, dbpath_prefix=None):
interface.Fixture.__init__(self, logger, job_num, fixturelib, dbpath_prefix=dbpath_prefix)
def setup(self):
"""Setup the fixture."""
pass
def await_ready(self):
"""Wait for the fixture to be ready."""
pass
def teardown(self, finished=False, kill=False):
"""Teardown the fixture."""
pass
def is_running(self):
"""Check if the fixture is running."""
return False

8
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/hooks/BUILD.bazel Normal file
View File

@ -0,0 +1,8 @@
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
# TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets
py_library(
name = "all_python_files",
srcs = glob(["*.py"]),
visibility = ["//visibility:public"],
)

29
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/hooks/test_hook.py Normal file
View File

@ -0,0 +1,29 @@
"""Test hook for external module."""
from buildscripts.resmokelib.testing.hooks import interface
class TestExternalHook(interface.Hook):
"""A simple test hook for external module testing."""
REGISTERED_NAME = "TestExternalHook"
IS_BACKGROUND = False
def __init__(self, hook_logger, fixture, description):
interface.Hook.__init__(self, hook_logger, fixture, description)
def before_suite(self, test_report):
"""Run before the suite."""
pass
def after_suite(self, test_report, teardown_flag=None):
"""Run after the suite."""
pass
def before_test(self, test, test_report):
"""Run before each test."""
pass
def after_test(self, test, test_report):
"""Run after each test."""
pass

5
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/matrix_suites/mappings/test_external_matrix.yml Normal file
View File

@ -0,0 +1,5 @@
base_suite: core
description: Test matrix suite from external module that references built-in core suite
overrides:
- "test_override.external_test"
- "test_override.external_env"

15
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/matrix_suites/overrides/test_override.yml Normal file
View File

@ -0,0 +1,15 @@
- name: external_test
value:
executor:
config:
shell_options:
global_vars:
TestData:
externalModuleTest: true
- name: external_env
value:
executor:
config:
env_vars:
TEST_EXTERNAL_MODULE: "1"

14
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/suites/pytest_env_vars.yml Normal file
View File

@ -0,0 +1,14 @@
# Example pytest suite showing how to define environment variables
# For Python tests, env_vars can be defined directly in executor.config
test_kind: py_test
selector:
roots:
- buildscripts/tests/**/test_*.py
executor:
config:
env_vars:
PYTEST_VAR_1: "pytest_value1"
PYTEST_VAR_2: "pytest_value2"

21
buildscripts/tests/resmoke_end2end/test_external_module_fixtures_hooks/suites/test_env_vars.yml Normal file
View File

@ -0,0 +1,21 @@
# Example suite showing how to define environment variables for tests
# Environment variables are defined in executor.config.env_vars
# This works for all test types (js_test, py_test, cpp_unittest, etc.)
test_kind: js_test
selector:
roots:
- jstests/core/query/find/find1.js
executor:
config:
env_vars:
TEST_VAR_1: "value1"
TEST_VAR_2: "value2"
shell_options:
global_vars:
TestData:
externalModuleTest: true
fixture:
class: MongoDFixture

19
buildscripts/tests/resmokelib/test_bazel_suite_parser.py
View File

@ -138,10 +138,9 @@ class TestIdentifierResolution(unittest.TestCase):
content = 'load("//jstests/suites:selectors.bzl", "sharding_srcs", "core_srcs")'
result = _parse_load_statements(content, "buildscripts/resmokeconfig")
self.assertEqual(
result["sharding_srcs"], os.path.join("jstests", "suites", "selectors.bzl")
)
self.assertEqual(result["core_srcs"], os.path.join("jstests", "suites", "selectors.bzl"))
expected_path = os.path.join(os.getcwd(), "jstests", "suites", "selectors.bzl")
self.assertEqual(result["sharding_srcs"], expected_path)
self.assertEqual(result["core_srcs"], expected_path)
def test_parse_load_statements_multiple_loads(self):
"""Test parsing multiple load statements."""
@ -152,18 +151,20 @@ load("//jstests/core:tests.bzl", "core_tests")
result = _parse_load_statements(content, "buildscripts/resmokeconfig")
self.assertEqual(
result["sharding_srcs"], os.path.join("jstests", "suites", "selectors.bzl")
result["sharding_srcs"], os.path.join(os.getcwd(), "jstests", "suites", "selectors.bzl")
)
self.assertEqual(
result["core_tests"], os.path.join(os.getcwd(), "jstests", "core", "tests.bzl")
)
self.assertEqual(result["core_tests"], os.path.join("jstests", "core", "tests.bzl"))
def test_parse_load_statements_relative_path(self):
"""Test parsing load statement with relative path."""
content = 'load(":local_defs.bzl", "local_srcs")'
result = _parse_load_statements(content, "buildscripts/resmokeconfig")
self.assertEqual(
result["local_srcs"], os.path.join("buildscripts", "resmokeconfig", "local_defs.bzl")
)
# Package-relative path: :local_defs.bzl resolves relative to package directory
expected_path = os.path.join("buildscripts", "resmokeconfig", "local_defs.bzl")
self.assertEqual(result["local_srcs"], expected_path)
def test_resolve_identifier_simple(self):
"""Test resolving a simple identifier to list of labels."""