mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-104261: Add option to select which strategies to run (#35342)

Browse Source 
GitOrigin-RevId: 816492466d3f123d3a0cb4a9953c811f14398afc
This commit is contained in:
Steve McClure 2025-04-24 12:09:28 -04:00 committed by MongoDB Bot
parent 7860cb1180
commit 51235eb7a1
9 changed files with 109 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
buildscripts/resmokelib/config.py
View File

@ -118,6 +118,7 @@ DEFAULTS = {
"majority_read_concern": "on",
"enable_enterprise_tests": "on",
"enable_evergreen_api_test_selection": False,
"test_selection_strategies_array": None,
"shell_seed": None,
"storage_engine": "wiredTiger",
"storage_engine_cache_size_gb": None,
@ -353,6 +354,9 @@ ENABLE_ENTERPRISE_TESTS = None
# if set, enables test selection using the Evergreen API
ENABLE_EVERGREEN_API_TEST_SELECTION = None
# If set, requests Evergreen to use the specified test selection strategies.
EVERGREEN_TEST_SELECTION_STRATEGY = None
# URL to connect to the Evergreen service.
EVERGREEN_URL = None

1
buildscripts/resmokelib/configure_resmoke.py
View File

@ -438,6 +438,7 @@ be invoked as either:
_config.MULTIVERSION_BIN_VERSION = config.pop("old_bin_version")
_config.ENABLE_EVERGREEN_API_TEST_SELECTION = config.pop("enable_evergreen_api_test_selection")
_config.EVERGREEN_TEST_SELECTION_STRATEGY = config.pop("test_selection_strategies_array")
_config.INSTALL_DIR = config.pop("install_dir")
if values.command == "run" and _config.INSTALL_DIR is None:

7
buildscripts/resmokelib/run/__init__.py
View File

@ -1696,6 +1696,13 @@ class RunPlugin(PluginInterface):
help="Enable test selection using the Evergreen API",
)
parser.add_argument(
"--evergreenTestSelectionStrategy",
dest="test_selection_strategies_array",
action="append",
help="Specify test selection strategy. Can be specified multiple times.",
)
configure_resmoke.add_otel_args(parser)
mongodb_server_options = parser.add_argument_group(

40
buildscripts/resmokelib/testing/suite.py
View File

@ -111,8 +111,8 @@ class Suite(object):
self._tests, self._excluded = self._get_tests_for_kind(self.test_kind)
return self._excluded
def _get_tests_for_kind(self, test_kind):
"""Return the tests to run based on the 'test_kind'-specific filtering policy."""
def _get_tests_for_kind(self, test_kind) -> tuple[List[any], List[str]]:
"""Return the tests to run and those that were excluded, based on the 'test_kind'-specific filtering policy."""
selector_config = self.get_selector_config()
# The mongos_test doesn't have to filter anything, the selector_config is just the
@ -131,32 +131,30 @@ class Suite(object):
# 3. Test selection is enabled
if tests and _config.EVERGREEN_TASK_ID and _config.ENABLE_EVERGREEN_API_TEST_SELECTION:
evg_api = evergreen_conn.get_evergreen_api()
test_selection_strategy = str(_config.EVERGREEN_TEST_SELECTION_STRATEGY)
request = {
"project_id": str(_config.EVERGREEN_PROJECT_NAME),
"build_variant": str(_config.EVERGREEN_VARIANT_NAME),
"requester": str(_config.EVERGREEN_REQUESTER),
"task_id": str(_config.EVERGREEN_TASK_ID),
"task_name": str(_config.EVERGREEN_TASK_NAME),
"tests": tests,
"strategies": test_selection_strategy,
}
try:
result = evg_api.select_tests(
str(_config.EVERGREEN_PROJECT_NAME),
str(_config.EVERGREEN_VARIANT_NAME),
str(_config.EVERGREEN_REQUESTER),
str(_config.EVERGREEN_TASK_ID),
str(_config.EVERGREEN_TASK_NAME),
tests,
)
result = evg_api.select_tests(**request)
except Exception as ex:
print(
"ERROR: failure using the select tests evergreen endpoint with the following arguments:"
)
print(f"Project name: {str(_config.EVERGREEN_PROJECT_NAME)}")
print(f"Variant name: {str(_config.EVERGREEN_VARIANT_NAME)}")
print(f"Requester: {str(_config.EVERGREEN_REQUESTER)}")
print(f"Task ID: {str(_config.EVERGREEN_TASK_ID)}")
print(f"Task name: {str(_config.EVERGREEN_TASK_NAME)}")
print(f"Tests: {tests}")
raise ex
message = f"Failure using the select tests evergreen endpoint with the following request:\n{request}"
raise RuntimeError(message) from ex
evergreen_filtered_tests = result["tests"]
evergreen_excluded_tests = set(evergreen_filtered_tests).symmetric_difference(
set(tests)
)
print(f"Evergreen excluded the following tests: {evergreen_excluded_tests}")
print(
f"Evergreen applied the following test selection strategies: {test_selection_strategy}"
)
print(f"to exclude the following tests: {evergreen_excluded_tests}")
excluded.extend(evergreen_excluded_tests)
tests = evergreen_filtered_tests

65
buildscripts/tests/resmokelib/testing/test_suite.py
View File

@ -48,3 +48,68 @@ class TestNumJobsToStart(unittest.TestCase):
under_test._config.JOBS = 100
under_test._config.REPEAT_TESTS = num_repeat
self.assertEqual(self.num_tests * num_repeat, self.suite.get_num_jobs_to_start())
class TestGetTestsForKind(unittest.TestCase):
def setUp(self):
self.suite = under_test.Suite("suite_name", {"test_kind": "js_test"})
self.suite._tests = ["t/test1.js", "t/test2.js", "t/test3.js"]
self.suite._suite_config = {
"selector": {"roots": ["testroot/**"]},
"include_files": ["testroot/test1.js, testroot/test2.js", "testroot/test3.js"],
}
self.default_evergreen_task_id = under_test._config.EVERGREEN_TASK_ID
self.default_enable_evergreen_api_test_selection = (
under_test._config.ENABLE_EVERGREEN_API_TEST_SELECTION
)
self.default_evergreen_test_selection_strategy = (
under_test._config.EVERGREEN_TEST_SELECTION_STRATEGY
)
self.default_evergreen_project_name = under_test._config.EVERGREEN_PROJECT_NAME
self.default_evergreen_variant_name = under_test._config.EVERGREEN_VARIANT_NAME
self.default_evergreen_requester = under_test._config.EVERGREEN_REQUESTER
self.default_evergreen_task_id = under_test._config.EVERGREEN_TASK_ID
self.default_evergreen_task_name = under_test._config.EVERGREEN_TASK_NAME
def tearDown(self):
under_test._config.ENABLE_EVERGREEN_API_TEST_SELECTION = (
self.default_enable_evergreen_api_test_selection
)
under_test._config.EVERGREEN_PROJECT_NAME = self.default_evergreen_project_name
under_test._config.EVERGREEN_VARIANT_NAME = self.default_evergreen_variant_name
under_test._config.EVERGREEN_REQUESTER = self.default_evergreen_requester
under_test._config.EVERGREEN_TASK_ID = self.default_evergreen_task_id
under_test._config.EVERGREEN_TASK_NAME = self.default_evergreen_task_name
under_test._config.EVERGREEN_TEST_SELECTION_STRATEGY = (
self.default_evergreen_test_selection_strategy
)
def test_simple(self):
self.assertFalse(under_test._config.ENABLE_EVERGREEN_API_TEST_SELECTION)
tests, excluded = self.suite._get_tests_for_kind("js_test")
self.assertEqual(tests, ["testroot"])
self.assertEqual(excluded, [])
def test_with_test_selection_strategy(self):
under_test._config.ENABLE_EVERGREEN_API_TEST_SELECTION = True
under_test._config.EVERGREEN_PROJECT_NAME = "project_name"
under_test._config.EVERGREEN_VARIANT_NAME = "variant_name"
under_test._config.EVERGREEN_REQUESTER = "requester"
under_test._config.EVERGREEN_TASK_ID = "task_id"
under_test._config.EVERGREEN_TASK_NAME = "task_name"
under_test._config.EVERGREEN_TEST_SELECTION_STRATEGY = "strategy"
# Currently raises an exception because the Evergreen endpoint is still under development
with self.assertRaises(RuntimeError) as context:
self.suite._get_tests_for_kind("js_test")
self.assertEqual(
str(context.exception),
"Failure using the select tests evergreen endpoint with the following request:\n"
+ "{'project_id': 'project_name', 'build_variant': 'variant_name', 'requester': 'requester', 'task_id': 'task_id', 'task_name': 'task_name', 'tests': ['testroot'], 'strategies': 'strategy'}",
)
self.assertEqual(
str(context.exception.__cause__),
"400 Client Error: Bad Request for url: https://evergreen.mongodb.com/rest/v2/select/tests",
)

2
etc/evergreen.yml
View File

@ -99,6 +99,8 @@ parameters:
- key: enable_evergreen_api_test_selection
value: false
description: "Enable test selection using Evergreen API"
- key: test_selection_strategies_array
description: "Array of test selection strategies"
commit_queue_aliases:
- variant: "commit-queue"

6
evergreen/resmoke_tests_execute.sh
View File

@ -129,6 +129,12 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then
extra_args="$extra_args --enableEvergreenApiTestSelection"
fi
# Split comma separated list of strategies
IFS=',' read -a strategies <<< "$test_selection_strategies_array"
for strategy in "${strategies[@]}"; do
extra_args+=" --evergreenTestSelectionStrategy=${strategy}"
done
path_value="$PATH:/data/multiversion"
# Set the suite name to be the task name by default; unless overridden with the `suite` expansion.

8
poetry.lock generated
View File

@ -903,15 +903,15 @@ typing-extensions = ">=3.10.0"
[[package]]
name = "evergreen-py"
version = "3.10.6"
version = "3.11.1"
description = "Python client for the Evergreen API"
optional = false
python-versions = "<4.0,>=3.9"
groups = ["testing"]
markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\""
files = [
{file = "evergreen_py-3.10.6-py3-none-any.whl", hash = "sha256:b67fa81df2ec082263647f203a4d34e363eb696a81471c7372f3b18a86ef1370"},
{file = "evergreen_py-3.10.6.tar.gz", hash = "sha256:ef0e57d3f7cb0d7cdbf6418906443485f162a16871b77327943f505cc46068ee"},
{file = "evergreen_py-3.11.1-py3-none-any.whl", hash = "sha256:057ac41cffc6d6ca6f84ea1bd0365da15a869e0fe4b36f87cefa4516a34b1e16"},
{file = "evergreen_py-3.11.1.tar.gz", hash = "sha256:6e334a94dfdca3151185dc43c646bb1acd0f7984fd05a35cdf5dc0a7defe7b8c"},
]
[package.dependencies]
@ -5377,4 +5377,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<4.0"
content-hash = "50b707c58dd8e0ba2895852418ccf797905c5600618df6a2b673b926f2340d11"
content-hash = "6a229234b161f0731b4eb689cc18116538310e4ec0b977f3e816d4bae5a888ef"

2
pyproject.toml
View File

@ -134,7 +134,7 @@ cryptography = "^44.0.2"
[tool.poetry.group.testing.dependencies]
curatorbin = "^1.2.4"
PyKMIP = "0.10.0"
evergreen-py = "^3.9.0"
evergreen-py = "^3.11.0"
mock = "^5.1.0"
shrub-py = "^3.1.4"
ocspresponder = "^0.5.0"