diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py index 2c89c8d374c..daaa81da9de 100644 --- a/buildscripts/resmokelib/__init__.py +++ b/buildscripts/resmokelib/__init__.py @@ -7,6 +7,7 @@ from buildscripts.resmokelib import ( parser, reportfile, sighandler, + suite_hierarchy, suitesconfig, testing, utils, @@ -20,6 +21,7 @@ __all__ = [ "reportfile", "sighandler", "suitesconfig", + "suite_hierarchy", "testing", "utils", ] diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 30767a887b3..9aa8b1c805e 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -66,6 +66,7 @@ DEFAULTS = { "dry_run": None, "exclude_with_any_tags": None, "force_excluded_tests": False, + "skip_tests_covered_by_more_complex_suites": False, "fuzz_mongod_configs": None, "fuzz_runtime_params": None, "fuzz_runtime_stress": "off", @@ -401,6 +402,9 @@ EXCLUDE_WITH_ANY_TAGS = None # Allow test files passed as positional args to run even if they are excluded on the suite config. FORCE_EXCLUDED_TESTS = None +# Only run tests on the given suite that will not be run on a more complex suite. +SKIP_TESTS_COVERED_BY_MORE_COMPLEX_SUITES = None + # A tag which is implicited excluded. This is useful for temporarily disabling a test. EXCLUDED_TAG = "__TEMPORARILY_DISABLED__" diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index f75d5f77a09..593abb49536 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -609,6 +609,10 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py # Force invalid suite config _config.FORCE_EXCLUDED_TESTS = config.pop("force_excluded_tests") + _config.SKIP_TESTS_COVERED_BY_MORE_COMPLEX_SUITES = config.pop( + "skip_tests_covered_by_more_complex_suites" + ) + # Archival options. Archival is enabled only when running on evergreen. if not _config.EVERGREEN_TASK_ID: _config.ARCHIVE_FILE = None diff --git a/buildscripts/resmokelib/discovery/__init__.py b/buildscripts/resmokelib/discovery/__init__.py index a0badae46ec..7eecbfa3686 100644 --- a/buildscripts/resmokelib/discovery/__init__.py +++ b/buildscripts/resmokelib/discovery/__init__.py @@ -99,6 +99,15 @@ class DiscoveryPlugin(PluginInterface): TEST_DISCOVERY_SUBCOMMAND, help="Discover what tests are run by a suite." ) parser.add_argument("--suite", metavar="SUITE", help="Suite to run against.") + parser.add_argument( + "--skipTestsCoveredByMoreComplexSuites", + dest="skip_tests_covered_by_more_complex_suites", + action="store_true", + help=( + "Excludes tests from running on some suite_A if a more complex" + " suite_A_B will also run the same tests." + ), + ) parser = subparsers.add_parser( SUITECONFIG_SUBCOMMAND, help="Display configuration of a test suite." diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index ead3075be76..e06a1c33d23 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -1210,6 +1210,16 @@ class RunPlugin(PluginInterface): ), ) + parser.add_argument( + "--skipTestsCoveredByMoreComplexSuites", + dest="skip_tests_covered_by_more_complex_suites", + action="store_true", + help=( + "Excludes tests from running on some suite_A if a more complex" + " suite_A_B will also run the same tests." + ), + ) + parser.add_argument( "--genny", dest="genny_executable", diff --git a/buildscripts/resmokelib/suite_hierarchy.py b/buildscripts/resmokelib/suite_hierarchy.py new file mode 100644 index 00000000000..cc2f8d30a69 --- /dev/null +++ b/buildscripts/resmokelib/suite_hierarchy.py @@ -0,0 +1,159 @@ +# Represents a DAG describing the complexity of suites. As we go deeper into the graph, +# suites get simpler. For example, suppose we have suite_A and suite_A_B, where suite_A_B +# tests a strict superset of *features* (not tests) tested by suite_A. Then, the graph +# would contain: +# { +# suite_A_B: { +# suite_A: {} +# } +# } +# +# But suppose feature_B changed the interaction with the server fundamentally - for example, +# if feature_B was retryable writes - then in this case suite_A_B would not be a strict superset +# of the features tested by suite_A. In this case, suite_A_B cannot be considered more complex +# than suite_A; the two are incomparable. +SUITE_HIERARCHY = { + # Concurrency suites + "concurrency": {}, + "concurrency_compute_mode": {}, + "concurrency_embedded_router_causal_consistency_and_balancer": { + "concurrency_embedded_router_causal_consistency": {} + }, + "concurrency_embedded_router_clusterwide_ops_add_remove_shards": {}, + "concurrency_embedded_router_local_read_write_multi_stmt_txn_with_balancer": { + "concurrency_embedded_router_local_read_write_multi_stmt_txn": {} + }, + "concurrency_embedded_router_multi_stmt_txn_with_balancer": { + # concurrency_embedded_multi_stmt_txn is not considered a superset in terms + # of complexity of concurrency_embedded_router_replication because multi_stmt_txns + # are not a superset feature of regular operations. + "concurrency_embedded_router_multi_stmt_txn": {} + }, + "concurrency_embedded_router_replication_with_balancer": { + "concurrency_embedded_router_replication": {}, + }, + "concurrency_multitenancy_replication_with_atlas_proxy": {}, + "simulate_crash_concurrency_replication": {}, + "concurrency_sharded_replication_with_balancer_and_config_transitions_and_add_remove_shard": { + "concurrency_sharded_with_balancer_and_auto_bootstrap": {} + }, + "concurrency_sharded_replication_with_balancer_and_config_transitions": { + # The auto_bootstrap suites maintain a static config shard and so the config_transitions + # suite is a superset of it because it also transitions the config shard to a dedicated + # replica set. + "concurrency_sharded_with_balancer_and_auto_bootstrap": { + "concurrency_sharded_with_auto_bootstrap": {} + } + }, + "concurrency_sharded_causal_consistency_and_balancer": { + "concurrency_sharded_causal_consistency": {} + }, + "concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer": { + "concurrency_sharded_local_read_write_multi_stmt_txn": {} + }, + "concurrency_sharded_multi_stmt_txn_with_balancer_and_config_transitions": { + "concurrency_sharded_multi_stmt_txn_with_balancer": { + "concurrency_sharded_multi_stmt_txn": {} + } + }, + "concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary": { + "concurrency_sharded_multi_stmt_txn": {} + }, + "concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions": { + "concurrency_sharded_stepdown_terminate_kill_primary_with_balancer": { + # The stepdown suite is not considered a superset of concurrency_sharded_replication + # because the stepdown suite uses retryable writes whereas the vanilla suite does not. + # Therefore the commands being sent to the server are fundamentally different. + "concurrency_sharded_with_stepdowns": {} + } + }, + "concurrency_sharded_clusterwide_ops_add_remove_shards": {}, + "concurrency_replication_causal_consistency": {}, + "concurrency_replication_causal_consistency_with_replica_set_endpoint": {}, + "concurrency_replication_for_backup_restore": {}, + "concurrency_replication_for_export_import": {}, + "concurrency_replication_multi_stmt_txn": {}, + "concurrency_replication_multi_stmt_txn_with_replica_set_endpoint": {}, + "concurrency_replication_with_replica_set_endpoint": {}, + "concurrency_replication": {}, + "concurrency_sharded_initial_sync": {"concurrency_sharded_causal_consistency": {}}, + # JScore passthrough suites +} + + +def compute_dag(complexity_graph): + """ + Computes a graph of the below form from the nested complexity graph. + { + node: { parents: set(...), children: set(...) }, + ... + } + where the parents are the direct parents and children are direct + children. Note that if the original nested graph had multiple paths connecting + a node to another (A->B->C and A->C are both edges), then C would have both + B and A as its direct parents, and A would have B and C as its direct children + """ + graph = {} + + # Initially place all the known ancestor nodes in the frontier. + frontier = [] + for ancestor, descendants in complexity_graph.items(): + frontier.append((ancestor, descendants)) + + while frontier: + parent, children = frontier.pop(0) + if parent not in graph: + graph[parent] = {"parents": set(), "children": set()} + + for child, grandchildren in children.items(): + if child not in graph: + graph[child] = {"parents": set(), "children": set()} + graph[parent]["children"].add(child) + graph[child]["parents"].add(parent) + + frontier.append((child, grandchildren)) + + return graph + + +def compute_ancestors(node, dag): + """Returns all the ancestor, direct and indirect, of the given node.""" + ancestors = set() + + frontier = [node] + while frontier: + node = frontier.pop(0) + ancestors = ancestors.union(dag[node]["parents"]) + frontier += list(dag[node]["parents"]) + + return ancestors + + +def compute_minimal_test_set(suite_name, dag, tests_in_suite): + """ + Given a DAG that represents which suite is more complex than + another suite, and the set of tests that are usually run in each suite, + returns the minimal set of tests that need to be run for the given suite. + suite_name: the suite for which we want to calculate the minimal test set. + dag: a dictionary of dictionaries of the form: + { + "node_N" : { + "parents": set(["node_i", ...]), + "children": set(["node_k", ...]) + } + } + tests_in_suite: a dictionary of suite_name -> set(tests) that can be run in the suite. + """ + + # To calculate the minimal set for a suite 'curr_suite': + # 1) Figure out who all the ancestors of 'curr_suite' are. + # 2) Subtract from curr_suite's test set the union of the test sets of all its ancestors. + + ancestors = compute_ancestors(suite_name, dag) + # Copy the given set + curr_test_set = set(tests_in_suite[suite_name]) + + for ancestor in ancestors: + curr_test_set -= tests_in_suite[ancestor] + + return curr_test_set diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py index fd6ed36aa20..2e73666b2d5 100644 --- a/buildscripts/resmokelib/suitesconfig.py +++ b/buildscripts/resmokelib/suitesconfig.py @@ -12,7 +12,7 @@ import yaml import buildscripts.resmokelib.utils.filesystem as fs from buildscripts.resmokelib import config as _config -from buildscripts.resmokelib import errors, utils +from buildscripts.resmokelib import errors, suite_hierarchy, utils from buildscripts.resmokelib.logging import loggers from buildscripts.resmokelib.testing import suite as _suite from buildscripts.resmokelib.utils import load_yaml_file @@ -120,6 +120,7 @@ def get_suites(suite_names_or_paths: list[str], test_files: list[str]) -> List[_ for suite_filename in suite_names_or_paths: suite_config = _get_suite_config(suite_filename) suite = _suite.Suite(suite_filename, suite_config) + if suite_roots: # Override the suite's default test files with those passed in from the command line. override_suite_config = suite_config.copy() @@ -146,10 +147,53 @@ def get_suites(suite_names_or_paths: list[str], test_files: list[str]) -> List[_ f"'{test}' excluded in '{suite.get_name()}'" ) suite = override_suite + + if _config.SKIP_TESTS_COVERED_BY_MORE_COMPLEX_SUITES: + if suite_roots: + raise ValueError( + "Cannot use '--skipTestsCoveredByMoreComplexSuites' when tests have been passed in from the command line." + ) + if _config.ORIGIN_SUITE: + raise ValueError( + "Cannot use '--originSuite' with '--skipTestsCoveredByMoreComplexSuites'." + ) + suite = _compute_minimal_test_set_suite(suite_filename) + suites.append(suite) return suites +def _compute_minimal_test_set_suite(origin_suite_name): + """ + Given a suite_A, returns the set of tests compatible with it, but incompatible + with any suite_A_B more complex than it. + + The relationship of "more complex" is defined in suite_hierarchy.py. + """ + + # Compute the dag from the complexity graph + dag = suite_hierarchy.compute_dag(suite_hierarchy.SUITE_HIERARCHY) + + # If we don't know how to compute the minimal test set because the suite's + # information isn't present in the hierarchy, just return the suite as is. + if origin_suite_name not in dag: + suite_config = _get_suite_config(origin_suite_name) + suite = _suite.Suite(origin_suite_name, suite_config) + return suite + + # Build a dictionary of the tests in each suite. + tests_in_suite = {} + for suite_in_dag in dag.keys(): + suite_config = _get_suite_config(suite_in_dag) + suite = _suite.Suite(suite_in_dag, suite_config) + tests_in_suite[suite_in_dag] = set(suite.tests) + + tests = suite_hierarchy.compute_minimal_test_set(origin_suite_name, dag, tests_in_suite) + min_suite_config = _get_suite_config(origin_suite_name).copy() + min_suite_config.update(_make_suite_roots(list(tests))) + return _suite.Suite(origin_suite_name, min_suite_config) + + def _make_suite_roots(files): return {"selector": {"roots": files}} @@ -527,5 +571,4 @@ class SuiteFinder(object): def get_suite(suite_name_or_path) -> _suite.Suite: """Retrieve the Suite instance corresponding to a suite configuration file.""" - suite_config = _get_suite_config(suite_name_or_path) - return _suite.Suite(suite_name_or_path, suite_config) + return get_suites([suite_name_or_path], None)[0] diff --git a/buildscripts/tests/resmokelib/test_suite_hierarchy.py b/buildscripts/tests/resmokelib/test_suite_hierarchy.py new file mode 100644 index 00000000000..9c46e364e3c --- /dev/null +++ b/buildscripts/tests/resmokelib/test_suite_hierarchy.py @@ -0,0 +1,224 @@ +"""Unit tests for buildscripts/resmokelib/suite_hierarchy.py.""" + +import unittest + +from buildscripts.resmokelib.suite_hierarchy import ( + compute_ancestors, + compute_dag, + compute_minimal_test_set, +) + +tests_in_suite = { + "A": set(["t1", "t2", "t3"]), + "B": set(["t1", "t2", "t3", "t4"]), + "C": set(["t1", "t2", "t3", "t5"]), + "D": set(["t2", "t3", "t5", "t6"]), + "E": set(["t1", "t2", "t3", "t7"]), + "P": set( + [ + "t1", + ] + ), + "Q": set(["t1", "t7"]), +} + +# Graph 1: +# A P +# | | +# | V +# | Q +# | | +# V | +# B <-----+ +# | +# +-------+ +# | | +# V V +# C D +# | +# V +# E +# +# Different equivalent ways of representing Graph 1 +graph1 = [ + {"A": {"B": {"C": {"E": {}}, "D": {}}}, "P": {"Q": {"B": {}}}}, + { + "A": { + "B": {}, + }, + "P": {"Q": {"B": {}}}, + "B": {"C": {"E": {}}, "D": {}}, + }, + {"A": {"B": {"C": {"E": {}}, "D": {}}}, "P": {"Q": {}}, "Q": {"B": {}}}, + { + "A": {"B": {}}, + "B": {"C": {}, "D": {}}, + "C": {"E": {}}, + "D": {}, + "P": {"Q": {}}, + "Q": {"B": {}}, + }, +] + +correct_dag1 = { + "A": {"parents": set(), "children": set(["B"])}, + "B": {"parents": set(["A", "Q"]), "children": set(["C", "D"])}, + "C": {"parents": set(["B"]), "children": set(["E"])}, + "D": {"parents": set(["B"]), "children": set()}, + "E": {"parents": set(["C"]), "children": set()}, + "P": {"parents": set(), "children": set(["Q"])}, + "Q": {"parents": set(["P"]), "children": set(["B"])}, +} + +correct_ancestors1 = [ + ("A", set()), + ("B", set(["A", "P", "Q"])), + ("C", set(["A", "P", "Q", "B"])), + ("D", set(["A", "P", "Q", "B"])), + ("E", set(["A", "P", "Q", "B", "C"])), + ("P", set()), + ("Q", set(["P"])), +] + +# Minimal set for graph 1 +correct_minimal_set1 = { + "A": set(["t1", "t2", "t3"]), + "B": set(["t4"]), + "C": set(["t5"]), + "D": set(["t5", "t6"]), + "E": set(), + "P": set( + [ + "t1", + ] + ), + "Q": set(["t7"]), +} + +# Graph 2: +# A--+ P (disconnected) +# | | +# | | +# | | Q (disconnected) +# | | +# V | +# B | +# | | +# | | +# | | +# V | +# C <+ +# +# Different equivalent ways of representing Graph 2 +graph2 = [ + {"A": {"B": {"C": {}}, "C": {}}, "P": {}, "Q": {}}, + {"B": {"C": {}}, "A": {"B": {}, "C": {}}, "P": {}, "Q": {}}, +] + +correct_dag2 = { + "A": {"parents": set(), "children": set(["B", "C"])}, + "B": {"parents": set(["A"]), "children": set(["C"])}, + "C": {"parents": set(["A", "B"]), "children": set()}, + "P": {"parents": set(), "children": set()}, + "Q": {"parents": set(), "children": set()}, +} + +correct_ancestors2 = [ + ("A", set()), + ("B", set(["A"])), + ("C", set(["A", "B"])), + ("P", set()), + ("Q", set()), +] + +# Minimal set for graph 2 +correct_minimal_set2 = { + "A": set(["t1", "t2", "t3"]), + "B": set(["t4"]), + "C": set(["t5"]), + "P": set(["t1"]), + "Q": set(["t1", "t7"]), +} + + +class TestSuiteHierarchy(unittest.TestCase): + def test_compute_dag_empty(self): + correct_dag = {} + self.assertEqual(correct_dag, compute_dag({})) + + def test_compute_dag_graph1(self): + for graph in graph1: + dag = compute_dag(graph) + self.assertEqual( + correct_dag1, + dag, + f"Expected: \n{correct_dag1}\nbut received\n{dag}.\nTest case:\n{graph}", + ) + + def test_compute_dag_graph2(self): + for graph in graph2: + dag = compute_dag(graph) + self.assertEqual( + correct_dag2, + dag, + f"Expected: \n{correct_dag2}\nbut received\n{dag}.\nTest case:\n{graph}", + ) + + def test_compute_ancestors_graph1(self): + for node, answer in correct_ancestors1: + ancestors = compute_ancestors(node, correct_dag1) + self.assertEqual(answer, ancestors, f"Expected\n{answer}\nbut received\n{ancestors}") + + def test_compute_ancestors_graph2(self): + for node, answer in correct_ancestors2: + ancestors = compute_ancestors(node, correct_dag2) + self.assertEqual(answer, ancestors, f"Expected\n{answer}\nbut received\n{ancestors}") + + def test_compute_minimal_test_set1(self): + for node, tests in correct_minimal_set1.items(): + computed_tests = compute_minimal_test_set(node, correct_dag1, tests_in_suite) + self.assertEqual( + tests, + computed_tests, + f"On node {node}\nExpected\n{tests}\nbut received\n{computed_tests}", + ) + + def test_compute_minimal_test_set2(self): + for node, tests in correct_minimal_set2.items(): + computed_tests = compute_minimal_test_set(node, correct_dag2, tests_in_suite) + self.assertEqual( + tests, + computed_tests, + f"On node {node}\nExpected\n{tests}\nbut received\n{computed_tests}", + ) + + def test_union_of_minimal_equals_union_of_full(self): + # Test that the union(minimal set of each ancestor) is equal + # to the union(full test set of each ancestor) + for node, ancestors in correct_ancestors1: + min_set_union = set() + full_union = set() + for ancestor in ancestors: + min_set_union = min_set_union.union( + compute_minimal_test_set(ancestor, correct_dag1, tests_in_suite) + ) + full_union = full_union.union(tests_in_suite[ancestor]) + self.assertEqual( + min_set_union, + full_union, + f"Min set union\n{min_set_union}\n != full union\n{full_union}", + ) + + for node, ancestors in correct_ancestors2: + min_set_union = set() + full_union = set() + for ancestor in ancestors: + min_set_union = min_set_union.union( + compute_minimal_test_set(ancestor, correct_dag2, tests_in_suite) + ) + full_union = full_union.union(tests_in_suite[ancestor]) + self.assertEqual( + min_set_union, + full_union, + f"Min set union\n{min_set_union}\n != full union\n{full_union}", + ) diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh index 343942139a3..21682f70ee2 100644 --- a/evergreen/prelude_mongo_task_generator.sh +++ b/evergreen/prelude_mongo_task_generator.sh @@ -1,6 +1,6 @@ function setup_mongo_task_generator { if [ ! -f mongo-task-generator ]; then - curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.17/mongo-task-generator --output mongo-task-generator + curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.18/mongo-task-generator --output mongo-task-generator chmod +x mongo-task-generator fi }