diff --git a/bazel/resmoke/BUILD.bazel b/bazel/resmoke/BUILD.bazel index 165044c5221..7a026a7cc8e 100644 --- a/bazel/resmoke/BUILD.bazel +++ b/bazel/resmoke/BUILD.bazel @@ -64,6 +64,17 @@ genrule( visibility = ["//visibility:public"], ) +genrule( + name = "all_ifr_flags", + srcs = [], + outs = ["all_ifr_flags.txt"], + cmd = """ + awk '/^STABLE_all_ifr_flags/ { for (i=2; i<=NF; i++) print $$i }' bazel-out/stable-status.txt > $@ + """, + stamp = True, + visibility = ["//visibility:public"], +) + # We want to use keys from volatile-status in the resmoke `py_test`, so this creates a file that # it can depend on. genrule( diff --git a/bazel/resmoke/resmoke.bzl b/bazel/resmoke/resmoke.bzl index a80a2f43810..56d6f10aa0d 100644 --- a/bazel/resmoke/resmoke.bzl +++ b/bazel/resmoke/resmoke.bzl @@ -227,6 +227,7 @@ def resmoke_suite_test( "//bazel/resmoke:on_feature_flags", "//bazel/resmoke:off_feature_flags", "//bazel/resmoke:unreleased_ifr_flags", + "//bazel/resmoke:all_ifr_flags", "//bazel/resmoke:volatile_status", "//bazel/resmoke:resource_monitor", ":%s" % historic_runtimes, diff --git a/bazel/workspace_status.py b/bazel/workspace_status.py index 6c2e7ce6457..6e3e55b0c07 100644 --- a/bazel/workspace_status.py +++ b/bazel/workspace_status.py @@ -60,6 +60,13 @@ def print_feature_flag_status(): ] print_stable_key("unreleased_ifr_flags", " ".join(unreleased_ifr_flags)) + all_ifr_flags = [ + name + for name, flag in all_flags.items() + if idl_binder.is_incremental_feature_rollout_flag(flag) + ] + print_stable_key("all_ifr_flags", " ".join(all_ifr_flags)) + def print_evergreen_expansions(): for expansion in [ diff --git a/buildscripts/idl/gen_all_feature_flag_list.py b/buildscripts/idl/gen_all_feature_flag_list.py index 81b8a5ae1b4..14e404fe6b4 100755 --- a/buildscripts/idl/gen_all_feature_flag_list.py +++ b/buildscripts/idl/gen_all_feature_flag_list.py @@ -87,6 +87,15 @@ def get_all_unreleased_ifr_feature_flags(idl_dirs: list[str] = DEFAULT_IDL_DIRS) ] +def get_all_ifr_feature_flags(idl_dirs: list[str] = DEFAULT_IDL_DIRS): + """Generate a list of all IFR feature flags regardless of phase.""" + all_flags = lib.get_all_feature_flags(idl_dirs) + + return [ + name for name, flag in all_flags.items() if binder.is_incremental_feature_rollout_flag(flag) + ] + + def write_feature_flags_to_file(flags: list[str], filename: str): """Helper function to write feature flags to a file.""" with open(filename, "w") as output_file: diff --git a/buildscripts/idl/idl/binder.py b/buildscripts/idl/idl/binder.py index dec3537a8f9..035b720c509 100644 --- a/buildscripts/idl/idl/binder.py +++ b/buildscripts/idl/idl/binder.py @@ -1929,6 +1929,18 @@ def is_unreleased_incremental_rollout_feature_flag(feature_flag): ) +def is_incremental_feature_rollout_flag(feature_flag): + """Determine if an idl.FeatureFlag is an Incremental Feature Rollout (IFR) flag + in any phase (in_development, rollout, or released) without validating its syntax. + """ + # type: (syntax.FeatureFlag) -> bool + + if not feature_flag.incremental_rollout_phase: + return False + phase = ast.FeatureFlagRolloutPhase.bind(feature_flag.incremental_rollout_phase) + return phase is not None and phase != ast.FeatureFlagRolloutPhase.NOT_FOR_INCREMENTAL_ROLLOUT + + def bind(parsed_spec): # type: (syntax.IDLSpec) -> ast.IDLBoundSpec """Read an idl.syntax, create an idl.ast tree, and validate the final IDL Specification.""" diff --git a/buildscripts/resmokeconfig/feature_flag_test.idl b/buildscripts/resmokeconfig/feature_flag_test.idl index bbe0cdacdda..72d00765907 100644 --- a/buildscripts/resmokeconfig/feature_flag_test.idl +++ b/buildscripts/resmokeconfig/feature_flag_test.idl @@ -6,3 +6,14 @@ feature_flags: cpp_varname: gFeatureFlagToaster default: false fcv_gated: true + featureFlagTestIFR: + description: "IFR feature flag used by fixture builder unit tests" + cpp_varname: gFeatureFlagTestIFR + default: false + fcv_gated: false + incremental_rollout_phase: in_development + featureFlagTestNonIFR: + description: "Non-IFR feature flag used by fixture builder unit tests" + cpp_varname: gFeatureFlagTestNonIFR + default: false + fcv_gated: false diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index c6b2611958f..c34f43c83bd 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -584,6 +584,10 @@ DISABLED_FEATURE_FLAGS = None # List of feature flags to enable. ENABLED_FEATURE_FLAGS = [] +# Set of all IFR (Incremental Feature Rollout) feature flag names, used to filter +# IFR flags from injection in multiversion fixtures. +IFR_FEATURE_FLAGS = None + # The path to the mongo executable used by resmoke.py. MONGO_EXECUTABLE = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 570b7a9569a..17f931df3f3 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -44,6 +44,7 @@ BASE_16_TO_INT = 16 COLLECTOR_ENDPOINT = "otel-collector.prod.corp.mongodb.com:443" BAZEL_GENERATED_OFF_FEATURE_FLAGS = "bazel/resmoke/off_feature_flags.txt" BAZEL_GENERATED_UNRELEASED_IFR_FEATURE_FLAGS = "bazel/resmoke/unreleased_ifr_feature_flags.txt" +BAZEL_GENERATED_ALL_IFR_FEATURE_FLAGS = "bazel/resmoke/all_ifr_flags.txt" EVERGREEN_EXPANSIONS_FILE = "../expansions.yml" @@ -562,11 +563,17 @@ flags in common: {common_set} (default_disabled_set | disabled_feature_flags_set) - enabled_feature_flags_set ) + if os.path.exists(BAZEL_GENERATED_ALL_IFR_FEATURE_FLAGS): + all_ifr_set = set(process_feature_flag_file(BAZEL_GENERATED_ALL_IFR_FEATURE_FLAGS)) + else: + all_ifr_set = set(gen_all_feature_flag_list.get_all_ifr_feature_flags()) + return ( list(enabled_feature_flags_set), list(disabled_feature_flags_set), default_disabled_feature_flags, off_feature_flags, + all_ifr_set, ) _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests") @@ -575,6 +582,7 @@ flags in common: {common_set} _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file") _config.ENABLED_FEATURE_FLAGS = [] _config.DISABLED_FEATURE_FLAGS = [] + _config.IFR_FEATURE_FLAGS = None default_disabled_feature_flags = [] off_feature_flags = [] if values["command"] == "run": @@ -583,6 +591,7 @@ flags in common: {common_set} _config.DISABLED_FEATURE_FLAGS, default_disabled_feature_flags, off_feature_flags, + _config.IFR_FEATURE_FLAGS, ) = set_up_feature_flags() else: # Explicitly ignore these run-related options. diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index c8bbd86a491..8964dc80a36 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -401,8 +401,36 @@ def mongo_shell_program( mongo_set_parameters = test_data.get("setParametersMongo", {}).copy() feature_flag_dict = {} + # In multiversion suites, exclude IFR (Incremental Feature Rollout) flags from + # TestData.setParameters. IFR flags are excluded from resmoke-managed multiversion + # fixtures by the Python fixture builders, but TestData.setParameters is consumed by + # JS test-created clusters (ShardingTest, ReplSetTest) which don't have the same + # filtering. Including IFR flags in multiversion suites causes them to leak into + # test-created clusters where they are incompatible — even latest-version nodes must + # not activate IFR-gated features when older nodes in the same cluster cannot handle + # the resulting commands. In non-multiversion suites all servers are latest-version, + # so IFR flags are safe to include. + # + # Multiversion can be signalled either via resmoke CLI flags (MIXED_BIN_VERSIONS, + # MULTIVERSION_BIN_VERSION) or via TestData variables set in suite YAML configs + # (useRandomBinVersionsWithinReplicaSet, mongosBinVersion). Both paths must be + # checked to catch suites like sharding_last_continuous where JS tests create + # their own mixed-version clusters using TestData. + is_multiversion = ( + config.MIXED_BIN_VERSIONS is not None + or config.MULTIVERSION_BIN_VERSION is not None + or test_data.get("useRandomBinVersionsWithinReplicaSet") is not None + or test_data.get("mongosBinVersion") is not None + ) + ifr_flags = set(config.IFR_FEATURE_FLAGS or []) + + def include_flag(ff: str) -> bool: + if is_multiversion and ff in ifr_flags: + return False + return True + if config.ENABLED_FEATURE_FLAGS is not None: - feature_flag_dict = {ff: "true" for ff in config.ENABLED_FEATURE_FLAGS} + feature_flag_dict = {ff: "true" for ff in config.ENABLED_FEATURE_FLAGS if include_flag(ff)} if config.DISABLED_FEATURE_FLAGS is not None: feature_flag_dict |= {ff: "false" for ff in config.DISABLED_FEATURE_FLAGS} diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py index 1a0bb6d4a80..d0655baa64f 100644 --- a/buildscripts/resmokelib/testing/fixtures/_builder.py +++ b/buildscripts/resmokelib/testing/fixtures/_builder.py @@ -25,7 +25,9 @@ RETRIEVE_LOCK = threading.Lock() _BUILDERS = {} # type: ignore -def make_fixture(class_name, logger, job_num, *args, enable_feature_flags=True, **kwargs): +def make_fixture( + class_name, logger, job_num, *args, enable_feature_flags=True, exclude_ifr_flags=False, **kwargs +): """Provide factory function for creating Fixture instances.""" fixturelib = FixtureLib() @@ -46,6 +48,7 @@ def make_fixture(class_name, logger, job_num, *args, enable_feature_flags=True, fixturelib, *args, add_feature_flags=bool(config.ENABLED_FEATURE_FLAGS), + exclude_ifr_flags=exclude_ifr_flags, **kwargs, ) @@ -399,6 +402,7 @@ class ReplSetBuilder(FixtureBuilder): _class, mongod_logger, replset.job_num, + exclude_ifr_flags=is_multiversion, mongod_executable=executables[BinVersionEnum.NEW], mongod_options=new_fixture_mongod_options, preserve_dbpath=replset.preserve_dbpath, @@ -750,6 +754,7 @@ class ShardedClusterBuilder(FixtureBuilder): _class, mongos_logger, sharded_cluster.job_num, + exclude_ifr_flags=is_multiversion, mongos_executable=executables[BinVersionEnum.NEW], **new_fixture_mongos_kwargs, ) diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py index ade7f66e412..1ddacc119c5 100644 --- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py +++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py @@ -161,6 +161,7 @@ class _FixtureConfig(object): self.LAST_CONTINUOUS_MONGOS_BINARY = LAST_CONTINUOUS_MONGOS_BINARY self.USE_LEGACY_MULTIVERSION = config.USE_LEGACY_MULTIVERSION self.ENABLED_FEATURE_FLAGS = config.ENABLED_FEATURE_FLAGS + self.IFR_FEATURE_FLAGS = config.IFR_FEATURE_FLAGS self.EVERGREEN_TASK_ID = config.EVERGREEN_TASK_ID self.MAJORITY_READ_CONCERN = config.MAJORITY_READ_CONCERN self.NO_JOURNAL = config.NO_JOURNAL diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index e07a274fe7e..83cedb90cd4 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -957,6 +957,7 @@ class _MongoSFixture(interface.Fixture, interface._DockerComposeInterface): mongos_executable=None, mongos_options=None, add_feature_flags=False, + exclude_ifr_flags=False, use_priority_port=False, uds_path_prefix=None, ): @@ -976,7 +977,10 @@ class _MongoSFixture(interface.Fixture, interface._DockerComposeInterface): ).copy() if add_feature_flags: + ifr_flags = set(self.config.IFR_FEATURE_FLAGS or []) for ff in self.config.ENABLED_FEATURE_FLAGS: + if exclude_ifr_flags and ff in ifr_flags: + continue self.mongos_options["set_parameters"][ff] = "true" self.mongos = None diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 160707d8bec..f98769a42d7 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -36,6 +36,7 @@ class MongoDFixture(interface.Fixture, interface._DockerComposeInterface): mongod_executable: Optional[str] = None, mongod_options: Optional[dict] = None, add_feature_flags: bool = False, + exclude_ifr_flags: bool = False, dbpath_prefix: Optional[str] = None, preserve_dbpath: bool = False, port: Optional[int] = None, @@ -54,6 +55,8 @@ class MongoDFixture(interface.Fixture, interface._DockerComposeInterface): mongod_executable (Optional[str], optional): Optional path to mongod executable. Defaults to None. mongod_options (Optional[dict], optional): Optional mongod startup options. Defaults to None. add_feature_flags (bool, optional): Sets all feature flags to true when set. Defaults to False. + exclude_ifr_flags (bool, optional): When True and add_feature_flags is True, + Incremental Feature Rollout (IFR) flags are excluded from injection (used in multiversion fixtures). Defaults to False. dbpath_prefix (Optional[str], optional): Sets the dbpath_prefix. Defaults to None. preserve_dbpath (bool, optional): preserve_dbpath. Defaults to False. port (Optional[int], optional): Port to use for mongod. Defaults to None. @@ -103,7 +106,10 @@ class MongoDFixture(interface.Fixture, interface._DockerComposeInterface): self.mongod_options["set_parameters"] = {} if add_feature_flags: + ifr_flags = set(self.config.IFR_FEATURE_FLAGS or []) for ff in self.config.ENABLED_FEATURE_FLAGS: + if exclude_ifr_flags and ff in ifr_flags: + continue self.mongod_options["set_parameters"][ff] = "true" if "dbpath" in self.mongod_options and dbpath_prefix is not None: diff --git a/buildscripts/tests/resmokelib/testing/fixtures/test_builder.py b/buildscripts/tests/resmokelib/testing/fixtures/test_builder.py index 97546c55ed9..fe23bb1c686 100644 --- a/buildscripts/tests/resmokelib/testing/fixtures/test_builder.py +++ b/buildscripts/tests/resmokelib/testing/fixtures/test_builder.py @@ -160,3 +160,161 @@ class TestBuildShardedCluster(unittest.TestCase): self.assertNotIn(ff_name, sharded_cluster.shards[0].nodes[1].mongod_options[SET_PARAMS]) self.assertNotIn(ff_name, sharded_cluster.shards[1].nodes[0].mongod_options[SET_PARAMS]) self.assertNotIn(ff_name, sharded_cluster.mongos[0].mongos_options[SET_PARAMS]) + + def test_build_sharded_cluster_multiversion_excludes_ifr_flags(self): + """IFR flags should be excluded from new-binary nodes in multiversion mode. + + Uses featureFlagTestIFR and featureFlagTestNonIFR defined in + buildscripts/resmokeconfig/feature_flag_test.idl. These are discovered + by the IDL scan in --runAllFeatureFlagTests. + """ + ifr_flag = "featureFlagTestIFR" + non_ifr_flag = "featureFlagTestNonIFR" + parser.set_run_options("--runAllFeatureFlagTests", False) + + self.assertIn(ifr_flag, config.IFR_FEATURE_FLAGS) + self.assertNotIn(non_ifr_flag, config.IFR_FEATURE_FLAGS) + self.assertIn(ifr_flag, config.ENABLED_FEATURE_FLAGS) + self.assertIn(non_ifr_flag, config.ENABLED_FEATURE_FLAGS) + + fixture_config = { + "mongod_options": {SET_PARAMS: {"enableTestCommands": 1}}, + "configsvr_options": {"num_nodes": 2}, + "num_shards": 1, + "num_rs_nodes_per_shard": 2, + "mixed_bin_versions": "new_old", + "old_bin_version": "last_lts", + } + sharded_cluster = under_test.make_fixture( + self.fixture_class_name, self.mock_logger, self.job_num, **fixture_config + ) + + # Non-IFR flag IS set on new-binary configsvr nodes + self.assertIn( + non_ifr_flag, + sharded_cluster.configsvr.nodes[0].mongod_options[SET_PARAMS], + ) + self.assertIn( + non_ifr_flag, + sharded_cluster.configsvr.nodes[1].mongod_options[SET_PARAMS], + ) + # IFR flag is NOT set on new-binary configsvr nodes (multiversion) + self.assertNotIn( + ifr_flag, + sharded_cluster.configsvr.nodes[0].mongod_options[SET_PARAMS], + ) + self.assertNotIn( + ifr_flag, + sharded_cluster.configsvr.nodes[1].mongod_options[SET_PARAMS], + ) + + # Non-IFR flag IS set on new-binary shard node + self.assertIn( + non_ifr_flag, + sharded_cluster.shards[0].nodes[0].mongod_options[SET_PARAMS], + ) + # IFR flag is NOT set on new-binary shard node (multiversion) + self.assertNotIn( + ifr_flag, + sharded_cluster.shards[0].nodes[0].mongod_options[SET_PARAMS], + ) + # Neither flag is on old-binary shard node + self.assertNotIn( + ifr_flag, + sharded_cluster.shards[0].nodes[1].mongod_options[SET_PARAMS], + ) + self.assertNotIn( + non_ifr_flag, + sharded_cluster.shards[0].nodes[1].mongod_options[SET_PARAMS], + ) + + # Access the new mongos fixture inside the FixtureContainer (which starts + # with the old version in multiversion mode). + new_mongos = sharded_cluster.mongos[0]._fixtures[under_test.BinVersionEnum.NEW] + # IFR flag is NOT set on new-binary mongos (multiversion) + self.assertNotIn(ifr_flag, new_mongos.mongos_options[SET_PARAMS]) + # Non-IFR flag IS set on new-binary mongos + self.assertIn(non_ifr_flag, new_mongos.mongos_options[SET_PARAMS]) + + def test_build_sharded_cluster_non_multiversion_keeps_ifr_flags(self): + """IFR flags should be included in non-multiversion mode.""" + ifr_flag = "featureFlagTestIFR" + parser.set_run_options("--runAllFeatureFlagTests", False) + + fixture_config = {"mongod_options": {SET_PARAMS: {"enableTestCommands": 1}}} + sharded_cluster = under_test.make_fixture( + self.fixture_class_name, self.mock_logger, self.job_num, **fixture_config + ) + + # IFR flag IS set when not in multiversion mode + self.assertIn( + ifr_flag, + sharded_cluster.configsvr.nodes[0].mongod_options[SET_PARAMS], + ) + self.assertIn( + ifr_flag, + sharded_cluster.shards[0].nodes[0].mongod_options[SET_PARAMS], + ) + self.assertIn( + ifr_flag, + sharded_cluster.mongos[0].mongos_options[SET_PARAMS], + ) + + +class TestMakeFixtureIFRExclusion(unittest.TestCase): + """Test that make_fixture correctly handles exclude_ifr_flags for MongoDFixture. + + Uses featureFlagTestIFR and featureFlagTestNonIFR defined in + buildscripts/resmokeconfig/feature_flag_test.idl. + """ + + mock_logger = None + job_num = 0 + + @classmethod + def setUpClass(cls): + cls.mock_logger = MagicMock(spec=logging.Logger) + logging.loggers._FIXTURE_LOGGER_REGISTRY[cls.job_num] = cls.mock_logger + + def tearDown(self): + network.PortAllocator.reset() + + def test_mongod_exclude_ifr_flags(self): + """MongoDFixture should exclude IFR flags when exclude_ifr_flags=True.""" + ifr_flag = "featureFlagTestIFR" + non_ifr_flag = "featureFlagTestNonIFR" + parser.set_run_options("--runAllFeatureFlagTests", False) + + mongod = under_test.make_fixture( + "MongoDFixture", + self.mock_logger, + self.job_num, + exclude_ifr_flags=True, + ) + self.assertNotIn(ifr_flag, mongod.mongod_options[SET_PARAMS]) + self.assertIn(non_ifr_flag, mongod.mongod_options[SET_PARAMS]) + + def test_mongod_include_ifr_flags_when_not_excluded(self): + """MongoDFixture should include IFR flags when exclude_ifr_flags=False (default).""" + ifr_flag = "featureFlagTestIFR" + parser.set_run_options("--runAllFeatureFlagTests", False) + + mongod = under_test.make_fixture( + "MongoDFixture", + self.mock_logger, + self.job_num, + ) + self.assertIn(ifr_flag, mongod.mongod_options[SET_PARAMS]) + + def test_mongod_no_flags_when_feature_flags_disabled(self): + """MongoDFixture should not inject any flags when enable_feature_flags=False.""" + ifr_flag = "featureFlagTestIFR" + parser.set_run_options("--runAllFeatureFlagTests", False) + + mongod = under_test.make_fixture( + "MongoDFixture", + self.mock_logger, + self.job_num, + enable_feature_flags=False, + ) + self.assertNotIn(ifr_flag, mongod.mongod_options[SET_PARAMS]) diff --git a/src/mongo/db/feature_flag_test.idl.tpl b/src/mongo/db/feature_flag_test.idl.tpl index afecbb238ac..acfe636db1f 100644 --- a/src/mongo/db/feature_flag_test.idl.tpl +++ b/src/mongo/db/feature_flag_test.idl.tpl @@ -103,6 +103,18 @@ feature_flags: serialize_on_outgoing_requests: true version: $ver_str(latest) + featureFlagTestIFR: + description: "IFR feature flag used by fixture builder unit tests" + cpp_varname: gFeatureFlagTestIFR + incremental_rollout_phase: in_development + fcv_gated: false + + featureFlagTestNonIFR: + description: "Non-IFR feature flag used by fixture builder unit tests" + cpp_varname: gFeatureFlagTestNonIFR + default: false + fcv_gated: false + server_parameters: spTestNeedsFeatureFlagToaster: description: "Server Parameter gated on featureFlagToaster"