Revert "SERVER-84420: Remove unnecessary tasserts."

This reverts commit dd42977758.

README.md

@@ -1,6 +1,6 @@
 # ![Logo](docs/leaf.svg) MongoDB README
 
-Welcome to MongoDB!
+Welcome to MongoDB 7.2!
 
 ## Components
 

buildscripts/evergreen_expansions2bash.py

@@ -73,6 +73,10 @@ def _load_expansions(expansions_file) -> dict:
         return expansions
 
 
+def _clean_key(key):
+    return key.replace("-", "_")
+
+
 @click.command()
 @click.argument("expansions_file", type=str)
 @click.argument("defaults_file", type=str)
@@ -87,7 +91,7 @@ def _main(expansions_file: str, defaults_file: str):
                 expansions[key] = value
 
         for key, value in expansions.items():
-            print(f"{key}={quote(value)}; ", end="")
+            print(f"{_clean_key(key)}={quote(value)}; ", end="")
 
     except Exception as ex:  # pylint: disable=broad-except
         _error(ex)

buildscripts/generate_version_expansions.py

@@ -61,8 +61,8 @@ def generate_version_expansions():
             raise ValueError("Unable to parse version from stdin and no version.json provided")
 
     if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
+        expansions["suffix"] = "v7.2-latest"
+        expansions["src_suffix"] = "v7.2-latest"
         expansions["is_release"] = "false"
     else:
         expansions["suffix"] = version_line

buildscripts/idl/idl/cpp_types.py

@@ -601,15 +601,19 @@ class _ObjectBsonCppTypeBase(BsonCppTypeBase):
     def gen_serializer_expression(self, indented_writer, expression, should_shapify=False):
         # type: (writer.IndentedTextWriter, str, bool) -> str
         method_name = writer.get_method_name(self._ast_type.serializer)
-        if self._ast_type.deserialize_with_tenant:  # SerializationContext is tied to tenant deserialization
-            indented_writer.write_line(
-                common.template_args(
-                    'const BSONObj localObject = ${expression}.${method_name}(getSerializationContext());',
-                    expression=expression, method_name=method_name))
-        else:
-            indented_writer.write_line(
-                common.template_args('const BSONObj localObject = ${expression}.${method_name}();',
-                                     expression=expression, method_name=method_name))
+        function_arguments = []
+        # SerializationContext is tied to tenant deserialization
+        if self._ast_type.deserialize_with_tenant:
+            function_arguments.append('getSerializationContext()')
+        # Provide options if custom shapification required.
+        if should_shapify:
+            function_arguments.append('options')
+
+        indented_writer.write_line(
+            common.template_args(
+                'const BSONObj localObject = ${expression}.${method_name}(${function_arguments});',
+                expression=expression, method_name=method_name,
+                function_arguments=', '.join(function_arguments)))
         return "localObject"
 
 

buildscripts/idl/idl/struct_types.py

@@ -329,7 +329,7 @@ class _StructTypeInfo(StructTypeInfoBase):
         # type: () -> MethodInfo
         args = ['BSONObjBuilder* builder']
         if self._struct.query_shape_component:
-            args.append("SerializationOptions options = {}")
+            args.append("const SerializationOptions& options = {}")
         return MethodInfo(
             common.title_case(self._struct.cpp_name), 'serialize', args, 'void', const=True)
 
@@ -337,7 +337,7 @@ class _StructTypeInfo(StructTypeInfoBase):
         # type: () -> MethodInfo
         args = []
         if self._struct.query_shape_component:
-            args.append("SerializationOptions options = {}")
+            args.append("const SerializationOptions& options = {}")
         return MethodInfo(
             common.title_case(self._struct.cpp_name), 'toBSON', args, 'BSONObj', const=True)
 

buildscripts/idl/tests/test_generator.py

@@ -218,6 +218,92 @@ class TestGenerator(testcase.IDLTestcase):
         """)
         self.assertIn(expected, source)
 
+    def test_object_type_with_custom_serializer_and_query_shape_specification_custom(self) -> None:
+        """Serialization with custom query_shape."""
+        _, source = self.assert_generate("""
+        types:
+                serialization_context:
+                    bson_serialization_type: any
+                    description: foo
+                    cpp_type: foo
+                    internal_only: true
+
+                object_type_with_custom_serializer:
+                    bson_serialization_type: object
+                    description: ObjWithCustomSerializer
+                    cpp_type: ObjWithCustomSerializer
+                    serializer: ObjWithCustomSerializer::toBSON
+                    deserializer: ObjWithCustomSerializer::parse
+
+        structs:
+                QueryShapeSpec:
+                    description: QueryShape
+                    query_shape_component: true
+                    fields:
+                        internalObject:
+                            type: object_type_with_custom_serializer
+                            optional: false
+                            description: internalObject
+                            query_shape: custom
+        """)
+
+        expected = dedent("""
+        void QueryShapeSpec::serialize(BSONObjBuilder* builder, const SerializationOptions& options) const {
+            _hasMembers.required();
+
+            {
+                const BSONObj localObject = _internalObject.toBSON(options);
+                builder->append(kInternalObjectFieldName, localObject);
+            }
+
+        }""")
+        self.assertIn(expected, source)
+
+    def test_array_of_object_type_with_custom_serializer_and_query_shape_specification_custom(
+            self) -> None:
+        """Serialization with custom query_shape used, array use case."""
+        _, source = self.assert_generate("""
+        types:
+                serialization_context:
+                    bson_serialization_type: any
+                    description: foo
+                    cpp_type: foo
+                    internal_only: true
+
+                object_type_with_custom_serializer:
+                    bson_serialization_type: object
+                    description: ObjWithCustomSerializer
+                    cpp_type: ObjWithCustomSerializer
+                    serializer: ObjWithCustomSerializer::toBSON
+                    deserializer: ObjWithCustomSerializer::parse
+
+        structs:
+                QueryShapeSpec:
+                    description: QueryShape
+                    query_shape_component: true
+                    fields:
+                        internalObjectArray:
+                            type: array<object_type_with_custom_serializer>
+                            optional: false
+                            description: internalObjectArray
+                            query_shape: custom
+        """)
+
+        expected = dedent("""
+        void QueryShapeSpec::serialize(BSONObjBuilder* builder, const SerializationOptions& options) const {
+            _hasMembers.required();
+
+            {
+                BSONArrayBuilder arrayBuilder(builder->subarrayStart(kInternalObjectArrayFieldName));
+                for (const auto& item : _internalObjectArray) {
+                    const BSONObj localObject = item.toBSON(options);
+                    arrayBuilder.append(localObject);
+                }
+            }
+
+        }""")
+        self.assertIn(expected, source)
+
 
 if __name__ == '__main__':
 

buildscripts/linter/mongolint.py

@@ -20,6 +20,7 @@ GENERIC_FCV = [
     r'::kDowngradingFromLatestToLastLTS',
     r'::kDowngradingFromLatestToLastContinuous',
     r'\.isUpgradingOrDowngrading',
+    r'->isUpgradingOrDowngrading',
     r'::kDowngradingFromLatestToLastContinuous',
     r'::kUpgradingFromLastLTSToLastContinuous',
 ]

buildscripts/package_test.py

@@ -407,6 +407,10 @@ release_test_parser.add_argument("-e", "--edition", help="Server edition to run
 release_test_parser.add_argument("-v", "--server-version", type=str,
                                  help="Version of MongoDB to run tests for",
                                  choices=["all"] + list(versions), default="all")
+release_test_parser.add_argument(
+    "--evg-project", type=str, help=
+    "The evergreen project this is intended to run under (master only). Note that this interface is primarly for evergreen to set, and so the script will check if its is appropriate to run the tests.",
+    default="")
 branch_test_parser = subparsers.add_parser("branch")
 branch_test_parser.add_argument(
     "-t", "--test", type=str, help=
@@ -418,6 +422,19 @@ branch_test_parser.add_argument("-v", "--server-version", type=str,
                                 help="Server version being tested", required=True)
 args = parser.parse_args()
 
+if args.command == "release":
+    evg_project = args.evg_project
+    if not evg_project:
+        logging.error(
+            "Missing '--evg-project' command line option. If trying to run this locally, you will need to set the environment so that --evg-project=mongodb-mongo-master."
+        )
+        sys.exit(1)
+    if re.fullmatch(r"mongodb-mongo-v\d\.\d", evg_project):
+        logging.info(
+            "Non-master evergreen project detected: '%s', skipping release package testing which is expected to only be run from master branches.",
+            evg_project)
+        sys.exit(0)
+
 arch: str = args.arch
 if arch == "auto":
     arch = platform.machine()

buildscripts/resmokeconfig/suites/benchmarks_second_half.yml

@@ -60,6 +60,7 @@ selector:
   - build/install/bin/abt_translation_bm*
   # These benchmarks are being run as part of the benchmarks_streams.yml test suite.
   - build/install/bin/streams_operator_dag_bm*
+  - build/install/bin/streams_window_operator_bm*
   # These benchmarks are only run when modifying or upgrading the immutable library.
   - build/install/bin/immutable_absl_comparison_bm*
   - build/install/bin/immutable_std_comparison_bm*

buildscripts/resmokeconfig/suites/benchmarks_streams.yml

@@ -7,7 +7,6 @@ selector:
   # The trailing asterisk is for handling the .exe extension on Windows.
   - build/**/system_resource_canary_bm*
   - build/install/bin/streams_operator_dag_bm*
-  - build/install/bin/streams_window_operator_bm*
 
 executor:
   config: {}

buildscripts/resmokeconfig/suites/cqf_experimental_no_passthrough.yml

@@ -15,8 +15,6 @@ selector:
   - jstests/noPassthrough/libs/*.js
   # Disable inmem_full as per SERVER-27014
   - jstests/noPassthrough/inmem_full.js
-  # TODO SERVER-81313 re-enable this test
-  - jstests/noPassthrough/queryStats/query_stats_changeStreams.js
   exclude_with_any_tags:
   - cqf_experimental_incompatible
   - cqf_incompatible

buildscripts/resmokeconfig/suites/multitenancy_with_mongoq_jscore_passthrough.yml

@@ -48,6 +48,7 @@ selector:
   - jstests/core/**/plan_cache_shell_helpers.js
   - jstests/core/**/plan_cache_stats_shard_and_host.js
   - jstests/core/**/profile_query_hash.js
+  - jstests/core/query/plan_cache/match_or_duplicate_predicate_plan_cache.js
   - jstests/core/sbe/from_plan_cache_flag.js
   - jstests/core/sbe/plan_cache_sbe_with_or_queries.js
   - jstests/core/sbe_plan_cache_autoparameterize_collscan.js

buildscripts/resmokeconfig/suites/no_passthrough.yml

@@ -19,8 +19,6 @@ selector:
   - jstests/noPassthrough/libs/*.js
   # Disable inmem_full as per SERVER-27014
   - jstests/noPassthrough/inmem_full.js
-  # TODO SERVER-81313 re-enable this test
-  - jstests/noPassthrough/queryStats/query_stats_changeStreams.js
   exclude_with_any_tags:
 
 # noPassthrough tests start their own mongod's.

buildscripts/resmokeconfig/suites/search_auth.yml

@@ -8,6 +8,9 @@ selector:
   roots:
   - src/mongo/db/modules/*/jstests/search/*.js
   - src/mongo/db/modules/*/jstests/mongot/*.js
+  exclude_files:
+  # Skip any tests that run with auth explicitly.
+  - src/mongo/db/modules/*/jstests/search/auth_list_search_indexes_agg.js
 
 executor:
   config:

buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml

@@ -8,6 +8,9 @@ selector:
   roots:
   - src/mongo/db/modules/*/jstests/search/*.js
   - src/mongo/db/modules/*/jstests/mongot/*.js
+  exclude_files:
+  # Skip any tests that run with auth explicitly.
+  - src/mongo/db/modules/*/jstests/search/auth_list_search_indexes_agg.js
 
 executor:
   config:

buildscripts/resmokelib/config.py

@@ -105,6 +105,7 @@ DEFAULTS = {
     "shuffle": None,
     "stagger_jobs": None,
     "majority_read_concern": "on",
+    "enable_enterprise_tests": None,
     "shell_seed": None,
     "storage_engine": "wiredTiger",
     "storage_engine_cache_size_gb": None,
@@ -637,7 +638,7 @@ USE_LEGACY_MULTIVERSION = True
 # Expansions file location
 # in CI, the expansions file is located in the ${workdir}, one dir up
 # from src, the checkout directory
-EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else None
+EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else "expansions.yml"
 
 # Symbolizer secrets
 SYMBOLIZER_CLIENT_SECRET = None

buildscripts/resmokelib/configure_resmoke.py

@@ -321,6 +321,7 @@ be invoked as either:
     _config.JOBS = config.pop("jobs")
     _config.LINEAR_CHAIN = config.pop("linear_chain") == "on"
     _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on"
+    _config.ENABLE_ENTERPRISE_TESTS = config.pop("enable_enterprise_tests")
     _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions")
     if _config.MIXED_BIN_VERSIONS is not None:
         _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-")

buildscripts/resmokelib/hang_analyzer/gen_hang_analyzer_tasks.py

@@ -101,8 +101,11 @@ def generate(expansions_file: str = "../expansions.yml",
 
     # gather information from the current task being run
     expansions = read_config_file(expansions_file)
-    distro = expansions.get(
-        "large_distro_name") if "large_distro_name" in expansions else expansions.get("distro_id")
+    distro = None
+    for distro_expansion in ["core_analyzer_distro_name", "large_distro_name", "distro_id"]:
+        if distro := expansions.get(distro_expansion, None):
+            break
+    assert distro is not None
     current_task_name = expansions.get("task_name")
     task_id = expansions.get("task_id")
     execution = expansions.get("execution")

buildscripts/resmokelib/run/__init__.py

@@ -185,7 +185,8 @@ class TestRunner(Subcommand):
     def generate_multiversion_exclude_tags(self):
         """Generate multiversion exclude tags file."""
         generate_multiversion_exclude_tags.generate_exclude_yaml(
-            config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, self._resmoke_logger)
+            config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, config.EXPANSIONS_FILE,
+            self._resmoke_logger)
 
     @staticmethod
     def _find_suites_by_test(suites: List[Suite]):
@@ -1106,6 +1107,11 @@ class RunPlugin(PluginInterface):
             metavar="ON|OFF", help=("Enable or disable majority read concern support."
                                     " Defaults to %(default)s."))
 
+        mongodb_server_options.add_argument(
+            "--enableEnterpriseTests", action="store", dest="enable_enterprise_tests", default="on",
+            choices=("on", "off"), metavar="ON|OFF",
+            help=("Enable or disable enterprise tests. Defaults to 'on'."))
+
         mongodb_server_options.add_argument("--flowControl", action="store", dest="flow_control",
                                             choices=("on", "off"), metavar="ON|OFF",
                                             help=("Enable or disable flow control."))

buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py

@@ -5,6 +5,8 @@ import re
 import tempfile
 from collections import defaultdict
 from subprocess import check_output
+from typing import Optional
+from github import GithubIntegration
 
 import requests
 
@@ -13,13 +15,35 @@ from buildscripts.resmokelib.config import MultiversionOptions
 from buildscripts.resmokelib.core.programs import get_path_env_var
 from buildscripts.resmokelib.utils import is_windows
 from buildscripts.util.fileops import read_yaml_file
+from buildscripts.util.read_config import read_config_file
 
 BACKPORT_REQUIRED_TAG = "backport_required_multiversion"
 
 # The directory in which BACKPORTS_REQUIRED_FILE resides.
 ETC_DIR = "etc"
 BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml"
-BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo"
+BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/10gen/mongo"
+
+
+def get_installation_access_token(app_id: int, private_key: str,
+                                  installation_id: int) -> Optional[str]:  # noqa: D406,D407,D413
+    """
+    Obtain an installation access token using JWT.
+
+    Args:
+    - app_id: The application ID for GitHub App.
+    - private_key: The private key associated with the GitHub App.
+    - installation_id: The installation ID of the GitHub App for a particular account.
+
+    Returns:
+    - Optional[str]: The installation access token. Returns `None` if there's an error obtaining the token.
+    """
+    integration = GithubIntegration(app_id, private_key)
+    auth = integration.get_access_token(installation_id)
+    if auth:
+        return auth.token
+    else:
+        raise Exception("Error obtaining installation token")
 
 
 def get_backports_required_hash_for_shell_version(mongo_shell_path=None):
@@ -51,10 +75,24 @@ def get_backports_required_hash_for_shell_version(mongo_shell_path=None):
         f"Could not find a valid commit hash from the {mongo_shell_path} mongo binary.")
 
 
-def get_old_yaml(commit_hash):
+def get_old_yaml(commit_hash, expansions_file):
     """Download BACKPORTS_REQUIRED_FILE from the old commit and return the yaml."""
+
+    if not os.path.exists(expansions_file):
+        raise FileNotFoundError(f"The specified file does not exist: {expansions_file}")
+    expansions = read_config_file(expansions_file)
+
+    # Obtain installation access tokens using app credentials
+    access_token_10gen_mongo = get_installation_access_token(
+        expansions["app_id_10gen_mongo"], expansions["private_key_10gen_mongo"],
+        expansions["installation_id_10gen_mongo"])
+
     response = requests.get(
-        f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}')
+        f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}',
+        headers={
+            'Authorization': f'token {access_token_10gen_mongo}',
+        })
+
     # If the response was successful, no exception will be raised.
     response.raise_for_status()
 
@@ -68,7 +106,8 @@ def get_old_yaml(commit_hash):
     return backports_required_old
 
 
-def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Logger) -> None:
+def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: str,
+                          logger: logging.Logger) -> None:
     """
     Create a tag file associating multiversion tests to tags for exclusion.
 
@@ -98,7 +137,7 @@ def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Log
 
     # Get the yaml contents from the old commit.
     logger.info(f"Downloading file from commit hash of old branch {old_version_commit_hash}")
-    backports_required_old = get_old_yaml(old_version_commit_hash)
+    backports_required_old = get_old_yaml(old_version_commit_hash, expansions_file)
 
     def diff(list1, list2):
         return [elem for elem in (list1 or []) if elem not in (list2 or [])]

buildscripts/resmokelib/selector.py

@@ -20,6 +20,8 @@ from buildscripts.resmokelib import utils
 from buildscripts.resmokelib.utils import globstar
 from buildscripts.resmokelib.utils import jscomment
 
+ENTERPRISE_TEST_DIR = os.path.normpath("src/mongo/db/modules/enterprise/jstests")
+
 ########################
 #  Test file explorer  #
 ########################
@@ -272,6 +274,13 @@ class _TestList(object):
         for path in paths.evaluated:
             self._filtered.discard(path)
 
+    def filter_enterprise_tests(self):
+        """Exclude tests that start with the enterprise module directory from the test list."""
+        self._filtered = {
+            test
+            for test in self._filtered if not os.path.normpath(test).startswith(ENTERPRISE_TEST_DIR)
+        }
+
     def match_tag_expression(self, tag_expression, get_tags):
         """Filter the test list to only include tests that match the tag expression.
 
@@ -511,6 +520,9 @@ class _Selector(object):
         # 5. Apply the include files last with force=True to take precedence over the tags.
         if self._tests_are_files and selector_config.include_files:
             test_list.include_files(selector_config.include_files)
+        # 6: Apply the enterprise tests filter
+        if self.get_enterprise_tests_status() == "off":
+            test_list.filter_enterprise_tests()
 
         return self.sort_tests(*test_list.get_tests())
 
@@ -526,6 +538,11 @@ class _Selector(object):
         """Retrieve the tags associated with the give test file."""
         return []
 
+    @staticmethod
+    def get_enterprise_tests_status() -> str:
+        """Get the status of enterprise tests from the configuration."""
+        return config.ENABLE_ENTERPRISE_TESTS
+
 
 class _JSTestSelectorConfig(_SelectorConfig):
     """_SelectorConfig subclass for JavaScript tests."""

buildscripts/resmokelib/testing/fixtures/shardedcluster.py

@@ -367,6 +367,7 @@ class ShardedClusterFixture(interface.Fixture, interface._DockerComposeInterface
             del mongod_options["shardsvr"]
             mongod_options["configsvr"] = ""
             replset_config_options["configsvr"] = True
+            mongod_options["set_parameters"]["featureFlagTransitionToCatalogShard"] = "true"
             mongod_options["storageEngine"] = "wiredTiger"
 
             configsvr_options = self.configsvr_options.copy()
@@ -415,6 +416,8 @@ class ShardedClusterFixture(interface.Fixture, interface._DockerComposeInterface
         """Return options that may be passed to a mongos."""
         mongos_options = self.mongos_options.copy()
         mongos_options["configdb"] = self.configsvr.get_internal_connection_string()
+        if self.config_shard is not None:
+            mongos_options["set_parameters"]["featureFlagTransitionToCatalogShard"] = "true"
         mongos_options["set_parameters"] = mongos_options.get("set_parameters",
                                                               self.fixturelib.make_historic(
                                                                   {})).copy()

buildscripts/sync_repo_with_copybara.py

@@ -67,10 +67,20 @@ def main():
 
     try:
         run_command(" ".join(docker_cmd))
-    except subprocess.CalledProcessError as e:
-        # Handle the specific error case for "No new changes..." between two repos
-        if "No new changes to import for resolved ref" not in str(e.stderr):
-            raise
+    except subprocess.CalledProcessError as err:
+        error_message = str(err.stderr)
+        acceptable_error_messages = [
+            # Indicates the two repositories are identical
+            "No new changes to import for resolved ref",
+            # Indicates differences exist but no changes affect the destination, for example: exclusion rules
+            "Iterative workflow produced no changes in the destination for resolved ref",
+        ]
+
+        if any(acceptable_message in error_message
+               for acceptable_message in acceptable_error_messages):
+            return
+
+        raise
 
 
 if __name__ == "__main__":

buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py

@@ -10,6 +10,7 @@ from buildscripts.resmokelib.run import generate_multiversion_exclude_tags as un
 from buildscripts.util.fileops import read_yaml_file
 
 EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml"
+EXPANSIONS_FILE = "expansions.yml"
 
 
 class TestGenerateExcludeYaml(unittest.TestCase):
@@ -39,6 +40,7 @@ class TestGenerateExcludeYaml(unittest.TestCase):
 
                 output = os.path.join(self._tmpdir.name, EXCLUDE_TAGS_FILE)
                 under_test.generate_exclude_yaml(old_bin_version=old_bin_version, output=output,
+                                                 expansions_file=EXPANSIONS_FILE,
                                                  logger=MagicMock())
 
                 mock_read_yaml.assert_called_once()

buildscripts/validate_commit_message.py

@@ -66,6 +66,9 @@ COMMON_PUBLIC_PATTERN = r"""
     """
 """Common Public pattern format."""
 
+COMMON_10GENREPO_COMMIT_QUEUE_PATTERN = r' ^\'(?P<repo>10gen/mongo)\'\s.*commit\squeue\smerge.*SERVER-[0-9]+'
+"""Common commit queue format."""
+
 COMMON_LINT_PATTERN = r"(?P<lint>Fix\slint)"
 """Common Lint pattern format."""
 
@@ -120,6 +123,12 @@ def old_patch_description(pattern: str) -> str:
 # NOTE: re.VERBOSE is for visibility / debugging. As such significant white space must be
 # escaped (e.g ' ' to \s).
 VALID_PATTERNS = [
+    re.compile(
+        new_patch_description(COMMON_10GENREPO_COMMIT_QUEUE_PATTERN),
+        re.MULTILINE | re.DOTALL | re.VERBOSE),
+    re.compile(
+        old_patch_description(COMMON_10GENREPO_COMMIT_QUEUE_PATTERN),
+        re.MULTILINE | re.DOTALL | re.VERBOSE),
     re.compile(
         new_patch_description(COMMON_PUBLIC_PATTERN),
         re.MULTILINE | re.DOTALL | re.VERBOSE,

copybara.staging.sky

@@ -1,28 +0,0 @@
-# This configuration is for migrating code from one Git repository to another using Copybara.
-# It selectively copies content, excluding specific paths and preserving authorship.
-sourceUrl = "git@github.com:10gen/mongo.git"
-destinationUrl = "git@github.com:10gen/mongo-copybara.git"
-
-core.workflow(
-    name = "default",
-    origin = git.origin(
-        url = sourceUrl,
-        ref = "master",
-        # VersionSelector
-    ),
-    destination = git.destination(
-        url = destinationUrl,
-        fetch = "master",
-        push = "master",
-    ),
-    # Change path to the folder you want to publish publicly
-    origin_files = glob(["**"], exclude=["src/mongo/db/modules/**"]),
-
-    authoring = authoring.pass_thru("MongoDB <mongodb@mongodb.com>"),
-
-    mode = "ITERATIVE",
-    # Change the path here to the folder you want to publish publicly
-    # transformations = [
-	#     core.move("path/to/folder/you/want/exported", ""),
-	# ],
-)

etc/backports_required_for_multiversion_tests.yml

@@ -392,7 +392,7 @@ last-continuous:
   - test_file: jstests/sharding/query/sharded_lookup_execution.js
     ticket: SERVER-77427
   - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
-    ticket: SERVER-77506
+    ticket: SERVER-84723
   - test_file: jstests/sharding/fsync_lock_unlock.js
     ticket: SERVER-73685
   - test_file: jstests/sharding/fsync_lock_ddl_lock.js
@@ -495,6 +495,64 @@ last-continuous:
     ticket: SERVER-81985
   - test_file: jstests/core/timeseries/timeseries_collection_uuid.js
     ticket: SERVER-81379
+  - test_file: src/mongo/db/modules/enterprise/jstests/fle2/basic_update.js
+    ticket: SERVER-82984
+  - test_file: src/mongo/db/modules/enterprise/jstests/fle2/basic_replace.js
+    ticket: SERVER-82984
+  - test_file: jstests/core/timeseries/timeseries_project_pushdown.js
+    ticket: SERVER-82447
+  - test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js
+    ticket: SERVER-79848
+  - test_file: jstests/core/find_with_resume_after_param.js
+    ticket: SERVER-79848
+  - test_file: jstests/core/timeseries/timeseries_resume_after.js
+    ticket: SERVER-79848
+  - test_file: src/mongo/db/modules/enterprise/jstests/external_auth_oidc/oidc_internal_authz.js
+    ticket: SERVER-78066
+  - test_file: src/mongo/db/modules/enterprise/jstests/external_auth_oidc/oidc_omit_client_id.js
+    ticket: SERVER-78066
+  - test_file: jstests/core/query/explode_for_sort_plan_cache.js
+    ticket: SERVER-83777
+  - test_file: jstests/core/txns/out_not_blocked_by_txn.js
+    ticket: SERVER-72703
+  - test_file: jstests/core/command_let_variables.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/index/index_filter_commands_invalidate_plan_cache_entries.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/index/wildcard/wildcard_index_cached_plans.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/query/expr/expr_index_use.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/query/plan_cache/plan_cache_list_plans.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/query/plan_cache/plan_cache_list_shapes.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/query/plan_cache/plan_cache_stats_shard_and_host.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/query/plan_cache/plan_cache_shell_helpers.js
+    ticket: SERVER-83685
+  - test_file: jstests/core/views/views_aggregation.js
+    ticket: SERVER-83685
+  - test_file: jstests/noPassthroughWithMongod/lookup_with_limit.js
+    ticket: SERVER-83685
+  - test_file: jstests/sharding/move_primary_basic.js
+    ticket: SERVER-82353
+  - test_file: jstests/sharding/store_historical_placement_data.js
+    ticket: SERVER-82353
+  - test_file: jstests/noPassthrough/disallow_new_transactions_at_shutdown_on_mongos.js
+    ticket: SERVER-77667
+  - test_file: jstests/core/notablescan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/clustered_coll_scan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/eof_plan.js
+    ticket: SERVER-83119
+  - test_file: jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster_config_shard.js
+    ticket: SERVER-84548
+  - test_file: src/mongo/db/modules/enterprise/jstests/hot_backups/multiVersion/sharded_last_cont_pit_backup_restore_simple_config_shard.js
+    ticket: SERVER-84548
+  - test_file: src/mongo/db/modules/enterprise/jstests/hot_backups/multiVersion/sharded_last_cont_backup_restore_simple_config_shard.js
+    ticket: SERVER-84548
   suites: null
 last-lts:
   all:
@@ -943,7 +1001,7 @@ last-lts:
   - test_file: jstests/sharding/query/sharded_lookup_execution.js
     ticket: SERVER-77427
   - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
-    ticket: SERVER-77506
+    ticket: SERVER-84723
   - test_file: jstests/sharding/fsync_lock_unlock.js
     ticket: SERVER-73685
   - test_file: jstests/sharding/fsync_lock_ddl_lock.js
@@ -1052,4 +1110,40 @@ last-lts:
     ticket: SERVER-81985
   - test_file: jstests/core/timeseries/timeseries_collection_uuid.js
     ticket: SERVER-81379
+  - test_file: src/mongo/db/modules/enterprise/jstests/fle2/basic_update.js
+    ticket: SERVER-82984
+  - test_file: src/mongo/db/modules/enterprise/jstests/fle2/basic_replace.js
+    ticket: SERVER-82984
+  - test_file: jstests/core/timeseries/timeseries_project_pushdown.js
+    ticket: SERVER-82447
+  - test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js
+    ticket: SERVER-79848
+  - test_file: jstests/core/find_with_resume_after_param.js
+    ticket: SERVER-79848
+  - test_file: jstests/core/timeseries/timeseries_resume_after.js
+    ticket: SERVER-79848
+  - test_file: jstests/core/query/explode_for_sort_plan_cache.js
+    ticket: SERVER-83777
+  - test_file: jstests/core/txns/out_not_blocked_by_txn.js
+    ticket: SERVER-72703
+  - test_file: jstests/sharding/move_primary_basic.js
+    ticket: SERVER-82353
+  - test_file: jstests/sharding/store_historical_placement_data.js
+    ticket: SERVER-82353
+  - test_file: jstests/core/timeseries/timeseries_bucket_level_filter.js
+    ticket: SERVER-84130
+  - test_file: jstests/noPassthrough/disallow_new_transactions_at_shutdown_on_mongos.js
+    ticket: SERVER-77667
+  - test_file: jstests/core/notablescan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/clustered_coll_scan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/eof_plan.js
+    ticket: SERVER-83119
+  - test_file: jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster_config_shard.js
+    ticket: SERVER-84548
+  - test_file: src/mongo/db/modules/enterprise/jstests/hot_backups/multiVersion/sharded_last_cont_pit_backup_restore_simple_config_shard.js
+    ticket: SERVER-84548
+  - test_file: src/mongo/db/modules/enterprise/jstests/hot_backups/multiVersion/sharded_last_cont_backup_restore_simple_config_shard.js
+    ticket: SERVER-84548
   suites: null

etc/evergreen.yml

@@ -53,13 +53,15 @@
 # - func: "set up venv"
 
 include:
+- filename: etc/evergreen_yml_components/configuration.yml
 - filename: etc/evergreen_yml_components/definitions.yml
 - filename: etc/evergreen_yml_components/variants/task_generation.yml
 - filename: etc/evergreen_yml_components/variants/sanitizer.yml
 - filename: etc/evergreen_yml_components/variants/in_memory.yml
 - filename: etc/evergreen_yml_components/variants/ninja.yml
 - filename: etc/evergreen_yml_components/variants/compile_static_analysis.yml
-- filename: etc/evergreen_yml_components/variants/config_shard.yml
+# TODO: (SERVER-86211) Re-enable config shard evergreen variant.
+# - filename: etc/evergreen_yml_components/variants/config_shard.yml
 - filename: etc/evergreen_yml_components/perf_tasks.yml
 
 variables:
@@ -78,15 +80,15 @@ variables:
   - name: generate_buildid_to_debug_symbols_mapping
     variant: linux-x86-dynamic-compile
 
-- &linux_x86_compile_variant_dependency
+- &amazon2_x86_compile_variant_dependency
   depends_on:
   - name: archive_dist_test_debug
-    variant: &linux_x86_compile_variant_name linux-x86-compile #static compile
+    variant: &amazon2_x86_compile_variant_name amazon2-x86-compile #static compile
   - name: version_gen
     variant: generate-tasks-for-version
     omit_generated_tasks: true
   - name: generate_buildid_to_debug_symbols_mapping
-    variant: linux-x86-compile
+    variant: amazon2-x86-compile
 
 - &amazon_linux2_arm64_dynamic_compile_variant_dependency
   depends_on:
@@ -152,15 +154,16 @@ variables:
   multiversion_edition: enterprise
   repo_edition: enterprise
   large_distro_name: rhel80-medium
+  core_analyzer_distro_name: rhel80-large
   num_scons_link_jobs_available: 0.99
   compile_variant: *linux_x86_dynamic_compile_variant_name
 
-- &linux_x86_expansions
+- &amazon2_x86_expansions
   multiversion_platform: amazon2
   multiversion_edition: enterprise
   repo_edition: enterprise
   num_scons_link_jobs_available: 0.99
-  compile_variant: *linux_x86_compile_variant_name
+  compile_variant: *amazon2_x86_compile_variant_name
 
 - &linux_arm64_generic_expansions
   multiversion_platform: amazon2
@@ -399,14 +402,14 @@ buildvariants:
 ###########################################
 #         Linux buildvariants             #
 ###########################################
-- <<: *linux_x86_compile_variant_dependency
+- <<: *amazon2_x86_compile_variant_dependency
   name: perf-standalone-classic-query-engine.intel.aws.2023-09
   display_name: PERF Standalone (Classic Query Engine) Intel AWS 2023-09
   # cron: &cron_query_perf_schedule "0 0 * * 4"  # 00:00 on Thursday
   activate: false
   modules: *perf_modules
   expansions:
-    <<: *linux_x86_expansions
+    <<: *amazon2_x86_expansions
     idle_timeout_factor: 1.5
     exec_timeout_factor: 1.5
     mongodb_setup_release: 2022-11
@@ -418,7 +421,7 @@ buildvariants:
     project_dir: *perf_project_dir
     authentication: enabled
     storageEngine: wiredTiger
-    compile_variant: *linux_x86_compile_variant_name
+    compile_variant: *amazon2_x86_compile_variant_name
   run_on:
     - amazon2
   tasks: &classic_engine_tasks
@@ -456,6 +459,7 @@ buildvariants:
     target_resmoke_time: 15
     max_sub_suites: 5
     large_distro_name: rhel80-medium
+    core_analyzer_distro_name: rhel80-large
     compile_variant: *linux_x86_dynamic_debug_compile_variant_name
   tasks:
   - name: .aggregation !.encrypt !.feature_flag_guarded
@@ -1385,7 +1389,6 @@ buildvariants:
     max_sub_suites: 5
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
-      --excludeWithAnyTags=requires_sbe
     large_distro_name: rhel80-medium
   depends_on:
   - name: archive_dist_test_debug
@@ -1402,9 +1405,8 @@ buildvariants:
   - name: .causally_consistent !.sharding
   - name: .change_stream_fuzzer
   - name: .change_streams
-  # TODO(SERVER-79848): Remove !.sharded
-  - name: .concurrency !.large !.ubsan !.no_txns !.compute_mode !.sharded
-  - name: .concurrency .large !.ubsan !.no_txns !.compute_mode !.sharded
+  - name: .concurrency !.large !.ubsan !.no_txns !.compute_mode
+  - name: .concurrency .large !.ubsan !.no_txns !.compute_mode
     distros:
     - rhel80-medium
   - name: .encrypt
@@ -2075,7 +2077,7 @@ buildvariants:
     <<: *aubsan-lite-required-expansions
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
-      --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits,requires_sbe
+      --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits
   tasks:
   - name: .aggregation !.sbe_only
   - name: .auth
@@ -2084,8 +2086,7 @@ buildvariants:
   - name: .change_streams
   # - name: disk_wiredtiger
   - name: .misc_js
-  # TODO(SERVER-79848): Remove !.sharded
-  - name: .concurrency !.ubsan !.no_txns !.kill_terminate !.compute_mode !.sharded
+  - name: .concurrency !.ubsan !.no_txns !.kill_terminate !.compute_mode
   - name: .encrypt
   - name: external_auth
   - name: external_auth_aws
@@ -2104,8 +2105,7 @@ buildvariants:
   - name: replica_sets_large_txns_format_jscore_passthrough
   - name: .replica_sets !.multi_oplog
   - name: .replica_sets .encrypt
-  # TODO(SERVER-79848): Re-enable resharding_fuzzer
-  # - name: .resharding_fuzzer
+  - name: .resharding_fuzzer
   - name: .retry
   - name: .rollbackfuzzer
   - name: .read_only
@@ -2249,6 +2249,7 @@ buildvariants:
     scons_cache_scope: shared
     separate_debug: off
     large_distro_name: rhel80-medium
+    core_analyzer_distro_name: rhel80-large
     multiversion_platform: rhel80
     multiversion_edition: enterprise
     test_flags: >-
@@ -2961,7 +2962,7 @@ buildvariants:
     max_sub_suites: 3
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
-      --excludeWithAnyTags=resource_intensive,requires_sbe
+      --excludeWithAnyTags=resource_intensive
 
 - <<: *enterprise-rhel-80-64-bit-dynamic-all-feature-flags-template
   name: enterprise-rhel-80-64-bit-dynamic-classic-engine-all-feature-flags-required-query-patch-only
@@ -2976,7 +2977,7 @@ buildvariants:
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
       --runAllFeatureFlagTests
-      --excludeWithAnyTags=resource_intensive,requires_sbe
+      --excludeWithAnyTags=resource_intensive,featureFlagSbeFull
   tasks:
   - name: analyze_shard_key_jscore_passthrough_gen
   - name: query_golden_classic
@@ -3000,9 +3001,8 @@ buildvariants:
   - name: change_streams_multitenant_sharded_collections_passthrough
   - name: .misc_js
   - name: .clustered_collections
-  # TODO(SERVER-79848): Remove !.sharded
-  - name: .concurrency !.large !.ubsan !.no_txns !.sharded
-  - name: .concurrency .large !.ubsan !.no_txns !.sharded
+  - name: .concurrency !.large !.ubsan !.no_txns
+  - name: .concurrency .large !.ubsan !.no_txns
     distros:
     - rhel80-medium
   - name: .config_fuzzer !.large

etc/evergreen_nightly.yml

@@ -1,25 +1,572 @@
 # Similar to etc/evergreen.yml, but for release builds.
 
 include:
+- filename: etc/evergreen_yml_components/configuration.yml
 - filename: etc/evergreen_yml_components/definitions.yml
 - filename: etc/evergreen_yml_components/variants/task_generation.yml
 - filename: etc/evergreen_yml_components/variants/atlas.yml
 - filename: etc/evergreen_yml_components/variants/misc_release.yml
 ### Comment out when using this file for a Rapid release branch. ###
-- filename: etc/evergreen_yml_components/variants/ibm.yml
+# - filename: etc/evergreen_yml_components/variants/ibm.yml
 ### Uncomment when using this file for a LTS release branch. ###
 # - filename: etc/evergreen_yml_components/variants/in_memory.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
-# - filename: etc/evergreen_yml_components/variants/sanitizer.yml
+- filename: etc/evergreen_yml_components/variants/sanitizer.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
-# - filename: etc/evergreen_yml_components/variants/ninja.yml
+- filename: etc/evergreen_yml_components/variants/ninja.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
-# - filename: etc/evergreen_yml_components/variants/classic_engine.yml
+- filename: etc/evergreen_yml_components/variants/classic_engine.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
+# TODO: (SERVER-86211) Re-enable config shard evergreen variant.
 # - filename: etc/evergreen_yml_components/variants/config_shard.yml
+- filename: etc/evergreen_yml_components/variants/compile_static_analysis.yml
 
 
 parameters:
 - key: evergreen_config_file_path
   value: "etc/evergreen_nightly.yml"
   description: "path to this file"
+
+
+variables:
+- &linux_x86_dynamic_debug_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_x86_dynamic_debug_compile_variant_name linux-x86-dynamic-debug-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: generate_buildid_to_debug_symbols_mapping
+    variant: linux-x86-dynamic-debug-compile-required
+
+- &windows_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &windows_compile_variant_name windows-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: generate_buildid_to_debug_symbols_mapping
+    variant: windows-compile-required
+
+- &amazon_linux2_arm64_dynamic_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &amazon_linux2_arm64_dynamic_compile_variant_name amazon-linux2-arm64-dynamic-compile
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: generate_buildid_to_debug_symbols_mapping
+    variant: amazon-linux2-arm64-dynamic-compile
+
+# Common compile variant dependency specifications.
+# THIS WAS COPIED TO config_shard.yml - ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THAT FILE.
+- &linux_x86_dynamic_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_x86_dynamic_compile_variant_name linux-x86-dynamic-compile
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: generate_buildid_to_debug_symbols_mapping
+    variant: linux-x86-dynamic-compile
+
+# THIS WAS COPIED TO config_shard.yml - ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THAT FILE.
+- &linux_x86_generic_expansions
+  multiversion_platform: rhel80
+  multiversion_edition: enterprise
+  repo_edition: enterprise
+  large_distro_name: rhel80-medium
+  num_scons_link_jobs_available: 0.99
+  compile_variant: *linux_x86_dynamic_compile_variant_name
+
+- &linux_debug_aubsan_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_debug_aubsan_compile_variant_name linux-debug-aubsan-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: generate_buildid_to_debug_symbols_mapping
+    variant: linux-debug-aubsan-compile-required
+
+# If you add anything to san_options, make sure the appropriate changes are
+# also made to SConstruct.
+# and also to the san_options in compile_static_analysis.yml and sanitizer.yml
+- aubsan_options: &aubsan_options
+  >-
+    UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
+    LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1"
+    ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:strict_string_checks=true:detect_invalid_pointer_pairs=1:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
+
+- &linux_arm64_generic_expansions
+  multiversion_platform: amazon2
+  multiversion_edition: enterprise
+  multiversion_architecture: aarch64
+  packager_arch: aarch64
+  packager_distro: amazon2
+  repo_edition: enterprise
+  large_distro_name: amazon2-arm64-large
+  num_scons_link_jobs_available: 0.99
+  compile_variant: *amazon_linux2_arm64_dynamic_compile_variant_name
+
+- &enterprise-amazon-linux2-arm64-all-feature-flags-template
+  <<: *amazon_linux2_arm64_dynamic_compile_variant_dependency
+  name: &enterprise-amazon-linux2-arm64-all-feature-flags enterprise-amazon-linux2-arm64-all-feature-flags
+  display_name: "! Amazon Linux 2 arm64 (all feature flags)"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+    - enterprise
+  run_on:
+    - amazon2-arm64-small
+  stepback: true
+  expansions: &enterprise-amazon-linux2-arm64-all-feature-flags-expansions
+    <<: *linux_arm64_generic_expansions
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    has_packages: false
+    jstestfuzz_num_generated_files: 40
+    jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 10
+    max_sub_suites: 5
+    idle_timeout_factor: 1.5
+    exec_timeout_factor: 1.5
+    test_flags: >-
+      --runAllFeatureFlagTests
+      --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source
+  tasks:
+  - name: analyze_shard_key_jscore_passthrough_gen
+  - name: query_golden_classic
+  - name: lint_fuzzer_sanity_patch
+  - name: test_api_version_compatibility
+  - name: check_feature_flag_tags
+  - name: check_for_todos
+  - name: .aggfuzzer !.cqf_only
+  - name: .aggregation
+  - name: aggregation_repeat_queries
+  - name: audit
+  - name: .auth
+  - name: buildscripts_test
+  - name: .bulk_write
+  - name: resmoke_end2end_tests
+  - name: unittest_shell_hang_analyzer_gen
+  - name: .causally_consistent !.sharding
+  - name: .change_streams
+  - name: .change_stream_fuzzer
+  # TODO SERVER-57866: Remove the explicit mentions of change stream multitenant suites.
+  - name: change_streams_multitenant_passthrough
+  - name: change_streams_multitenant_sharded_collections_passthrough
+  - name: .cqf
+  - name: .misc_js
+  - name: .clustered_collections
+  - name: .concurrency !.large !.ubsan !.no_txns
+  - name: .concurrency .large !.ubsan !.no_txns
+  - name: .config_fuzzer !.large
+  - name: .config_fuzzer .large
+    distros:
+    - amazon2-arm64-large
+  - name: .config_fuzzer_stress
+    distros:
+    - amazon2-arm64-large
+  - name: disk_wiredtiger
+  - name: .encrypt
+  - name: feature_flag_multiversion_gen
+  - name: idl_tests
+  - name: initial_sync_fuzzer_gen
+  - name: .jscore .common
+  - name: jsCore_column_store_indexes
+  - name: jsCore_min_batch_repeat_queries_ese_gsm
+  - name: jsCore_txns_large_txns_format
+  - name: jsCore_wildcard_indexes
+  - name: json_schema
+  - name: .jstestfuzz !.flow_control  # Flow control jstestfuzz take longer.
+  - name: libunwind_tests
+  - name: .multiversion_sanity_check
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: multiversion_gen
+  - name: powercycle_smoke
+  - name: .query_fuzzer !.cqf_only
+  - name: .random_multiversion_ds
+  - name: .read_write_concern .large
+  - name: .read_write_concern !.large
+  - name: .replica_sets !.encrypt !.auth
+  - name: replica_sets_api_version_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+  - name: replica_sets_reconfig_kill_primary_jscore_passthrough_gen
+  - name: change_streams_pre_images_replica_sets_stepdown_primary_jscore_passthrough_gen
+  - name: change_streams_pre_images_replica_sets_kill_secondary_jscore_passthrough_gen
+  - name: change_streams_change_collection_replica_sets_stepdown_primary_jscore_passthrough_gen
+  - name: change_streams_change_collection_replica_sets_kill_secondary_jscore_passthrough_gen
+  - name: retryable_writes_jscore_passthrough_gen
+  - name: retryable_writes_jscore_stepdown_passthrough_gen
+  - name: .read_only
+  - name: .rollbackfuzzer
+  - name: sasl
+  - name: search
+  - name: search_auth
+  - name: search_pinned_connections_auth
+  - name: search_ssl
+  - name: secondary_reads_passthrough_gen
+  - name: .serverless
+    distros:
+    - amazon2-arm64-large
+  - name: session_jscore_passthrough
+  - name: .shard_split
+  - name: .sharding .jscore !.wo_snapshot !.multi_stmt
+  - name: sharding_api_version_jscore_passthrough_gen
+  - name: sharding_api_strict_passthrough_gen
+  - name: .sharding .txns
+  - name: .sharding .common
+  - name: sharding_max_mirroring_opportunistic_secondary_targeting_ese_gcm_gen
+  - name: sharded_collections_single_writes_without_shard_key_jscore_passthrough_gen
+  - name: sharded_multi_stmt_txn_jscore_passthrough
+  - name: streams
+  - name: streams_auth
+  - name: .updatefuzzer
+  - name: vector_search
+  - name: vector_search_auth
+  - name: vector_search_ssl
+  - name: query_stats_passthrough
+  - name: query_stats_passthrough_writeonly
+  - name: query_stats_mongos_passthrough
+
+
+buildvariants:
+- &linux-64-debug-required-template
+  <<: *linux_x86_dynamic_debug_compile_variant_dependency
+  name: &linux-64-debug-required linux-64-debug-required
+  display_name: "! Linux x86 Shared Library DEBUG"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  run_on:
+  - rhel80-medium
+  expansions:
+    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
+    test_flags: --excludeWithAnyTags=requires_http_client
+    target_resmoke_time: 15
+    max_sub_suites: 5
+    large_distro_name: rhel80-medium
+    compile_variant: *linux_x86_dynamic_debug_compile_variant_name
+  tasks:
+  - name: .aggregation !.encrypt !.feature_flag_guarded
+  - name: .auth !.audit !.multiversion
+  - name: sharding_auth_gen
+  - name: .causally_consistent !.wo_snapshot
+  - name: .change_streams
+  - name: .clustered_collections
+  - name: .misc_js
+  - name: disk_wiredtiger
+  - name: .jscore .common
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: query_golden_classic
+  - name: libunwind_tests
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: .ocsp
+  - name: .read_write_concern
+  - name: .replica_sets !.encrypt !.fcbis
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+  - name: replica_sets_max_mirroring_large_txns_format_gen
+  - name: .retry
+  - name: .read_only
+  - name: session_jscore_passthrough
+  - name: sharded_multi_stmt_txn_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot
+  - name: sharding_gen
+  - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen
+
+- &enterprise-windows-template
+  <<: *windows_compile_variant_dependency
+  name: &enterprise-windows-required enterprise-windows-required
+  display_name: "! Enterprise Windows"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - windows-vsCurrent-small
+  expansions: &windows_required_expansions
+    compile_variant: *windows_compile_variant_name
+    burn_in_tests_build_variant: enterprise-windows-required
+    exe: ".exe"
+    content_type: application/zip
+    python: '/cygdrive/c/python/python39/python.exe'
+    ext: zip
+    multiversion_platform: windows
+    multiversion_edition: enterprise
+    jstestfuzz_num_generated_files: 35
+    target_resmoke_time: 20
+    max_sub_suites: 5
+    large_distro_name: windows-vsCurrent-large
+    push_path: windows
+    push_bucket: downloads.10gen.com
+    push_name: windows
+    push_arch: x86_64-enterprise
+    test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls
+    external_auth_jobs_max: 1
+  tasks:
+  - name: audit
+  - name: auth_audit_gen
+  - name: causally_consistent_jscore_txns_passthrough
+    distros:
+    - windows-vsCurrent-large
+  - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
+  - name: external_auth
+  - name: external_auth_aws
+  - name: external_auth_windows
+    distros:
+    - windows-2016-dc
+  - name: .jscore .common !.sharding
+  - name: jsCore_auth
+  - name: jsCore_ese
+  - name: jsCore_txns_large_txns_format
+  - name: .jstestfuzz .common
+  - name: noPassthrough_gen
+  - name: noPassthroughWithMongod_gen
+  - name: .replica_sets .common
+  - name: .replica_sets .multi_oplog !.gcm
+  - name: sasl
+  - name: .sharding .txns
+  - name: sharding_auth_audit_gen
+  - name: sharding_max_mirroring_opportunistic_secondary_targeting_ese_gen
+
+- name: run-all-affected-jstests
+  display_name: "! Run All Affected JStests"
+  patch_only: true
+  run_on:
+  - rhel80-medium
+  expansions:
+    large_distro_name: rhel80-large
+    burn_in_tag_include_all_required_and_suggested: true
+    burn_in_tag_exclude_build_variants: >-
+      macos-debug-suggested
+    burn_in_tag_include_build_variants:
+    burn_in_tag_compile_task_dependency: archive_dist_test_debug
+    compile_variant: *amazon_linux2_arm64_dynamic_compile_variant_name
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: *amazon_linux2_arm64_dynamic_compile_variant_name
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  tasks:
+  - name: burn_in_tags_gen
+
+- &enterprise-rhel-80-64-bit-dynamic-template
+  <<: *linux_x86_dynamic_compile_variant_dependency
+  name: &enterprise-rhel-80-64-bit-dynamic enterprise-rhel-80-64-bit-dynamic
+  display_name: "! Shared Library Enterprise RHEL 8.0"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-small
+  # THIS WAS COPIED TO config_shard.yml - ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THAT FILE.
+  expansions: &enterprise-rhel-80-64-bit-dynamic-expansions
+    <<: *linux_x86_generic_expansions
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    has_packages: false
+    jstestfuzz_num_generated_files: 40
+    jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 10
+    max_sub_suites: 5
+    idle_timeout_factor: 1.5
+    exec_timeout_factor: 1.5
+    large_distro_name: rhel80-medium
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: *linux_x86_dynamic_compile_variant_name
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  tasks:
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
+  - name: aggregation_repeat_queries
+  - name: analyze_shard_key_jscore_passthrough_gen
+  - name: audit
+  - name: .auth
+  - name: unittest_shell_hang_analyzer_gen
+  - name: .causally_consistent !.sharding
+  - name: .change_streams
+  - name: .change_stream_fuzzer
+  - name: .misc_js
+  - name: .concurrency !.large !.ubsan !.no_txns
+  - name: .concurrency .large !.ubsan !.no_txns
+    distros:
+    - rhel80-medium
+  - name: .config_fuzzer !.large
+  - name: .config_fuzzer .large
+    distros:
+    - rhel80-medium
+  - name: disk_wiredtiger
+  - name: .encrypt
+  - name: idl_tests
+  - name: initial_sync_fuzzer_gen
+  - name: fcv_upgrade_downgrade_replica_sets_jscore_passthrough_gen
+  - name: fcv_upgrade_downgrade_sharding_jscore_passthrough_gen
+  - name: fcv_upgrade_downgrade_sharded_collections_jscore_passthrough_gen
+  - name: jsCore
+    distros:
+    - rhel80-xlarge
+  - name: .jscore .common !jsCore
+  - name: jsCore_min_batch_repeat_queries_ese_gsm
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: .jstestfuzz !.flow_control  # Flow control jstestfuzz take longer.
+  - name: libunwind_tests
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: multiversion_gen
+  - name: .query_fuzzer !.feature_flag_guarded
+  - name: .random_multiversion_ds
+  - name: .read_write_concern .large
+    distros:
+    - rhel80-medium
+  - name: .read_write_concern !.large
+  - name: .replica_sets !.encrypt !.auth
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_api_version_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_reconfig_kill_primary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: change_streams_pre_images_replica_sets_stepdown_primary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: change_streams_pre_images_replica_sets_kill_secondary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: change_streams_change_collection_replica_sets_stepdown_primary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: change_streams_change_collection_replica_sets_kill_secondary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: retryable_writes_jscore_passthrough_gen
+  - name: retryable_writes_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-medium
+  - name: .read_only
+  - name: .rollbackfuzzer
+  - name: sasl
+  - name: search
+  - name: search_auth
+  - name: search_pinned_connections_auth
+  - name: search_ssl
+  - name: session_jscore_passthrough
+  - name: sharded_collections_single_writes_without_shard_key_jscore_passthrough_gen
+  - name: .sharding .jscore !.wo_snapshot !.multi_stmt
+  - name: sharding_api_version_jscore_passthrough_gen
+  - name: sharding_api_strict_passthrough_gen
+  - name: .sharding .txns
+  - name: .sharding .common
+  - name: .updatefuzzer
+  - name: secondary_reads_passthrough_gen
+  - name: .serverless
+    distros:
+    - rhel80-xlarge
+  - name: vector_search
+  - name: vector_search_auth
+  - name: vector_search_ssl
+
+- &rhel80-debug-aubsan-lite-template
+  <<: *linux_debug_aubsan_compile_variant_dependency
+  name: &rhel80-debug-aubsan-lite rhel80-debug-aubsan-lite
+  display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-build
+  expansions: &aubsan-lite-required-expansions
+    compile_variant: *linux_debug_aubsan_compile_variant_name
+    lang_environment: LANG=C
+    san_options: *aubsan_options
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits
+    resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under {A,UB}SAN build.
+    hang_analyzer_dump_core: false
+    max_sub_suites: 3
+    num_scons_link_jobs_available: 0.99
+    large_distro_name: rhel80-build
+    multiversion_platform: rhel80
+    multiversion_edition: enterprise
+    gcov_tool: /opt/mongodbtoolchain/v4/bin/gcov
+  tasks:
+  - name: jsCore
+  - name: jsCore_txns
+
+- <<: *enterprise-amazon-linux2-arm64-all-feature-flags-template
+  name: &commit-queue commit-queue
+  display_name: "~ Commit Queue"
+  cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter
+  stepback: false
+  expansions:
+    <<: *linux_arm64_generic_expansions
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    commit_queue_alternate_cache: amazon-linux2-arm64-compile
+    has_packages: false
+    compile_flags: >-
+      --ssl
+      MONGO_DISTMOD=amazon2
+      -j$(grep -c ^processor /proc/cpuinfo)
+      --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
+      --link-model=dynamic
+    crypt_task_compile_flags: >-
+      SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic
+      -Wl,--no-gnu-unique"
+      CCFLAGS="-fno-gnu-unique"
+    clang_tidy_toolchain: v4
+    num_scons_unit_cc_jobs_available: 0.75
+    compile_variant: *commit-queue
+  depends_on: []
+  tasks:
+  - name: compile_ninja_quick_TG
+  - name: compile_test_parallel_core_stream_TG
+    distros:
+    - amazon2-arm64-xlarge-commitqueue
+  - name: compile_test_parallel_unittest_stream_TG
+    distros:
+    - amazon2-arm64-xlarge-commitqueue
+  - name: compile_test_parallel_dbtest_stream_TG
+    distros:
+    - amazon2-arm64-xlarge-commitqueue
+  - name: jsCore
+    distros:
+    - amazon2-arm64-large
+  - name: .lint
+  - name: test_api_version_compatibility
+  - name: validate_commit_message
+  - name: lint_large_files_check
+  - name: check_feature_flag_tags
+  - name: resmoke_validation_tests
+  - name: version_gen_validation
+    distros:
+    - ubuntu2004-small

etc/evergreen_yml_components/configuration.yml

@@ -0,0 +1,138 @@
+####################################################
+#                configuration.yml                  #
+#####################################################
+#
+# This file contains global configuration values set for the mongodb-mongo-master and
+# mongodb-mongo-master-nightly projects.
+#
+
+command_type: system
+exec_timeout_secs: 21600  # 6 hours.
+oom_tracker: true
+pre_error_fails_task: true
+stepback: true
+
+## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
+parameters:
+- key: patch_compile_flags
+  description: "Additional SCons flags to be applied during scons compile invocations in this patch"
+
+- key: future_git_tag
+  description: "Future git tag to be added. If empty, we will use the most recent git tag instead."
+
+- key: last_lts_evg_version_id
+  description: "The Evergreen Version ID of the last-lts MongoDB binaries. Only binaries from release variants are used"
+
+- key: last_continuous_evg_version_id
+  description: "The Evergreen Version ID of the last-continuous MongoDB binaries. Only binaries from release variants are used"
+
+- key: antithesis_image_tag
+  description: "The docker tag to use when pushing images to Antithesis"
+
+- key: build_patch_id
+  description: "Patch id of evergreen patch to pull binaries from for testing."
+
+## Cron parameters.
+- key: project_required_suggested_cron
+  value: "0 */4 * * *" # Every 4 hours starting at 0000 UTC
+  description: "Cron schedule for required and suggested variants"
+- key: project_nightly_cron
+  value: "0 4 * * *" # Every day starting at 0400 UTC
+  description: "Cron schedule for nightly variants"
+- key: project_weekly_cron
+  value: "0 4 * * 0" # Every week starting 0400 UTC Sunday
+  description: "Cron schedule for nightly variants"
+
+
+#######################################
+#               Modules               #
+#######################################
+# if a module is added and to be added to the manifest
+# be sure to add the module to git.get_project revisions parameter
+modules:
+- name: enterprise
+  repo: git@github.com:10gen/mongo-enterprise-modules.git
+  prefix: src/mongo/db/modules
+  branch: v7.2
+
+- name: wtdevelop
+  repo: git@github.com:wiredtiger/wiredtiger.git
+  prefix: src/third_party
+  branch: develop
+
+- name: dsi
+  repo: git@github.com:10gen/dsi.git
+  prefix: ${workdir}/src
+  branch: master
+- name: genny
+  repo: git@github.com:10gen/genny.git
+  prefix: ${workdir}/src
+  branch: master
+- name: workloads
+  repo: git@github.com:10gen/workloads.git
+  prefix: ${workdir}/src
+  branch: master
+- name: linkbench
+  repo: git@github.com:10gen/linkbench.git
+  prefix: ${workdir}/src
+  branch: master
+- name: linkbench2
+  repo: git@github.com:10gen/linkbench2.git
+  prefix: ${workdir}/src
+  branch: master
+- name: tsbs
+  repo: git@github.com:mongodb-forks/tsbs.git
+  prefix: ${workdir}/src
+  branch: main
+- name: YCSB
+  repo: git@github.com:mongodb-labs/YCSB.git
+  prefix: ${workdir}/src
+  branch: production
+- name: flamegraph
+  repo: git@github.com:mongodb-forks/flamegraph.git
+  prefix: ${workdir}/src
+  branch: master
+- name: PrivateWorkloads
+  repo: git@github.com:10gen/PrivateWorkloads.git
+  prefix: ${workdir}/src
+  branch: production
+
+# Pre task steps
+pre:
+- func: "set task expansion macros"
+- func: "f_expansions_write"
+
+# Post task steps
+post:
+- func: "f_expansions_write"
+- func: "upload npm logs"
+- func: "attach local resmoke invocation"
+- func: "attach multiversion exclude tags"
+- func: "attach report"
+- func: "attach task errors"
+- func: "attach artifacts"
+- func: "save ec2 task artifacts"
+- func: "attach wiki page"
+- func: "upload docker compose"
+- func: "upload jstestfuzz minimized output"
+- func: "kill processes"
+- func: "save local client logs"
+- func: "save jepsen artifacts"
+- func: "save mongo coredumps"
+- func: "generate hang analyzer tasks"
+- func: "save failed unittests"
+- func: "save hang analyzer debugger files"
+- func: "save disk statistics"
+- func: "save system resource information"
+- func: "save UndoDB recordings"
+- func: "umount shared scons directory"
+- func: "umount tmp directory"
+- func: "cleanup FUSE watchdog"
+- func: "cleanup environment"
+- func: "cleanup jepsen docker test"
+
+# Timeout steps
+timeout:
+- func: "f_expansions_write"
+- func: "run hang analyzer"
+- func: "wait for resmoke to shutdown"

etc/evergreen_yml_components/definitions.yml

@@ -5,12 +5,6 @@
 # Any global task, function, or taskgroup definition can be added here. Note that if
 # the definition is added to another file, YAML anchors from this file can't be used.
 
-command_type: system
-exec_timeout_secs: 21600  # 6 hours.
-oom_tracker: true
-pre_error_fails_task: true
-stepback: true
-
 
 # Files that match an ignore-list pattern will not trigger a build, if they're the only modified
 # files in the patch.
@@ -27,92 +21,6 @@ ignore:
 - "!/etc/evergreen.yml"
 - "README"
 
-
-## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
-parameters:
-- key: patch_compile_flags
-  description: "Additional SCons flags to be applied during scons compile invocations in this patch"
-
-- key: future_git_tag
-  description: "Future git tag to be added. If empty, we will use the most recent git tag instead."
-
-- key: last_lts_evg_version_id
-  description: "The Evergreen Version ID of the last-lts MongoDB binaries. Only binaries from release variants are used"
-
-- key: last_continuous_evg_version_id
-  description: "The Evergreen Version ID of the last-continuous MongoDB binaries. Only binaries from release variants are used"
-
-- key: antithesis_image_tag
-  description: "The docker tag to use when pushing images to Antithesis"
-
-- key: build_patch_id
-  description: "Patch id of evergreen patch to pull binaries from for testing."
-
-## Cron parameters.
-- key: project_required_suggested_cron
-  value: "0 */4 * * *" # Every 4 hours starting at 0000 UTC
-  description: "Cron schedule for required and suggested variants"
-- key: project_nightly_cron
-  value: "0 4 * * *" # Every day starting at 0400 UTC
-  description: "Cron schedule for nightly variants"
-- key: project_weekly_cron
-  value: "0 4 * * 0" # Every week starting 0400 UTC Sunday
-  description: "Cron schedule for nightly variants"
-
-
-#######################################
-#               Modules               #
-#######################################
-# if a module is added and to be added to the manifest
-# be sure to add the module to git.get_project revisions parameter
-modules:
-- name: enterprise
-  repo: git@github.com:10gen/mongo-enterprise-modules.git
-  prefix: src/mongo/db/modules
-  branch: master
-
-- name: wtdevelop
-  repo: git@github.com:wiredtiger/wiredtiger.git
-  prefix: src/third_party
-  branch: develop
-
-- name: dsi
-  repo: git@github.com:10gen/dsi.git
-  prefix: ${workdir}/src
-  branch: master
-- name: genny
-  repo: git@github.com:10gen/genny.git
-  prefix: ${workdir}/src
-  branch: master
-- name: workloads
-  repo: git@github.com:10gen/workloads.git
-  prefix: ${workdir}/src
-  branch: master
-- name: linkbench
-  repo: git@github.com:10gen/linkbench.git
-  prefix: ${workdir}/src
-  branch: master
-- name: linkbench2
-  repo: git@github.com:10gen/linkbench2.git
-  prefix: ${workdir}/src
-  branch: master
-- name: tsbs
-  repo: git@github.com:mongodb-forks/tsbs.git
-  prefix: ${workdir}/src
-  branch: main
-- name: YCSB
-  repo: git@github.com:mongodb-labs/YCSB.git
-  prefix: ${workdir}/src
-  branch: production
-- name: flamegraph
-  repo: git@github.com:mongodb-forks/flamegraph.git
-  prefix: ${workdir}/src
-  branch: master
-- name: PrivateWorkloads
-  repo: git@github.com:10gen/PrivateWorkloads.git
-  prefix: ${workdir}/src
-  branch: production
-
 #######################################
 #             Variable                #
 #######################################
@@ -676,7 +584,7 @@ functions:
   "get buildnumber": &get_buildnumber
     command: keyval.inc
     params:
-      key: "${build_variant}_master"
+      key: "${build_variant}_v7.2"
       destination: "builder_num"
 
   "run diskstats": &run_diskstats
@@ -1637,60 +1545,6 @@ functions:
       args:
       - "./src/evergreen/jepsen_docker/cleanup.sh"
 
-  "f_dsi_pre_run":
-  - command: manifest.load
-  - command: expansions.update
-    params:
-      updates: []
-
-  "f_dsi_post_run":
-  - command: shell.exec
-    params:
-      script: ./src/dsi/run-dsi post_run
-  - command: perf.send
-    params:
-      file: ./build/CedarReports/cedar_report.json
-      aws_key: ${aws_key}
-      aws_secret: ${aws_secret}
-      bucket: genny-metrics
-      region: us-east-1
-      prefix: ${task_id}_${execution}
-  - command: attach.results
-    params:
-      file_location: ./build/EvergreenResultsJson/results.json
-  - command: s3.put
-    params:
-      aws_key: ${aws_key}
-      aws_secret: ${aws_secret}
-      local_file: ./build/Artifacts/DSIArtifacts.tgz
-      remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz
-      bucket: mciuploads
-      permissions: public-read
-      content_type: application/x-gzip
-      display_name: DSI Artifacts - Execution ${execution}
-  # TODO: https://jira.mongodb.org/browse/EVG-21107
-  #  - command: s3.put
-  #     params:
-  #       aws_key: ${aws_key}
-  #       aws_secret: ${aws_secret}
-  #       local_file: ./build/Documentation/index.html
-  #       remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html
-  #       bucket: mciuploads
-  #       permissions: public-read
-  #       content_type: text/html
-  #       display_name: Documentation
-  # TODO https://jira.mongodb.org/browse/PERF-4829
-  #   - command: s3.put
-  #     params:
-  #       aws_key: ${aws_key}
-  #       aws_secret: ${aws_secret}
-  #       local_file: bootstrap.yml
-  #       remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/bootstrap-${task_name}-${build_id}-${execution}.yml
-  #       bucket: mciuploads
-  #       permissions: public-read
-  #       content_type: text/plain
-  #       display_name: Task Bootstrap Config
-
   "run jepsen test":
   - *f_expansions_write
   - command: subprocess.exec
@@ -1944,6 +1798,7 @@ functions:
       - "src/evergreen/run_python_script.sh"
       - "buildscripts/package_test.py"
       - "release"
+      - "--evg-project=${project}"
 
   "set up EC2 instance": &set_up_ec2_instance
 
@@ -2187,24 +2042,6 @@ functions:
   - *tar_jepsen_mongos_logs
   - *archive_jepsen_mongos_logs
 
-  "save blackduck artifacts":
-  - command: archive.targz_pack
-    params:
-      target: blackduck.tgz
-      source_dir: /home/ec2-user/blackduck/runs/
-      include:
-      - "**"
-  - command: s3.put
-    params:
-      aws_key: ${aws_key}
-      aws_secret: ${aws_secret}
-      local_file: blackduck.tgz
-      remote_file: ${project}/${build_variant}/${version_id}/blackduck-${task_id}-${execution}.tgz
-      bucket: mciuploads
-      permissions: public-read
-      content_type: ${content_type|application/gzip}
-      display_name: BlackDuck logs
-
   ### Process & archive mongo coredumps ###
   "gather mongo coredumps": &gather_mongo_coredumps
     command: subprocess.exec
@@ -2739,55 +2576,10 @@ functions:
       args:
       - "./src/evergreen/antithesis_image_build_and_push.sh"
 
-# Pre task steps
-pre:
-- func: "set task expansion macros"
-- func: "f_expansions_write"
-- func: "f_dsi_pre_run"
-
-# Post task steps
-post:
-- func: "f_expansions_write"
-- func: "upload npm logs"
-- func: "attach local resmoke invocation"
-- func: "attach multiversion exclude tags"
-- func: "attach report"
-- func: "attach task errors"
-- func: "attach artifacts"
-- func: "save ec2 task artifacts"
-- func: "attach wiki page"
-- func: "upload docker compose"
-- func: "upload jstestfuzz minimized output"
-- func: "kill processes"
-- func: "save local client logs"
-- func: "save jepsen artifacts"
-- func: "save blackduck artifacts"
-- func: "save mongo coredumps"
-- func: "generate hang analyzer tasks"
-- func: "save failed unittests"
-- func: "save hang analyzer debugger files"
-- func: "save disk statistics"
-- func: "save system resource information"
-- func: "save UndoDB recordings"
-- func: "umount shared scons directory"
-- func: "umount tmp directory"
-- func: "cleanup FUSE watchdog"
-- func: "cleanup environment"
-- func: "cleanup jepsen docker test"
-- func: "f_dsi_post_run"
-
-# Timeout steps
-timeout:
-- func: "f_expansions_write"
-- func: "run hang analyzer"
-- func: "wait for resmoke to shutdown"
-
-
 #######################################
 #               Tasks                 #
 #######################################
 tasks:
-
 ## compile - build all scons targets except unittests ##
 - name: compile_dist_test
   tags: []
@@ -2950,6 +2742,8 @@ tasks:
       - "./build/**_bm.gcno"
       - "./build/**_bm.exe"
       - "./build/**_bm.pdb"
+      - "./build/**.so"
+      - "./build/**.debug"
   - command: s3.put
     params:
       aws_key: ${aws_key}
@@ -4202,18 +3996,6 @@ tasks:
       resmoke_jobs_max: 1
   - func: "send benchmark results"
 
-- <<: *benchmark_template
-  name: benchmarks_streams
-  tags: ["benchmarks"]
-  commands:
-  - func: "do benchmark setup"
-  - func: "run tests"
-    vars:
-      suites: benchmarks_streams
-      exec_timeout_secs: 18000  # 5 hour timeout.
-      resmoke_jobs_max: 1
-  - func: "send benchmark results"
-
 - <<: *run_jepsen_template
   name: jepsen_register_findAndModify
   tags: ["jepsen"]
@@ -7977,26 +7759,6 @@ tasks:
   - func: "upload pip requirements"
   - func: "run idl tests"
 
-- name: blackduck_scanner
-  tags: []
-  patchable: false
-  commands:
-  - *f_expansions_write
-  - func: "do non-compile setup"
-  - command: subprocess.exec
-    type: setup
-    params:
-      binary: bash
-      silent: true
-      args:
-      - "src/evergreen/blackduck_setup.sh"
-  - command: subprocess.exec
-    type: test
-    params:
-      binary: bash
-      args:
-      - "src/evergreen/blackduck_hub.sh"
-
 - name: tla_plus
   tags: []
   commands:
@@ -8241,6 +8003,7 @@ tasks:
   - command: shell.exec
     params:
       shell: bash
+      silent: true
       script: |
         set -oe
         echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}
@@ -8311,6 +8074,7 @@ tasks:
   - command: shell.exec
     params:
       shell: bash
+      silent: true
       script: |
         set -oe
         echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}
@@ -8849,6 +8613,7 @@ tasks:
   - command: shell.exec
     params:
       shell: bash
+      silent: true
       script: |
         set -oe
         echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}
@@ -9587,6 +9352,13 @@ task_groups:
   # - compile_all_but_not_unittests
   # - package
 
+- <<: *compile_task_group_template
+  name: compile_and_package_serial_no_unittests_TG
+  tasks:
+  - compile_dist_test
+  - archive_dist_test
+  - archive_dist_test_debug
+
 # SERVER-76006
 # This is a compile stream meant for non-cached and/or underpowered systems.
 # It joins most of the compile tasks together under a single host spread out

etc/evergreen_yml_components/perf_tasks.yml

@@ -17,6 +17,7 @@ functions:
     - command: expansions.write
       params:
         file: ./expansions.yml
+        redacted: true
     - command: shell.exec
       params:
         script: ./src/dsi/run-dsi run_workload
@@ -174,7 +175,7 @@ tasks:
       # Run the script to generate ssl cert files
       - command: shell.exec
         params:
-          script: AWS_ACCESS_KEY_ID=${perf_terraform_key} AWS_SECRET_ACCESS_KEY=${terraform_secret} ./src/dsi/run-dsi generate_ssl_cert
+          script: AWS_ACCESS_KEY_ID=${terraform_key} AWS_SECRET_ACCESS_KEY=${terraform_secret} ./src/dsi/run-dsi generate_ssl_cert
       # Upload files for further DSI usage
       - command: s3.put
         params:
@@ -201,26 +202,75 @@ tasks:
           content_type: text/plain
           display_name: root.crt
 
-  - name: industry_benchmarks
+  - name: ycsb.2023-09
     priority: 5
     commands:
       - func: f_dsi_run_workload
         vars:
-          test_control: "ycsb"
+          test_control: "ycsb.2023-09"
 
-  - name: ycsb_60GB
+  - name: ycsb_60GB.2023-09
     priority: 5
     commands:
       - func: f_dsi_run_workload
         vars:
-          test_control: "ycsb-60GB"
+          test_control: "ycsb-60GB.2023-09"
 
-  - name: ycsb_60GB.long
+  - name: ycsb_60GB.long.2023-09
     priority: 5
     commands:
       - func: f_dsi_run_workload
         vars:
-          test_control: "ycsb-60GB.long"
+          test_control: "ycsb-60GB.long.2023-09"
+
+  - name: ycsb_secondary_reads.2023-09
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb-secondary-reads.2023-09"
+
+  - name: ycsb_single_thread
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb-single-thread"
+
+  - name: ycsb_w1.2023-09
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb-w1.2023-09"
+
+  - name: ycsb_stepdowns.2023-09
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb_stepdowns.2023-09"
+
+  - name: ycsb_rolling_restarts.2023-09
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb_rolling_restarts.2023-09"
+
+  - name: ycsb_non_retryable_writes_stepdowns.2023-09
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb_non_retryable_writes_stepdowns.2023-09"
+
+  - name: ycsb_non_retryable_writes_rolling_restarts.2023-09
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ycsb_non_retryable_writes_rolling_restarts.2023-09"
 
   - name: linkbench
     priority: 5
@@ -266,6 +316,13 @@ tasks:
           test_control: "linkbench2"
           additional_tfvars: "tags: {expire-on-delta: 12}"
 
+  - name: locust_bulk_insert
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "locust_bulk_insert"
+
   - name: tsbs_load
     priority: 5
     commands:
@@ -357,6 +414,16 @@ tasks:
             {task_name: tsbs_query_optimizations,
              config_filename: ./src/genny/dist/etc/genny/workloads/query/TimeseriesTsbsOptimizations.yml}
 
+  - name: tsbs-expression-query
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "tsbs_query_genny"
+          test_control_params: |
+            {task_name: tsbs_expression_query,
+             config_filename: ./src/genny/dist/etc/genny/workloads/query/TimeseriesTsbsExpressionQuery.yml}
+
   - name: tpcc
     priority: 5
     commands:
@@ -385,6 +452,51 @@ tasks:
         vars:
           test_control: "crud_workloads_majority"
 
+  - name: cursor_manager
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "cursor_manager"
+
+  - name: mixed_workloads_genny_stepdowns
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "mixed_workloads_genny_stepdowns"
+
+  - name: mixed_workloads_genny_rolling_restarts
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "mixed_workloads_genny_rolling_restarts"
+
+  - name: big_update_10k
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "BigUpdate10k"
+
+  - name: startup
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "startup"
+
+  - name: dbcheck
+    priority: 5
+    exec_timeout_secs: 43200 # 12 hours
+    commands:
+      - func: f_dsi_run_workload
+        timeout_secs: 43200 # 12 hours
+        vars:
+          test_control: "dbcheck"
+          additional_tfvars: "tags: {expire-on-delta: 12}" # increase host expiration to 12 hours.
+
   - name: misc_workloads
     priority: 5
     commands:
@@ -406,13 +518,6 @@ tasks:
         vars:
           test_control: canaries
 
-  - name: non_sharded_workloads
-    priority: 5
-    commands:
-      - func: f_dsi_run_workload
-        vars:
-          test_control: "non_sharded"
-
   - name: bestbuy_agg
     priority: 5
     commands:
@@ -475,41 +580,6 @@ tasks:
             {scale: 4,
              columnstore: true}
 
-  - name: change_streams_latency
-    priority: 5
-    commands:
-      - func: f_dsi_run_workload
-        vars:
-          test_control: "change_streams_latency"
-
-  - name: change_streams_preimage_throughput
-    priority: 5
-    commands:
-      - func: f_dsi_run_workload
-        vars:
-          test_control: "change_streams_preimage_throughput"
-
-  - name: change_streams_preimage_latency
-    priority: 5
-    commands:
-      - func: f_dsi_run_workload
-        vars:
-          test_control: "change_streams_preimage_latency"
-
-  - name: change_streams_listen_throughput
-    priority: 5
-    commands:
-      - func: f_dsi_run_workload
-        vars:
-          test_control: "change_streams_listen_throughput"
-
-  - name: change_streams_multi_mongos
-    priority: 5
-    commands:
-      - func: f_dsi_run_workload
-        vars:
-          test_control: "change_streams_multi_mongos"
-
   - name: snapshot_reads
     priority: 5
     commands:
@@ -517,6 +587,13 @@ tasks:
         vars:
           test_control: "snapshot_reads"
 
+  - name: secondary_reads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "secondary_reads"
+
   - name: tpch_1_normalized
     priority: 5
     commands:
@@ -558,3 +635,644 @@ tasks:
           test_control_params: |
             {scale: 10,
              schema: denormalized}
+
+  - name: ssb_column_store_comparison
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "ssb_column_store_index"
+          test_control_params: |
+            {scale: 5}
+
+  - name: column_store_tpch_10_denormalized
+    priority: 5
+    exec_timeout_secs: 43200 # 12 hours
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "column_store_tpch"
+          test_control_params: |
+            {scale: 10,
+             schema: denormalized,
+             columnstore: true}
+
+  - name: column_store_tpch_10_denormalized_unindexed
+    priority: 5
+    exec_timeout_secs: 43200 # 12 hours
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "column_store_tpch"
+          test_control_params: |
+            {scale: 10,
+             schema: denormalized,
+             columnstore: false}
+
+  - name: mixed_workloads_genny_rate_limited_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mixed_workloads_genny_rate_limited
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/MixedWorkloadsGennyRateLimited.yml
+
+  - name: load_test_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: load_test
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/LoadTest.yml
+
+  - name: majority_reads10_k_threads_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: majority_reads10_k_threads
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/MajorityReads10KThreads.yml
+
+  - name: large_indexed_ins_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: large_indexed_ins
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/LargeIndexedIns.yml
+
+  - name: expressive_queries_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: expressive_queries
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/ExpressiveQueries.yml
+
+  - name: time_series_sort_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: time_series_sort
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/TimeSeriesSort.yml
+
+  - name: medical_workload_diagnosis_50_50_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: medical_workload_diagnosis_50_50
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/encrypted/medical_workload-diagnosis-50-50.yml
+
+  - name: ycsb_like_queryable_encrypt1_cfdefault_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: ycsb_like_queryable_encrypt1_cfdefault
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/encrypted/YCSBLikeQueryableEncrypt1Cfdefault.yml
+
+  - name: filter_with_complex_logical_expression_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: filter_with_complex_logical_expression
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/FilterWithComplexLogicalExpression.yml
+
+  - name: array_traversal_high_value
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: array_traversal
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/ArrayTraversal.yml
+
+# TODO PERF-3094: Remove these charts_events tasks.
+  - name: column_store_index_charts_events_1G
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "charts_events"
+          test_control_params: |
+            {scale: 1}
+
+  - name: column_store_index_charts_events_10G
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "charts_events"
+          test_control_params: |
+            {scale: 10}
+
+# TODO PERF-3094: Remove this task.
+  - name: bestbuy_4_inserts
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "bestbuy_4_inserts"
+
+  - name: non_sharded_workloads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "non_sharded"
+
+  - name: mongos_workloads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "mongos"
+
+  - name: mongos_large_catalog_workloads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "mongos_large_catalog"
+
+  - name: move_chunk_workloads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "move_chunk"
+
+  - name: move_chunk_waiting_workloads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "move_chunk_waiting"
+
+  - name: move_chunk_large_chunk_map_workloads
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "move_chunk_large_chunk_map"
+
+  - name: refine_shard_key_transaction_stress
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "refine_shard_key_transaction_stress"
+
+  - name: secondary_performance
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          # Unfortunately the dash/underscore style is different for mongodb_setup and test_control
+          test_control: "secondary_performance"
+          mongodb_setup: "secondary-performance"
+
+  - name: initialsync
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "initialsync"
+
+  - name: initialsync-fcbis
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "initialsync"
+          mongodb_setup: "replica-2node-fcbis"
+
+  - name: initialsync-logkeeper
+    priority: 5
+    exec_timeout_secs: 43200 # 12 hours
+    commands:
+      - func: f_dsi_run_workload
+        timeout_secs: 43200 # 12 hours
+        vars:
+          test_control: "initialsync-logkeeper"
+
+  - name: initialsync-logkeeper-fcbis
+    priority: 5
+    exec_timeout_secs: 43200 # 12 hours
+    commands:
+      - func: f_dsi_run_workload
+        timeout_secs: 43200 # 12 hours
+        vars:
+          test_control: "initialsync-logkeeper"
+          mongodb_setup: "initialsync-logkeeper-fcbis"
+
+  # The following two initial sync logkeeper automation tasks are only used in the commented-out
+  # "Linux ReplSet Initial Sync LogKeeper Snapshot Update" variant below and are only intended to be
+  # run in patch builds to update FCV for logkeeper datasets.
+
+  - name: initialsync-logkeeper-snapshot-update
+    priority: 5
+    exec_timeout_secs: 216000 # 2.5 days
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "initialsync-logkeeper-snapshot-update"
+
+  - name: initialsync-large
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "initialsync-large"
+
+  - name: initialsync-large-fcbis
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "initialsync-large"
+          mongodb_setup: "replica-2node-fcbis"
+
+  - name: change_streams_latency
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "change_streams_latency"
+
+  - name: change_streams_preimage_throughput
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "change_streams_preimage_throughput"
+
+  - name: change_streams_preimage_latency
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "change_streams_preimage_latency"
+
+  - name: change_streams_listen_throughput
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "change_streams_listen_throughput"
+
+  - name: change_streams_multi_mongos
+    priority: 5
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: "change_streams_multi_mongos"
+
+  - name: genny_execution_UserAcquisition
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: auto_genny_workload
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/execution/UserAcquisition.yml
+
+  - name: genny_scale_InsertRemove
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: auto_genny_workload
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/InsertRemove.yml
+
+  - name: genny_resharding_withIndexes
+    exec_timeout_secs: 172800 # 2 days
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: auto_genny_workload
+          auto_workload_path: ./src/genny/dist/etc/genny/workloads/sharding/ReshardCollectionWithIndexes.yml
+
+  - name: query_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: query,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: query_read_commands_large_dataset
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: query_large_dataset,
+             include_filter_2: regression,
+             exclude_filter: none,
+             threads: "1 4",
+             read_cmd: 'true',
+             share_dataset: 'true'}
+
+  - name: big_collection
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: query,
+             include_filter_2: getmore,
+             exclude_filter: none,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: views-query
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: query_identityview,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: views-aggregation
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: aggregation_identityview,
+             include_filter_2: regression,
+             exclude_filter: none,
+             threads: "1",
+             read_cmd: 'true'}
+
+  - name: where_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: where,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: update_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: update,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: insert_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: insert,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: compound_wildcard_index_write_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: compound-wildcard-insert compound-wildcard-remove compound-wildcard-update,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: compound_wildcard_index_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: compound-wildcard-query,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: wildcard-index-read_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: wildcard_read,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: wildcard-index-write_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: wildcard_write,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: tie-breaking-heuristics
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: tie-breaking,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: geo_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: geo,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: misc_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: command multi remove mixed,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: misc_custom_filter_default_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          mongodb_setup: mongo-perf-standalone-custom-filter-default.2023-02
+          test_control_params: |
+            {include_filter_1: command multi remove mixed,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: misc_custom_filter_slow_or_sample_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          mongodb_setup: mongo-perf-standalone-custom-filter-slow-or-sample.2023-02
+          test_control_params: |
+            {include_filter_1: command multi remove mixed,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: misc_custom_filter_complex_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          mongodb_setup: mongo-perf-standalone-custom-filter-complex.2023-02
+          test_control_params: |
+            {include_filter_1: command multi remove mixed,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: misc_custom_filter_whole_doc_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          mongodb_setup: mongo-perf-standalone-custom-filter-whole-doc.2023-02
+          test_control_params: |
+            {include_filter_1: command multi remove mixed,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: misc_slowms_everything_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          mongodb_setup: mongo-perf-standalone-slowms-everything.2023-02
+          test_control_params: |
+            {include_filter_1: command multi remove mixed,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: singleThreaded_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: single_threaded,
+             include_filter_2: core regression,
+             exclude_filter: none,
+             threads: "1",
+             read_cmd: 'true'}
+
+  - name: aggregation_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: aggregation,
+             include_filter_2: regression,
+             exclude_filter: js,
+             threads: "1",
+             read_cmd: 'true'}
+
+  - name: aggregation_read_commands_large_dataset
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: aggregation_large_dataset,
+             include_filter_2: regression,
+             exclude_filter: js,
+             threads: "1 4",
+             read_cmd: 'true',
+             share_dataset: 'true'}
+
+  - name: agg-query-comparison_read_commands
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: agg_query_comparison,
+             include_filter_2: core regression,
+             exclude_filter: single_threaded,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: pipeline-updates
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: pipeline-updates,
+             include_filter_2: regression,
+             exclude_filter: none,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: javascript
+    commands:
+      - func: f_dsi_run_workload
+        vars:
+          test_control: mongo-perf.2023-02
+          test_control_params: |
+            {include_filter_1: js,
+             include_filter_2: aggregation,
+             exclude_filter: none,
+             threads: "1 2 4 8",
+             read_cmd: 'true'}
+
+  - name: stream_workloads
+    commands:
+    - func: f_dsi_run_workload
+      vars:
+        test_control: streams.2023-10

etc/evergreen_yml_components/variants/classic_engine.yml

@@ -33,6 +33,7 @@ variables:
     idle_timeout_factor: 1.5
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
+    core_analyzer_distro_name: rhel80-large
     burn_in_tag_buildvariants: >-
       enterprise-rhel-80-64-bit-inmem
       enterprise-rhel-80-64-bit-multiversion

etc/evergreen_yml_components/variants/compile_static_analysis.yml

@@ -25,7 +25,7 @@ variables:
     - enterprise
   stepback: false
 
-- &linux-x86-compile-params  # Essential set of compile parameters used for Linux dev variants.
+- &amazon2-x86-compile-params  # Essential set of compile parameters used for Linux dev variants.
   run_on:
   - amazon2
   activate: true  # These compile variants run on every commit to reduce latency of the auto-reverter.
@@ -44,7 +44,7 @@ variables:
     --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
     --link-model=dynamic
 
-- &linux-x86-compile-expansions
+- &amazon2-x86-compile-expansions
   scons_cache_scope: shared
   scons_cache_mode: all
   has_packages: false
@@ -115,14 +115,14 @@ buildvariants:
       distros:
       - rhel80-large
 
-- <<: *linux-x86-compile-params
-  name: &linux-x86-compile linux-x86-compile #static compile
-  display_name: "Linux x86 Static Library"
+- <<: *amazon2-x86-compile-params
+  name: &amazon2-x86-compile amazon2-x86-compile #static compile
+  display_name: "Amazon2 x86 Static Compile"
   expansions:
-    <<: *linux-x86-compile-expansions
-    compile_variant: *linux-x86-compile
+    <<: *amazon2-x86-compile-expansions
+    compile_variant: *amazon2-x86-compile
   tasks:
-    - name: compile_test_parallel_core_stream_TG
+    - name: compile_and_package_serial_no_unittests_TG
     - name: generate_buildid_to_debug_symbols_mapping
     - name: run_pretty_printer_tests
 
@@ -133,25 +133,10 @@ buildvariants:
     <<: *amazon-linux2-arm64-compile-expansions
     compile_variant: *amazon-linux2-arm64-compile
   tasks:
-    - name: compile_test_parallel_core_stream_TG
+    - name: compile_and_package_serial_no_unittests_TG
     - name: generate_buildid_to_debug_symbols_mapping
     - name: run_pretty_printer_tests
 
-- <<: *generic_linux_compile_params
-  name: &linux-x86-dynamic-compile-future-tag-multiversion linux-x86-dynamic-compile-future-tag-multiversion
-  display_name: "Linux x86 Shared Library Compile (future git tag multiversion)"
-  modules:
-  - enterprise
-  expansions:
-    <<: *linux-x86-dynamic-compile-expansions
-    bv_future_git_tag: r100.0.0-9999
-    compile_variant: *linux-x86-dynamic-compile-future-tag-multiversion
-  depends_on:
-  - name: version_expansions_gen
-    variant: enterprise-rhel-80-64-bit-future-git-tag-multiversion-version-gen
-  tasks:
-  - name: compile_test_serial_TG
-
 - <<: *generic_linux_compile_params
   name: &linux-x86-dynamic-debug-compile-required linux-x86-dynamic-debug-compile-required  # TODO: replace with Sanitizer.
   display_name: "! Linux x86 Shared Library DEBUG Compile"
@@ -173,17 +158,6 @@ buildvariants:
   - name: compile_test_parallel_dbtest_stream_TG
   - name: generate_buildid_to_debug_symbols_mapping
 
-- <<: *generic_linux_compile_params
-  name: &linux-x86-dynamic-debug-wtdevelop-compile linux-x86-dynamic-debug-wtdevelop-compile
-  display_name: "~ Linux WiredTiger develop DEBUG Compile"
-  activate: false
-  modules:
-  - wtdevelop
-  expansions:
-    <<: *linux_debug_compile_expansions
-    use_wt_develop: true
-    compile_variant: *linux-x86-dynamic-debug-wtdevelop-compile
-
 - <<: *generic_linux_compile_params
   name: &linux-debug-aubsan-compile-required linux-debug-aubsan-compile-required
   display_name: "! Linux x86 Shared Library {A,UB}SAN Enterprise Compile"

etc/evergreen_yml_components/variants/config_shard.yml

@@ -37,6 +37,7 @@ variables:
     idle_timeout_factor: 1.5
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
+    core_analyzer_distro_name: rhel80-large
 
 buildvariants:
 - &enterprise-rhel-80-64-bit-dynamic-config-shard

etc/evergreen_yml_components/variants/misc_release.yml

@@ -2031,6 +2031,29 @@ buildvariants:
     - ubuntu2004-package
   - name: generate_buildid_to_debug_symbols_mapping
 
+- name: enterprise-ubuntu1804-64
+  display_name: Jepsen Tests on Enterprise Ubuntu 18.04
+  cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
+  modules:
+  - enterprise
+  run_on:
+  - ubuntu1804-small
+  expansions:
+    compile_flags: >-
+      --ssl
+      MONGO_DISTMOD=ubuntu1804
+      -j$(grep -c ^processor /proc/cpuinfo)
+      --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
+    scons_cache_scope: shared
+    large_distro_name: ubuntu1804-large
+    compile_variant: enterprise-ubuntu1804-64
+  tasks:
+  - name: compile_and_archive_dist_test_TG
+    distros:
+    - ubuntu1804-large
+  - name: .jepsen
+    distros:
+    - ubuntu1804-large
 
 - name: ubuntu2204
   display_name: Ubuntu 22.04
@@ -2873,14 +2896,3 @@ buildvariants:
   - name: streams
   - name: streams_auth
   - name: streams_kafka
-
-- name: &security-daily-cron security-daily-cron
-  modules:
-  - enterprise
-  display_name: "~ Security Daily Cron"
-  cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
-  run_on:
-  - rhel80-small
-  stepback: false
-  tasks:
-  - name: blackduck_scanner

etc/evergreen_yml_components/variants/perf.yml

@@ -0,0 +1,352 @@
+variables:
+- &amazon2_x86_compile_variant_dependency
+  depends_on:
+  - name: schedule_global_auto_tasks
+    variant: task_generation
+  - name: package_new_compile
+    variant: amazon2-x86-compile
+
+- &amazon_linux2_arm64_compile_variant_dependency
+  depends_on:
+  - name: schedule_global_auto_tasks
+    variant: task_generation
+  - name: package_new_compile
+    variant: amazon2-arm64-compile
+
+- &rhel70_x86_compile_variant_dependency
+  depends_on:
+  - name: schedule_global_auto_tasks
+    variant: task_generation
+  - name: package_new_compile
+    variant: rhel70-x86-compile
+
+- modules: &perf_modules
+    - enterprise
+    - dsi
+    - genny
+    - workloads
+    - linkbench
+    - linkbench2
+    - tsbs
+    - mongo-perf
+    - YCSB
+    - PrivateWorkloads
+    - py-tpcc
+    - flamegraph
+
+buildvariants:
+- <<: *amazon_linux2_arm64_compile_variant_dependency
+  name: perf-atlas-M60-real.arm.aws.2023-11
+  display_name: PERF M60-Atlas ReplSet ARM AWS 2023-11
+  cron: "0 0 * * 0,4"  # 00:00 on Sunday, Thursday
+  modules: *perf_modules
+  expansions:
+    mongodb_setup: atlas
+    canaries: none
+    atlas_setup: M60-repl
+    use_custom_build: true
+    infrastructure_provisioning: workload_client_arm.2023-04
+    infrastructure_provisioning_release: 2023-09
+    workload_setup: 2022-11
+    platform: linux
+    project_dir: &perf_project_dir dsi
+    storageEngine: wiredTiger
+    compile_variant: amazon2-arm64-compile
+  run_on:
+    - "rhel70-perf-atlas-large"
+  tasks:
+    - name: schedule_patch_auto_tasks
+    - name: schedule_variant_auto_tasks
+    - name: ycsb.2023-09
+    - name: ycsb_60GB.2023-09
+    - name: tpcc
+    - name: tpcc_majority
+    - name: linkbench
+    - name: linkbench2
+
+- <<: *amazon2_x86_compile_variant_dependency
+  name: perf-atlas-M60-real.intel.azure.2023-11
+  display_name: PERF M60-Atlas ReplSet Intel Azure 2023-11
+  cron: "0 0 * * 0,4"  # 00:00 on Sunday, Thursday
+  modules: *perf_modules
+  expansions:
+    mongodb_setup: atlas
+    canaries: none
+    atlas_setup: M60-repl-azure
+    use_custom_build_azure: true
+    compile_variant: amazon2-x86-compile
+    infrastructure_provisioning: workload_client_intel.2023-11
+    infrastructure_provisioning_release: 2023-09
+    workload_setup: 2022-11
+    platform: linux
+    project_dir: *perf_project_dir
+    storageEngine: wiredTiger
+  run_on:
+    - "rhel70-perf-atlas-large"
+  tasks: # Cannot use *3nodetasks because secondary_performance uses a special mongodb setup
+    - name: schedule_patch_auto_tasks
+    - name: schedule_variant_auto_tasks
+    - name: ycsb.2023-09
+    - name: ycsb_60GB.2023-09
+    - name: tpcc
+    - name: tpcc_majority
+    - name: linkbench
+    - name: linkbench2
+
+- <<: *amazon_linux2_arm64_compile_variant_dependency
+  name: perf-3-shard.arm.aws.2023-11
+  display_name: PERF 3-Shard Cluster ARM AWS 2023-11
+  cron: "0 0 * * 4"  # 00:00 on Thursday
+  modules: *perf_modules
+  expansions:
+    mongodb_setup_release: 2022-11
+    mongodb_setup: shard
+    infrastructure_provisioning_release: 2023-09
+    infrastructure_provisioning: shard
+    workload_setup: 2022-11
+    platform: linux
+    project_dir: *perf_project_dir
+    authentication: enabled
+    storageEngine: wiredTiger
+    compile_variant: amazon2-arm64-compile
+  run_on:
+    - "rhel70-perf-shard"
+  tasks:
+    - name: schedule_patch_auto_tasks
+    - name: schedule_variant_auto_tasks
+    - name: ycsb.2023-09
+    - name: ycsb_w1.2023-09
+    - name: crud_workloads_majority
+    - name: crud_workloads_w1
+    - name: misc_workloads
+    - name: map_reduce_workloads
+    - name: smoke_test
+    - name: mongos_workloads
+    - name: mongos_large_catalog_workloads
+    - name: move_chunk_workloads
+    - name: change_streams_latency
+    - name: change_streams_listen_throughput
+    - name: change_streams_multi_mongos
+    - name: tsbs_query_sharded
+    - name: tsbs_query_finance_sharded
+    - name: tsbs_query_sharded_balancer
+    - name: tsbs_query_finance_sharded_balancer
+
+- <<: *amazon_linux2_arm64_compile_variant_dependency
+  name: perf-3-node-replSet.arm.aws.2023-11
+  display_name: PERF 3-Node ReplSet ARM AWS 2023-11
+  cron: "0 0 * * 1,2,3,4,5,6"  # Everyday except Sunday at 00:00
+  modules: *perf_modules
+  expansions:
+    mongodb_setup_release: 2022-11
+    mongodb_setup: replica
+    infrastructure_provisioning_release: 2023-09
+    infrastructure_provisioning: replica
+    workload_setup: 2022-11
+    platform: linux
+    project_dir: *perf_project_dir
+    authentication: enabled
+    storageEngine: wiredTiger
+    compile_variant: amazon2-arm64-compile
+  run_on:
+    - "rhel70-perf-replset"
+  tasks:
+    - name: schedule_patch_auto_tasks
+    - name: schedule_variant_auto_tasks
+    - name: ycsb.2023-09
+    - name: ycsb_w1.2023-09
+    - name: ycsb_60GB.2023-09
+    - name: ycsb_60GB.long.2023-09
+    - name: ycsb_secondary_reads.2023-09
+    - name: crud_workloads_majority
+    - name: crud_workloads_w1
+    - name: misc_workloads
+    - name: map_reduce_workloads
+    - name: refine_shard_key_transaction_stress
+    - name: smoke_test
+    - name: secondary_performance # Uses a special 2 node mongodb setup
+    - name: non_sharded_workloads
+    - name: bestbuy_agg
+    - name: bestbuy_agg_merge_different_db
+    - name: bestbuy_agg_merge_same_db
+    - name: bestbuy_agg_merge_wordcount
+    - name: bestbuy_query
+    - name: change_streams_preimage_throughput
+    - name: change_streams_latency
+    - name: change_streams_preimage_latency
+    - name: change_streams_listen_throughput
+    - name: snapshot_reads
+    - name: secondary_reads
+    - name: tpcc
+    - name: tpcc_majority
+    - name: tpch_1_normalized
+    - name: tpch_1_denormalized
+    # TODO: Enable in SERVER-66572.
+    # - name: tpch_10_normalized
+    # - name: tpch_10_denormalized
+    - name: linkbench
+    - name: linkbench2
+    - name: tsbs_load
+    - name: tsbs_query
+    - name: tsbs_query_finance
+    - name: tsbs_query_manual_bucketing
+    - name: tsbs-query-genny
+    - name: tsbs-query-optimizations
+    - name: big_update_10k
+    - name: mixed_workloads_genny_rate_limited_high_value
+    - name: load_test_high_value
+    - name: majority_reads10_k_threads_high_value
+    - name: large_indexed_ins_high_value
+    - name: expressive_queries_high_value
+    - name: time_series_sort_high_value
+
+- <<: *amazon2_x86_compile_variant_dependency
+  name: perf-3-node-replSet-intel.intel.aws.2023-11
+  display_name: PERF 3-Node ReplSet Intel AWS 2023-11
+  cron: "0 0 * * 1,2,3,4,5,6"  # Everyday except Sunday at 00:00
+  modules: *perf_modules
+  expansions:
+    mongodb_setup_release: 2022-11
+    mongodb_setup: replica
+    infrastructure_provisioning_release: 2023-09
+    infrastructure_provisioning: replica-intel.2023-11
+    workload_setup: 2022-11
+    platform: linux
+    project_dir: *perf_project_dir
+    authentication: enabled
+    storageEngine: wiredTiger
+    compile_variant: amazon2-x86-compile
+  run_on:
+    - "rhel70-perf-replset"
+  tasks:
+    - name: schedule_patch_auto_tasks
+    - name: schedule_variant_auto_tasks
+    - name: ycsb.2023-09
+    - name: ycsb_60GB.2023-09
+    - name: crud_workloads_majority
+    - name: smoke_test
+    - name: linkbench
+    - name: linkbench2
+    - name: mixed_workloads_genny_rate_limited_high_value
+
+# On PERF-730 we changed the initial sync tests to use two nodes instead of three. To avoid
+# losing history, the name remains unchanged, but the display_name reflects the change to 2-Node.
+- <<: *amazon_linux2_arm64_compile_variant_dependency
+  name: perf-2-node-replSet-initialsync.arm.aws.2023-11
+  display_name: PERF 2-Node ReplSet Initial Sync ARM AWS 2023-11
+  cron: "0 0 * * 4"  # 00:00 on Thursday
+  modules: *perf_modules
+  expansions:
+    mongodb_setup_release: 2022-11
+    mongodb_setup: replica-2node
+    infrastructure_provisioning_release: 2023-09
+    infrastructure_provisioning: replica-2node
+    workload_setup: 2022-11
+    platform: linux
+    authentication: disabled
+    storageEngine: wiredTiger
+    compile_variant: amazon2-arm64-compile
+    project_dir: *perf_project_dir
+  run_on:
+    - "rhel70-perf-replset"
+  tasks:
+    - name: schedule_patch_auto_tasks
+    - name: schedule_variant_auto_tasks
+    - name: initialsync-large
+    - name: initialsync-large-fcbis
+
+- <<: *amazon_linux2_arm64_compile_variant_dependency
+  name: perf-mongo-perf-standalone.arm.aws.2023-11
+  display_name: PERF Monogo-Perf Standalone inMemory ARM AWS 2023-11
+  cron: &linux-microbench-cron "0 0 * * *"  # Everyday at 00:00
+  modules: *perf_modules
+  expansions:
+    mongodb_setup_release: 2022-11
+    mongodb_setup: mongo-perf-standalone.2023-02
+    infrastructure_provisioning_release: 2023-09
+    infrastructure_provisioning: workload_client_mongod_combined.2023-01
+    workload_setup: 2022-11
+    use_scons_cache: true
+    platform: linux
+    canaries: none
+    storageEngine: inMemory
+    project_dir: *perf_project_dir
+    compile_variant: amazon2-arm64-compile
+  run_on:
+  - "rhel70-perf-microbenchmarks"
+  tasks:
+    - name: big_collection
+    - name: genny_scale_InsertRemove
+    - name: genny_execution_UserAcquisition
+    - name: aggregation_read_commands
+    - name: aggregation_read_commands_large_dataset
+    - name: agg-query-comparison_read_commands
+    - name: query_read_commands
+    - name: query_read_commands_large_dataset
+    - name: views-aggregation
+    - name: views-query
+    - name: where_read_commands
+    - name: update_read_commands
+    - name: insert_read_commands
+    - name: wildcard-index-read_read_commands
+    - name: wildcard-index-write_read_commands
+    - name: geo_read_commands
+    - name: misc_read_commands
+    - name: misc_custom_filter_default_read_commands
+    - name: misc_custom_filter_slow_or_sample_read_commands
+    - name: misc_custom_filter_complex_read_commands
+    - name: misc_custom_filter_whole_doc_read_commands
+    - name: misc_slowms_everything_read_commands
+    - name: singleThreaded_read_commands
+    - name: pipeline-updates
+    - name: javascript
+    - name: compound_wildcard_index_write_commands
+    - name: compound_wildcard_index_read_commands
+
+- <<: *amazon2_x86_compile_variant_dependency
+  name: perf-mongo-perf-standalone.intel.aws.2023-11
+  display_name: PERF Mongo-Perf Standalone inMemory Intel AWS 2023-11
+  cron: *linux-microbench-cron
+  modules: *perf_modules
+  expansions:
+    mongodb_setup_release: 2022-11
+    mongodb_setup: mongo-perf-standalone.2023-02
+    infrastructure_provisioning_release: 2023-09
+    infrastructure_provisioning: workload_client_mongod_combined_intel.2023-11
+    workload_setup: 2022-11
+    use_scons_cache: true
+    platform: linux
+    canaries: none
+    storageEngine: inMemory
+    project_dir: *perf_project_dir
+    compile_variant: amazon2-x86-compile
+  run_on:
+  - "rhel70-perf-microbenchmarks"
+  tasks:
+  - name: big_collection
+  - name: genny_scale_InsertRemove
+  - name: genny_execution_UserAcquisition
+  - name: aggregation_read_commands
+  - name: aggregation_read_commands_large_dataset
+  - name: agg-query-comparison_read_commands
+  - name: query_read_commands
+  - name: query_read_commands_large_dataset
+  - name: views-aggregation
+  - name: views-query
+  - name: where_read_commands
+  - name: update_read_commands
+  - name: insert_read_commands
+  - name: wildcard-index-read_read_commands
+  - name: wildcard-index-write_read_commands
+  - name: geo_read_commands
+  - name: misc_read_commands
+  - name: misc_custom_filter_default_read_commands
+  - name: misc_custom_filter_slow_or_sample_read_commands
+  - name: misc_custom_filter_complex_read_commands
+  - name: misc_custom_filter_whole_doc_read_commands
+  - name: misc_slowms_everything_read_commands
+  - name: singleThreaded_read_commands
+  - name: pipeline-updates
+  - name: javascript
+  - name: compound_wildcard_index_write_commands
+  - name: compound_wildcard_index_read_commands

etc/evergreen_yml_components/variants/sanitizer.yml

@@ -32,6 +32,7 @@ buildvariants:
       --ssl
       --ocsp-stapling=off
       -j$(grep -c ^processor /proc/cpuinfo)
+      --link-model=dynamic
     multiversion_platform: rhel80
     multiversion_edition: enterprise
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under ASAN build.
@@ -69,6 +70,7 @@ buildvariants:
       --ssl
       --ocsp-stapling=off
       -j$(grep -c ^processor /proc/cpuinfo)
+      --link-model=dynamic
     test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling,requires_increased_memlock_limits,corrupts_data
     multiversion_platform: rhel80
     multiversion_edition: enterprise
@@ -152,6 +154,7 @@ buildvariants:
       --ssl
       --ocsp-stapling=off
       -j$(grep -c ^processor /proc/cpuinfo)
+      --link-model=dynamic
     test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits,corrupts_data
     multiversion_platform: rhel80
     multiversion_edition: enterprise

evergreen/blackduck_hub.sh

@@ -1,27 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-set -o errexit
-set -o verbose
-
-# activate the virtualenv if it has been set up
-activate_venv
-
-additional_args=
-if [ "$branch_name" != "master" ]; then
-  additional_args="--vulnerabilities_only"
-fi
-
-# BlackDuck crashes on this gzip file because it is not well-formed
-# invalid compressed data--format violated
-rm ./src/third_party/zstandard/zstd/tests/gzip/hufts-segv.gz
-
-# Remove package.json since it only exists for vscode
-# MongoDB server does not use Node.JS code so we strip this file to not confuse BlackDuck Detect
-# Otherwise we need to run npm install to install everything in package.json or disable the NPM
-# scanner.
-rm package.json
-
-python buildscripts/blackduck_hub.py -v scan_and_report --build_logger=mci.buildlogger --build_logger_task_id=${task_id} --report_file=report.json $additional_args

evergreen/blackduck_setup.sh

@@ -1,16 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-# Create the Evergreen API credentials
-cat > .restconfig.json << END_OF_CREDS
-{
-"baseurl": "${blackduck_url}",
-"username": "${blackduck_username}",
-"password": "${blackduck_password}",
-"token": "${blackduck_token}",
-"debug": false,
-"insecure" : false
-}
-END_OF_CREDS

evergreen/external_auth_azure_setup.sh

@@ -24,12 +24,15 @@ cat << EOF > $HOME/azure_e2e_config.json
     "oidc_azure_managed_identity_api_version": "${oidc_azure_managed_identity_api_version}"
 }
 EOF
-cat << EOF > $HOME/azure_remote_key
+cat << EOF > $HOME/oidc_azure_container_key
 ${oidc_azure_container_key}
 EOF
 
-# EVG project variables do not preserve line breaks - rather these are replaced with spaces, so we will need to convert our pem back into proper format
-sed s/\ OPENSSH\ PRIVATE\ KEY/OPENSSHPRIVATEKEY/g $HOME/azure_remote_key | sed s/\ /\\n/g | sed s/OPENSSHPRIVATEKEY/\ OPENSSH\ PRIVATE\ KEY/g > $HOME/azure_remote_key
+# EVG project variables do not preserve line breaks so we store them as base64 and decode here
+sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode > $HOME/azure_remote_key
+
+# Clean up temp file
+rm -f $HOME/oidc_azure_container_key
 
 # SSH will complain and fail if the private key permissions are too lenient (by default it is created with 644), so modify to run the test
 chmod 600 $HOME/azure_remote_key

jstests/aggregation/bugs/skip_limit_overflow.js

@@ -13,13 +13,13 @@
  * ]
  */
 import {aggPlanHasStage, getAggPlanStages} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.server39788;
 coll.drop();
 assert.commandWorked(db.runCommand({create: coll.getName()}));
 
-const sbeEnabled = checkSBEEnabled(db);
+const sbeFullyEnabled = checkSbeFullyEnabled(db);
 
 function testPipeline(pipeline, expectedResult, optimizedAwayStages) {
     const explainOutput = coll.explain().aggregate(pipeline);
@@ -66,25 +66,27 @@ function testPipeline(pipeline, expectedResult, optimizedAwayStages) {
 // values == MAX_LONG, another one is 1.
 testPipeline(
     [{$sort: {x: -1}}, {$skip: NumberLong("9223372036854775807")}, {$limit: 1}],
-    sbeEnabled ? {
-        LIMIT: {path: "limitAmount", expectedValue: [1]},
-        SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
-    }
-               : {
-                     $limit: {path: "$limit", expectedValue: [NumberLong(1)]},
-                     SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
-                 },
+    sbeFullyEnabled
+        ? {
+              LIMIT: {path: "limitAmount", expectedValue: [1]},
+              SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
+          }
+        : {
+              $limit: {path: "$limit", expectedValue: [NumberLong(1)]},
+              SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
+          },
     ["$skip"]);
 testPipeline(
     [{$sort: {x: -1}}, {$skip: 1}, {$limit: NumberLong("9223372036854775807")}],
-    sbeEnabled ? {
-        LIMIT: {path: "limitAmount", expectedValue: [NumberLong("9223372036854775807")]},
-        SKIP: {path: "skipAmount", expectedValue: [1]}
-    }
-               : {
-                     $limit: {path: "$limit", expectedValue: [NumberLong("9223372036854775807")]},
-                     SKIP: {path: "skipAmount", expectedValue: [1]}
-                 },
+    sbeFullyEnabled
+        ? {
+              LIMIT: {path: "limitAmount", expectedValue: [NumberLong("9223372036854775807")]},
+              SKIP: {path: "skipAmount", expectedValue: [1]}
+          }
+        : {
+              $limit: {path: "$limit", expectedValue: [NumberLong("9223372036854775807")]},
+              SKIP: {path: "skipAmount", expectedValue: [1]}
+          },
     ["$skip"]);
 
 // Case where limit + skip do not overflow. Limit == MAX_LONG and skip is 0. Should be able to
@@ -119,7 +121,7 @@ testPipeline(
         {$skip: 10},
         {$limit: 1}
     ],
-    sbeEnabled
+    sbeFullyEnabled
         ? {
               SORT: {path: "limitAmount", expectedValue: [NumberLong("9223372036854775807")]},
               SKIP: {path: "skipAmount", expectedValue: [10, NumberLong("9223372036854775800")]},
@@ -166,29 +168,31 @@ testPipeline([{$sort: {x: -1}}, {$skip: 35361718}, {$limit: 674761616283}],
 // One skip == MAX_LONG - 1, another one is 1. Should merge two skip stages into one and push down.
 testPipeline(
     [{$sort: {x: -1}}, {$skip: 1}, {$skip: NumberLong("9223372036854775806")}, {$limit: 1}],
-    sbeEnabled ? {
-        LIMIT: {path: "limitAmount", expectedValue: [1]},
-        SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
-    }
-               : {
-                     $limit: {path: "$limit", expectedValue: [NumberLong(1)]},
-                     SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
-                 },
+    sbeFullyEnabled
+        ? {
+              LIMIT: {path: "limitAmount", expectedValue: [1]},
+              SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
+          }
+        : {
+              $limit: {path: "$limit", expectedValue: [NumberLong(1)]},
+              SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
+          },
     ["$skip", "$sort"]);
 
 // Case where where overflow of limit + skip + skip prevents limit stage and one of the skip stages
 // from being absorbed.  One skip == MAX_LONG, another one is 1. Should absorb the first skip.
 testPipeline(
     [{$sort: {x: -1}}, {$skip: 1}, {$skip: NumberLong("9223372036854775807")}, {$limit: 1}],
-    (sbeEnabled ? {
-        LIMIT: {path: "limitAmount", expectedValue: [1]},
-        SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807"), 1]},
-    }
-                : {
-                      $limit: {path: "$limit", expectedValue: [NumberLong(1)]},
-                      SKIP: {path: "skipAmount", expectedValue: [1]},
-                      $skip: {path: "$skip", expectedValue: [NumberLong("9223372036854775807")]}
-                  }),
+    (sbeFullyEnabled
+         ? {
+               LIMIT: {path: "limitAmount", expectedValue: [1]},
+               SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807"), 1]},
+           }
+         : {
+               $limit: {path: "$limit", expectedValue: [NumberLong(1)]},
+               SKIP: {path: "skipAmount", expectedValue: [1]},
+               $skip: {path: "$skip", expectedValue: [NumberLong("9223372036854775807")]}
+           }),
     ["$sort"]);
 
 // Cases where both limit and skip == MAX_LONG.
@@ -210,14 +214,15 @@ testPipeline(
         {$skip: NumberLong("9223372036854775807")},
         {$limit: NumberLong("9223372036854775807")}
     ],
-    sbeEnabled ? {
-        LIMIT: {path: "limitAmount", expectedValue: [NumberLong("9223372036854775807")]},
-        SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
-    }
-               : {
-                     $limit: {path: "$limit", expectedValue: [NumberLong("9223372036854775807")]},
-                     SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
-                 },
+    sbeFullyEnabled
+        ? {
+              LIMIT: {path: "limitAmount", expectedValue: [NumberLong("9223372036854775807")]},
+              SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
+          }
+        : {
+              $limit: {path: "$limit", expectedValue: [NumberLong("9223372036854775807")]},
+              SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807")]}
+          },
     ["$skip", "$sort"]);
 
 // Case where sample size is > MAX_LONG.
@@ -241,7 +246,7 @@ testPipeline(
         {$skip: NumberLong("3")},
         {$skip: NumberLong("4")},
     ],
-    sbeEnabled
+    sbeFullyEnabled
         ? {SKIP: {path: "skipAmount", expectedValue: [NumberLong("9223372036854775807"), 10]}}
         : {
               $skip: {path: "$skip", expectedValue: [NumberLong("9223372036854775807")]},

jstests/aggregation/explain_limit.js

@@ -9,20 +9,20 @@
 //   assumes_no_implicit_index_creation,
 // ]
 import {getAggPlanStages} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 let coll = db.explain_limit;
 
 const kCollSize = 105;
 const kLimit = 10;
-const isSBEEnabled = checkSBEEnabled(db);
+const isSbeFullyEnabled = checkSbeFullyEnabled(db);
 
 // Return whether or explain() was successful and contained the appropriate fields given the
 // requested verbosity. Checks that the number of documents examined and returned are correct given
 // the value of the limit.
 function checkResults({results, verbosity}) {
     const [cursorSubdocs, limitAmount] = (() => {
-        if (verbosity != "queryPlanner" && isSBEEnabled) {
+        if (verbosity != "queryPlanner" && isSbeFullyEnabled) {
             // We cannot use the "executionStats" section for SBE plans without some pre-processing,
             // since it has different explain format. To find execution stats for the LIMIT stages
             // from the "queryPlanner" section (there could be multiple of such stages if we're in a

jstests/aggregation/expressions/concat_arrays.js

@@ -14,7 +14,7 @@
 import "jstests/libs/sbe_assert_error_override.js";
 
 import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.projection_expr_concat_arrays;
 coll.drop();
@@ -133,7 +133,7 @@ runAndAssertThrows(["$dbl_arr", "$dbl_val"]);
 // Confirm edge case where if invalid input precedes null or missing inputs, the command fails.
 // Note that when the SBE engine is enabled, null will be returned before invalid input because
 // we check if any values are null before checking whether all values are arrays.
-let evalFn = checkSBEEnabled(db) ? runAndAssertNull : runAndAssertThrows;
+let evalFn = checkSbeFullyEnabled(db) ? runAndAssertNull : runAndAssertThrows;
 evalFn(["$int_arr", "$dbl_val", "$null_val"]);
 evalFn(["$int_arr", "some_string_value", "$null_val"]);
 evalFn(["$dbl_val", "$null_val"]);

jstests/aggregation/expressions/internal_key_string_value.js

@@ -0,0 +1,195 @@
+// Tests the behavior of $_internalKeyStringValue when used in agg expressions.
+
+const coll = db[jsTestName()];
+
+// Testing behavior for basic data types.
+coll.drop();
+let docs = [
+    {_id: 0, a: NumberInt(1)},
+    {_id: 1, a: NumberLong(2)},
+    {_id: 2, a: 3.0},
+    {_id: 3, a: NumberDecimal("4.000000")},
+    {_id: 4, a: "abc"},
+    {_id: 5, a: ISODate("2024-01-01")},
+    {_id: 6, a: [1, 2, 3]},
+    {_id: 7, a: {b: 1, c: 2, d: 3}},
+];
+assert.commandWorked(coll.insert(docs));
+
+let results =
+    coll.aggregate(
+            [{$sort: {_id: 1}}, {$addFields: {b: {$_internalKeyStringValue: {input: "$a"}}}}])
+        .toArray();
+assert.eq(results,
+          [
+              {_id: 0, a: NumberInt(1), b: BinData(0, "KwIE")},
+              {_id: 1, a: NumberLong(2), b: BinData(0, "KwQE")},
+              {_id: 2, a: 3.0, b: BinData(0, "KwYE")},
+              {_id: 3, a: NumberDecimal("4.000000"), b: BinData(0, "KwgE")},
+              {_id: 4, a: "abc", b: BinData(0, "PGFiYwAE")},
+              {_id: 5, a: ISODate("2024-01-01"), b: BinData(0, "eIAAAYzCUfQABA==")},
+              {_id: 6, a: [1, 2, 3], b: BinData(0, "UCsCKwQrBgAE")},
+              {_id: 7, a: {b: 1, c: 2, d: 3}, b: BinData(0, "Rh5iACsCHmMAKwQeZAArBgAE")},
+          ],
+          results);
+
+// Testing behavior for same numeric values of different types.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: NumberInt(1)},
+    {_id: 1, a: NumberLong(1)},
+    {_id: 2, a: 1.0},
+    {_id: 3, a: NumberDecimal("1.000")},
+    {_id: 4, a: NumberDecimal("1.000000")},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate(
+                  [{$sort: {_id: 1}}, {$addFields: {b: {$_internalKeyStringValue: {input: "$a"}}}}])
+              .toArray();
+assert.eq(docs.length, results.length, results);
+assert(results.every(result => bsonWoCompare(result.b, results[0].b) === 0), results);
+
+// Testing behavior for same numeric values of different types inside object.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: {b: NumberInt(1)}},
+    {_id: 1, a: {b: 1.0}},
+    {_id: 2, a: {b: NumberDecimal("1.000")}},
+    {_id: 3, a: {b: NumberDecimal("1.000000")}},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate(
+                  [{$sort: {_id: 1}}, {$addFields: {c: {$_internalKeyStringValue: {input: "$a"}}}}])
+              .toArray();
+assert.eq(docs.length, results.length, results);
+assert(results.every(result => bsonWoCompare(result.c, results[0].c) === 0), results);
+
+// Testing behavior for close numeric values.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: 1},
+    {_id: 1, a: 1.00001},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate(
+                  [{$sort: {_id: 1}}, {$addFields: {b: {$_internalKeyStringValue: {input: "$a"}}}}])
+              .toArray();
+assert.eq(2, results.length, results);
+assert(bsonWoCompare(results[0].b, results[1].b) !== 0, results);
+
+// Testing behavior for large numeric values.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: 1e20},
+    {_id: 1, a: NumberDecimal("1e21")},
+];
+assert.commandWorked(coll.insert(docs));
+
+results =
+    coll.aggregate([
+            {$sort: {_id: 1}},
+            {
+                $addFields:
+                    {b: {$_internalKeyStringValue: {input: "$a"}}, c: {$toHashedIndexKey: "$a"}}
+            }
+        ])
+        .toArray();
+assert.eq(2, results.length, results);
+assert(bsonWoCompare(results[0].b, results[1].b) !== 0, results);
+// $toHashedIndexKey hashes large numbers greater than 2^63 to the same result.
+assert(bsonWoCompare(results[0].c, results[1].c) === 0, results);
+
+// Testing behavior for strings under case-sensitive collation that doesn't match.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: "aAa"},
+    {_id: 1, a: "AaA"},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate([
+                  {$sort: {_id: 1}},
+                  {
+                      $addFields: {
+                          b: {
+                              $_internalKeyStringValue:
+                                  {input: "$a", collation: {locale: "en", strength: 3}}
+                          }
+                      }
+                  }
+              ])
+              .toArray();
+assert.eq(2, results.length, results);
+assert(bsonWoCompare(results[0].b, results[1].b) !== 0, results);
+
+// Testing behavior for strings under case-sensitive collation that matches.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: "aAa"},
+    {_id: 1, a: "aAa"},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate([
+                  {$sort: {_id: 1}},
+                  {
+                      $addFields: {
+                          b: {
+                              $_internalKeyStringValue:
+                                  {input: "$a", collation: {locale: "en", strength: 3}}
+                          }
+                      }
+                  }
+              ])
+              .toArray();
+assert.eq(docs.length, results.length, results);
+assert(results.every(result => bsonWoCompare(result.c, results[0].c) === 0), results);
+
+// Testing behavior for strings under case-insensitive collation that doesn't match.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: "aAa"},
+    {_id: 1, a: "aBa"},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate([
+                  {$sort: {_id: 1}},
+                  {
+                      $addFields: {
+                          b: {
+                              $_internalKeyStringValue:
+                                  {input: "$a", collation: {locale: "en", strength: 1}}
+                          }
+                      }
+                  }
+              ])
+              .toArray();
+assert.eq(2, results.length, results);
+assert(bsonWoCompare(results[0].b, results[1].b) !== 0, results);
+
+// Testing behavior for strings under case-insensitive collation that matches.
+assert(coll.drop());
+docs = [
+    {_id: 0, a: "aAa"},
+    {_id: 1, a: "AaA"},
+];
+assert.commandWorked(coll.insert(docs));
+
+results = coll.aggregate([
+                  {$sort: {_id: 1}},
+                  {
+                      $addFields: {
+                          b: {
+                              $_internalKeyStringValue:
+                                  {input: "$a", collation: {locale: "en", strength: 1}}
+                          }
+                      }
+                  }
+              ])
+              .toArray();
+assert.eq(docs.length, results.length, results);
+assert(results.every(result => bsonWoCompare(result.b, results[0].b) === 0), results);

jstests/aggregation/optimize_away_pipeline.js

@@ -25,9 +25,10 @@ import {
     planHasStage,
 } from "jstests/libs/analyze_plan.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled, checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
-const sbeEnabled = checkSBEEnabled(db);
+const sbeFullyEnabled = checkSbeFullyEnabled(db);
+const sbeRestricted = checkSbeRestrictedOrFullyEnabled(db);
 
 const coll = db.optimize_away_pipeline;
 coll.drop();
@@ -144,10 +145,14 @@ function testGetMore({command = null, expectedResult = null} = {}) {
     assert.sameMembers(documents, expectedResult);
 }
 
-// Calls 'assertPushdownEnabled' if sbeEnabled is 'true'. Otherwise, it calls
+// Calls 'assertPushdownEnabled' if sbeFullyEnabled is 'true' or if pipeline has SBE eligible stages
+// ($group, $lookup, $_internalUnpackBucket, and $search) under sbeRestricted. Otherwise, it calls
 // 'assertPushdownDisabled'.
-function assertPipelineIfSbeEnabled(assertPushdownEnabled, assertPushdownDisabled) {
-    return sbeEnabled ? assertPushdownEnabled() : assertPushdownDisabled();
+function assertPipelineIfSbeEnabled(
+    assertPushdownEnabled, assertPushdownDisabled, hasEligibleRestrictedStage = false) {
+    return sbeFullyEnabled || (sbeRestricted && hasEligibleRestrictedStage)
+        ? assertPushdownEnabled()
+        : assertPushdownDisabled();
 }
 
 // Basic pipelines.
@@ -284,7 +289,8 @@ assertPipelineIfSbeEnabled(
             expectedStages: ["COLLSCAN", "PROJECTION_SIMPLE"],
             expectedResult: [{_id: "null", s: 50}],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 // Test that we can optimize away a pipeline with a $text search predicate.
 assert.commandWorked(coll.createIndex({y: "text"}));
@@ -435,7 +441,8 @@ assertPipelineIfSbeEnabled(
             expectedResult: [{_id: null, s: 30}],
             optimizedAwayStages: ["$sort", "$limit"],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 // Test that $limit can be pushed down before a group, but it prohibits the DISTINCT_SCAN
 // optimization.
@@ -465,7 +472,8 @@ assertPipelineIfSbeEnabled(
             expectedStages: ["COLLSCAN", "LIMIT"],
             optimizedAwayStages: ["$limit"],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 pipeline = [{$sort: {x: 1}}, {$limit: 2}, {$group: {_id: "$x"}}];
 assertPipelineIfSbeEnabled(
@@ -481,7 +489,8 @@ assertPipelineIfSbeEnabled(
             expectedStages: ["IXSCAN", "PROJECTION_COVERED", "LIMIT"],
             optimizedAwayStages: ["$sort", "$limit"],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 // $limit after a group has no effect on our ability to produce a DISTINCT_SCAN plan.
 assertPipelineUsesAggregation({
@@ -581,7 +590,8 @@ assertPipelineIfSbeEnabled(
             pipeline: pipeline,
             expectedStages: ["COLLSCAN", "PROJECTION_SIMPLE"],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 pipeline = [{$group: {_id: "$a", b: {$sum: "$b"}}}, {$group: {_id: "$c", x: {$sum: "$b"}}}];
 assertPipelineIfSbeEnabled(
     function() {
@@ -598,7 +608,8 @@ assertPipelineIfSbeEnabled(
             pipeline: pipeline,
             expectedStages: ["COLLSCAN", "PROJECTION_SIMPLE"],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 function assertTransformByShape(expected, actual, message) {
     assert.eq(Object.keys(expected).sort(), Object.keys(actual).sort(), message);
@@ -622,7 +633,8 @@ assertPipelineIfSbeEnabled(
         let projStage = getAggPlanStage(explain, "PROJECTION_SIMPLE");
         assert.neq(null, projStage, explain);
         assertTransformByShape({a: 1, b: 1, _id: 0}, projStage.transformBy, explain);
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 // Similar as above, but with $addFields stage at the front of the pipeline.
 pipeline = [{$addFields: {z: "abc"}}, {$group: {_id: "$a", b: {$sum: "$b"}}}];
@@ -660,7 +672,8 @@ function assertProjectionCanBeRemovedBeforeGroup(pipeline, projectionType = "PRO
                 pipeline: pipeline,
                 expectedStages: ["COLLSCAN", projectionType, "$group"],
             });
-        });
+        },
+        true /* hasEligibleRestrictedStage */);
 }
 
 // Asserts that a projection stage is not optimized out of a pipeline with a projection and a group
@@ -679,7 +692,8 @@ function assertProjectionIsNotRemoved(pipeline, projectionType = "PROJECTION_SIM
                 pipeline: pipeline,
                 expectedStages: ["COLLSCAN", projectionType, "$group"],
             });
-        });
+        },
+        true /* hasEligibleRestrictedStage */);
 }
 
 // Test that an inclusion projection is optimized away if it is redundant/unnecessary.
@@ -741,7 +755,8 @@ assertPipelineIfSbeEnabled(
             pipeline: pipeline,
             expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT", "$group"],
         });
-    });
+    },
+    true /* hasEligibleRestrictedStage */);
 
 // We generate a projection stage from dependency analysis, even if the pipeline begins with an
 // exclusion projection.

jstests/aggregation/query_limits_test.js

@@ -11,7 +11,6 @@
  */
 
 import {checkCascadesOptimizerEnabled} from "jstests/libs/optimizer_utils.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
 
 (function() {
 "use strict";
@@ -27,7 +26,6 @@ if (debugBuild || !_optimizationsEnabled() || _isAddressSanitizerActive() ||
 }
 
 const isBonsaiEnabled = checkCascadesOptimizerEnabled(db);
-const isSBEEnabled = checkSBEEnabled(db);
 
 const coll = db.query_limits_test;
 coll.drop();

jstests/aggregation/sources/facet/facet_stats.js

@@ -8,6 +8,8 @@
 //  assumes_read_concern_unchanged,
 //  assumes_against_mongod_not_mongos,
 //  does_not_support_repeated_reads,
+//  # oplog queries could affect the statistics
+//  assumes_standalone_mongod
 // ]
 
 const testDB = db.getSiblingDB("facet_stats");
@@ -50,4 +52,4 @@ curScannedKeys = queryExecutor.scanned - curScannedKeys;
 // cardinality(local) + cardinality(local) * cardinality(foreign) = 3 + 3 * 3 = 12.
 assert.eq(12, curScannedObjects);
 // $facet sub-pipelines cannot make use of indexes. Hence scanned keys should be 0.
-assert.eq(0, curScannedKeys);
+assert.eq(0, curScannedKeys);

jstests/aggregation/sources/lookup/lookup_equijoin_semantics_hj.js

@@ -1,16 +1,12 @@
 /**
  * Tests for $lookup with localField/foreignField syntax using hash join algorithm.
+ *
+ * @tags: [featureFlagSbeFull]
  */
 import {
     JoinAlgorithm,
     runTests
 } from "jstests/aggregation/sources/lookup/lookup_equijoin_semantics_lib.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
-if (!checkSBEEnabled(db)) {
-    jsTestLog("Skipping the test because it only applies to $lookup in SBE");
-    quit();
-}
 
 runTests({
     localColl: db.lookup_arrays_semantics_local_hj,

jstests/aggregation/sources/lookup/lookup_equijoin_semantics_lib.js

@@ -16,7 +16,7 @@
 
 import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
 import {getAggPlanStages} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
 export const JoinAlgorithm = {
     HJ: {name: "HJ", strategy: "HashJoin"},
@@ -47,7 +47,7 @@ export function setupCollections(testConfig, localRecords, foreignRecords, forei
 export function checkJoinConfiguration(testConfig, explain) {
     const {currentJoinAlgorithm} = testConfig;
     const eqLookupNodes = getAggPlanStages(explain, "EQ_LOOKUP");
-    if (checkSBEEnabled(db)) {
+    if (checkSbeRestrictedOrFullyEnabled(db)) {
         if (eqLookupNodes.length > 0) {
             // The $lookup stage has been lowered. Check that it's using the expected join strategy.
             assert.eq(currentJoinAlgorithm.strategy, eqLookupNodes[0].strategy, "Join strategy");
@@ -264,7 +264,9 @@ export function runTests(testConfig) {
             {_id: 11, a: [[null, 1], 2]},
         ];
 
-        if (checkSBEEnabled(db)) {
+        // $lookup is allowed to run with sbe when internalQueryFrameworkControl is set to
+        // 'trySbeRestricted'.
+        if (checkSbeRestrictedOrFullyEnabled(db)) {
             // When lowered to SBE, "undefined" should only match "undefined".
             runTest_SingleForeignRecord(testConfig, {
                 testDescription: "Undefined in foreign, top-level field in local",

jstests/aggregation/sources/lookup/lookup_query_stats.js

@@ -19,9 +19,9 @@ import {
     getQueryInfoAtTopLevelOrFirstStage,
     getSbePlanStages
 } from "jstests/libs/sbe_explain_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
-const isSBELookupEnabled = checkSBEEnabled(db);
+const isSBELookupEnabled = checkSbeRestrictedOrFullyEnabled(db);
 const testDB = db.getSiblingDB("lookup_query_stats");
 testDB.dropDatabase();
 

jstests/aggregation/sources/match/skip_with_limit.js

@@ -143,4 +143,4 @@ assertAggregateCommandFailed([{$limit: -1}], 5107201);
 assert.eq(coll.aggregate([{$skip: 0}]).itcount(), 20);
 
 // Verifies that command fails executing when $limit argument is zero.
-assertAggregateCommandFailed([{$limit: 0}], 15958);
+assertAggregateCommandFailed([{$limit: 0}], 15958);

jstests/aggregation/sources/project/remove_redundant_projects.js

@@ -4,14 +4,18 @@
 //   do_not_wrap_aggregations_in_facets,
 //   requires_pipeline_optimization,
 // ]
-import {orderedArrayEq} from "jstests/aggregation/extras/utils.js";
+import {documentEq, orderedArrayEq} from "jstests/aggregation/extras/utils.js";
 import {
+    getPlanStages,
     getWinningPlan,
     isAggregationPlan,
-    isQueryPlan,
-    planHasStage
+    isQueryPlan
 } from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {
+    checkSbeCompletelyDisabled,
+    checkSbeFullyEnabled,
+    checkSbeRestrictedOrFullyEnabled
+} from "jstests/libs/sbe_util.js";
 
 let coll = db.remove_redundant_projects;
 coll.drop();
@@ -20,17 +24,20 @@ assert.commandWorked(coll.insert({_id: {a: 1, b: 1}, a: 1, c: {d: 1}, e: ['elem1
 
 let indexSpec = {a: 1, 'c.d': 1, 'e.0': 1};
 
-const sbeEnabled = checkSBEEnabled(db);
+const sbeFullyEnabled = checkSbeFullyEnabled(db);
+const sbeRestricted = checkSbeRestrictedOrFullyEnabled(db);
 
 /**
  * Helper to test that for a given pipeline, the same results are returned whether or not an
  * index is present.  Also tests whether a projection is absorbed by the pipeline
  * ('expectProjectToCoalesce') and the corresponding project stage ('removedProjectStage') does
- * not exist in the explain output.
+ * not exist in the explain output. When 'expectProjectToCoalesce' is true, the caller should
+ * specify which projects are expected to coalesce in 'expectedCoalescedProjects'.
  */
 function assertResultsMatch({
     pipeline = [],
     expectProjectToCoalesce = false,
+    expectedCoalescedProjects = [],
     removedProjectStage = null,
     index = indexSpec,
     pipelineOptimizedAway = false
@@ -55,13 +62,34 @@ function assertResultsMatch({
             assert(isAggregationPlan(explain), explain);
             result = getWinningPlan(explain.stages[0].$cursor.queryPlanner);
         }
+        // Check that $project uses the query system and all expectedCoalescedProjects are
+        // actually present in explain.
+        if (expectProjectToCoalesce) {
+            assert.gte(
+                expectedCoalescedProjects.length,
+                1,
+                "If we expect project to coalesce, there should be at least one such projection in expectedCoalescedProjects " +
+                    tojson(expectedCoalescedProjects));
 
-        // Check that $project uses the query system.
-        assert.eq(expectProjectToCoalesce,
-                  planHasStage(db, result, "PROJECTION_DEFAULT") ||
-                      planHasStage(db, result, "PROJECTION_COVERED") ||
-                      planHasStage(db, result, "PROJECTION_SIMPLE"),
-                  explain);
+            const projects = [
+                ...getPlanStages(result, "PROJECTION_DEFAULT"),
+                ...getPlanStages(result, "PROJECTION_COVERED"),
+                ...getPlanStages(result, "PROJECTION_SIMPLE")
+            ];
+
+            assert.gte(projects.length, 1, explain);
+
+            const areAllexpectedCoalescedProjectsPresent =
+                expectedCoalescedProjects.every(coalescedProject => {
+                    return projects.some(project => {
+                        return documentEq(project.transformBy, coalescedProject);
+                    });
+                });
+
+            assert(areAllexpectedCoalescedProjectsPresent,
+                   "There were missing or extra coalesced projects in " +
+                       tojson(expectedCoalescedProjects) + " with explain " + tojson(explain));
+        }
 
         if (!pipelineOptimizedAway) {
             // Check that $project was removed from pipeline and pushed to the query system.
@@ -84,17 +112,20 @@ function assertResultsMatch({
 assertResultsMatch({
     pipeline: [{$project: {_id: 0, a: 1}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"a": true, "_id": false}],
     pipelineOptimizedAway: true
 });
 assertResultsMatch({
     pipeline: [{$project: {_id: 0, a: 1}}, {$group: {_id: null, a: {$sum: "$a"}}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"a": true, "_id": false}],
     removedProjectStage: {_id: 0, a: 1},
-    pipelineOptimizedAway: sbeEnabled
+    pipelineOptimizedAway: sbeRestricted
 });
 assertResultsMatch({
     pipeline: [{$sort: {a: -1}}, {$project: {_id: 0, a: 1}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"a": true, "_id": false}],
     pipelineOptimizedAway: true
 });
 assertResultsMatch({
@@ -104,12 +135,14 @@ assertResultsMatch({
         {$group: {_id: "$a", a: {$sum: "$a"}}}
     ],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"a": true, "_id": false}],
     removedProjectStage: {_id: 0, a: 1},
-    pipelineOptimizedAway: sbeEnabled
+    pipelineOptimizedAway: sbeRestricted
 });
 assertResultsMatch({
     pipeline: [{$project: {_id: 0, c: {d: 1}}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"c": {"d": true}, "_id": false}],
     pipelineOptimizedAway: true
 });
 
@@ -117,11 +150,13 @@ assertResultsMatch({
 assertResultsMatch({
     pipeline: [{$project: {_id: 0, f: "$a"}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"f": "$a", "_id": false}],
     pipelineOptimizedAway: true
 });
 assertResultsMatch({
     pipeline: [{$project: {_id: 0, a: 1, f: "$a"}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"a": true, "f": "$a", "_id": false}],
     pipelineOptimizedAway: true
 });
 
@@ -129,29 +164,44 @@ assertResultsMatch({
 assertResultsMatch({
     pipeline: [{$sort: {a: 1}}, {$project: {_id: 1, b: 1}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"_id": true, "b": true}],
     pipelineOptimizedAway: true
 });
+
+// When SBE is not enabled, we end up pushing down a projection which is internally generated by the
+// aggregation subsystem's dependency analysis logic.
 assertResultsMatch({
     pipeline: [{$sort: {a: 1}}, {$group: {_id: "$_id", a: {$sum: "$a"}}}, {$project: {arr: 1}}],
-    expectProjectToCoalesce: true,
-    pipelineOptimizedAway: sbeEnabled
+    expectProjectToCoalesce: checkSbeCompletelyDisabled(db) || sbeFullyEnabled,
+    expectedCoalescedProjects: sbeFullyEnabled ? [{"_id": true, "arr": true}]
+                                               : [{"_id": 1, "a": 1}],
+    pipelineOptimizedAway: sbeFullyEnabled
 });
 
 // Test that projections with computed fields are removed from the pipeline.
 assertResultsMatch({
     pipeline: [{$project: {computedField: {$sum: "$a"}}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"_id": true, "computedField": {"$sum": ["$a"]}}],
     pipelineOptimizedAway: true
 });
 assertResultsMatch({
     pipeline: [{$project: {a: ["$a", "$b"]}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"_id": true, "a": ["$a", "$b"]}],
     pipelineOptimizedAway: true
 });
 assertResultsMatch({
     pipeline:
         [{$project: {e: {$filter: {input: "$e", as: "item", cond: {"$eq": ["$$item", "elem0"]}}}}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{
+        "_id": true,
+        "e": {
+            "$filter":
+                {"input": "$e", "as": "item", "cond": {"$eq": ["$$item", {"$const": "elem0"}]}}
+        }
+    }],
     pipelineOptimizedAway: true
 });
 
@@ -162,7 +212,9 @@ assertResultsMatch({
         {$project: {_id: 0}}
     ],
     expectProjectToCoalesce: true,
-    pipelineOptimizedAway: sbeEnabled
+    expectedCoalescedProjects: sbeFullyEnabled ? [{"a": true, "_id": false}, {"_id": false}]
+                                               : [{"a": true, "_id": false}],
+    pipelineOptimizedAway: sbeFullyEnabled
 });
 
 // Test that projections on _id with nested fields are removed from pipeline.
@@ -173,6 +225,7 @@ indexSpec = {
 assertResultsMatch({
     pipeline: [{$match: {"_id.a": 1}}, {$project: {'_id.a': 1}}],
     expectProjectToCoalesce: true,
+    expectedCoalescedProjects: [{"_id": {"a": true}}],
     index: indexSpec,
     pipelineOptimizedAway: true,
     removedProjectStage: {'_id.a': 1},

jstests/aggregation/sources/setWindowFields/first.js

@@ -6,8 +6,6 @@ import {
     testAccumAgainstGroup
 } from "jstests/aggregation/extras/window_function_helpers.js";
 
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
 const coll = db[jsTestName()];
 coll.drop();
 

jstests/aggregation/sources/setWindowFields/last.js

@@ -6,8 +6,6 @@ import {
     testAccumAgainstGroup
 } from "jstests/aggregation/extras/window_function_helpers.js";
 
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
 const coll = db[jsTestName()];
 coll.drop();
 

jstests/aggregation/sources/setWindowFields/spill_to_disk.js

@@ -19,7 +19,6 @@ import {getAggPlanStages} from "jstests/libs/analyze_plan.js";
 import {DiscoverTopology} from "jstests/libs/discover_topology.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
 import {getLatestProfilerEntry} from "jstests/libs/profiler.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
 import {setParameterOnAllHosts} from "jstests/noPassthrough/libs/server_parameter_helpers.js";
 
 // Doc size was found through logging the size in the SpillableCache. Partition sizes were chosen
@@ -214,18 +213,24 @@ function testUsedDiskAppearsInExplain() {
         {$sort: {_id: 1}}
     ];
 
-    const stageName =
-        checkSBEEnabled(db, ["featureFlagSbeFull"]) ? "window" : "$_internalSetWindowFields";
-    let stages = getAggPlanStages(
-        coll.explain("allPlansExecution").aggregate(explainPipeline, {allowDiskUse: true}),
-        stageName);
+    let explainAllPlansExecution =
+        coll.explain("allPlansExecution").aggregate(explainPipeline, {allowDiskUse: true});
+
+    // If setWindowFields is pushed down to SBE, the stage name in explain will be 'window',
+    // otherwise it will be '$_internalSetWindowFields'.
+    let stages =
+        getAggPlanStages(explainAllPlansExecution, "window")
+            .concat(getAggPlanStages(explainAllPlansExecution, "$_internalSetWindowFields"));
+    assert.gt(stages.length, 0, stages);
     assert(stages[0]["usedDisk"], stages);
 
     // Run an explain query with the default memory limit, so 'usedDisk' should be false.
     changeSpillLimit({mode: 'off', maxDocs: null});
-    stages = getAggPlanStages(
-        coll.explain("allPlansExecution").aggregate(explainPipeline, {allowDiskUse: true}),
-        stageName);
+    explainAllPlansExecution =
+        coll.explain("allPlansExecution").aggregate(explainPipeline, {allowDiskUse: true});
+    stages = getAggPlanStages(explainAllPlansExecution, "window")
+                 .concat(getAggPlanStages(explainAllPlansExecution, "$_internalSetWindowFields"));
+    assert.gt(stages.length, 0, stages);
     assert(!stages[0]["usedDisk"], stages);
 }
 

jstests/aggregation/sources/setWindowFields/sum.js

@@ -6,7 +6,6 @@ import {
     seedWithTickerData,
     testAccumAgainstGroup
 } from "jstests/aggregation/extras/window_function_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db[jsTestName()];
 coll.drop();

jstests/aggregation/sources/shred_documents.js

@@ -1,5 +1,9 @@
 /**
  * Test $_internalShredDocuments parsing and make sure it doesn't modify documents.
+ * this test assumes {$meta: "indexKey"} will not be missing.
+ * @tags: [
+ *     do_not_wrap_aggregations_in_facets
+ * ]
  */
 "use strict";
 
@@ -7,7 +11,8 @@ import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
 
 const coll = db[jsTestName()];
 coll.insertMany(
-    [{a: 1, obj: {a: 1}, arr: [{a: 1}]}, {a: 1, obj: {a: 1}, arr: [{a: 1}]}, {}, {a: 1}]);
+    [{a: 1, obj: {a: 1}, arr: [{a: 1}]}, {a: 2, obj: {a: 1}, arr: [{a: 1}]}, {}, {a: 3}]);
+coll.createIndex({a: 1});
 assert.commandFailedWithCode(assert.throws(() => coll.aggregate({$_internalShredDocuments: 1})),
                                           7997500);
 assert.commandFailedWithCode(
@@ -40,3 +45,17 @@ assertNoop(matchExcludeGroup, 0);
 assertNoop(matchExcludeGroup, 1);
 assertNoop(matchExcludeGroup, 2);
 assertNoop(matchExcludeGroup, 3);
+
+assertNoop([{$match: {a: 1}}, {$addFields: {key: {$meta: "indexKey"}}}], 1);
+
+// The shred() function is also used by set windowFields so lets test that too.
+const res = coll.aggregate([
+                    {$match: {a: 1}},
+                    {$addFields: {key: {$meta: "indexKey"}}},
+                    {$setWindowFields: {sortBy: {a: 1}, output: {w: {$rank: {}}}}},
+                    {$limit: 1}
+                ])
+                .toArray();
+assert.eq(1, res.length);
+assert(res[0].hasOwnProperty("key"));
+assert.eq({a: 1}, res[0]["key"]);

jstests/aggregation/spill_to_disk.js

@@ -18,7 +18,7 @@ import "jstests/libs/sbe_assert_error_override.js";
 import {arrayEq, assertArrayEq} from "jstests/aggregation/extras/utils.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
 import {getSbePlanStages} from "jstests/libs/sbe_explain_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.spill_to_disk;
 coll.drop();
@@ -53,7 +53,7 @@ const sharded = FixtureHelpers.isSharded(coll);
 
 const memoryLimitMB = sharded ? 200 : 100;
 
-const isSbeEnabled = checkSBEEnabled(db);
+const isSbeGroupLookupPushdownEnabled = checkSbeRestrictedOrFullyEnabled(db);
 
 const bigStr = Array(1024 * 1024 + 1).toString();  // 1MB of ','
 for (let i = 0; i < memoryLimitMB + 1; i++)
@@ -76,7 +76,7 @@ function test({pipeline, expectedCodes, canSpillToDisk}) {
         assert.eq(new DBCommandCursor(coll.getDB(), res).itcount(),
                   coll.count());  // all tests output one doc per input doc
 
-        if (isSbeEnabled) {
+        if (isSbeGroupLookupPushdownEnabled) {
             const explain = db.runCommand({
                 explain:
                     {aggregate: coll.getName(), pipeline: pipeline, cursor: {}, allowDiskUse: true}
@@ -270,7 +270,7 @@ function testAccumulator({accumulator, sortInputBy, expectedOutput, ignoreArrayO
         assert.eq(results, expectedOutput);
     }
 
-    if (isSbeEnabled) {
+    if (isSbeGroupLookupPushdownEnabled) {
         const explain = coll.explain("executionStats").aggregate(pipeline);
         const groupStages = getSbePlanStages(explain, "group");
         assert.eq(groupStages.length, 1, groupStages);
@@ -444,7 +444,7 @@ function runTest_MultipleLocalForeignRecords({
     const results = localColl.aggregate(pipeline, {allowDiskUse: true}).toArray();
     const explain = localColl.explain('executionStats').aggregate(pipeline, {allowDiskUse: true});
     // If sharding is enabled, '$lookup' is not pushed down to SBE.
-    if (isSbeEnabled && !sharded) {
+    if (isSbeGroupLookupPushdownEnabled && !sharded) {
         const hLookups = getSbePlanStages(explain, 'hash_lookup');
         assert.eq(hLookups.length, 1, explain);
         const hLookup = hLookups[0];

jstests/auth/lib/commands_lib.js

@@ -463,6 +463,9 @@ export const authCommandsLib = {
           testname: 'transitionFromDedicatedConfigServer',
           command: {transitionFromDedicatedConfigServer: 1},
           skipUnlessSharded: true,
+          skipTest: (conn) => {
+            return !TestData.setParameters.featureFlagTransitionToCatalogShard;
+          },
           testcases: [
             {
               runOnDb: adminDbName,
@@ -477,6 +480,9 @@ export const authCommandsLib = {
           testname: "_configsvrTransitionFromDedicatedConfigServer",
           command: {_configsvrTransitionFromDedicatedConfigServer: 1},
           skipSharded: true,
+          skipTest: (conn) => {
+            return !TestData.setParameters.featureFlagTransitionToCatalogShard;
+          },
           testcases: [
               {
                 runOnDb: adminDbName,
@@ -492,6 +498,9 @@ export const authCommandsLib = {
           testname: "transitionToDedicatedConfigServer",
           command: { transitionToDedicatedConfigServer: 1 },
           skipUnlessSharded: true,
+          skipTest: (conn) => {
+            return !TestData.setParameters.featureFlagTransitionToCatalogShard;
+          },
           testcases: [
             {
               runOnDb: adminDbName,
@@ -507,6 +516,9 @@ export const authCommandsLib = {
           testname: "_configsvrTransitionToDedicatedConfigServer",
           command: {_configsvrTransitionToDedicatedConfigServer: 1},
           skipSharded: true,
+          skipTest: (conn) => {
+            return !TestData.setParameters.featureFlagTransitionToCatalogShard;
+          },
           testcases: [
               {
                 runOnDb: adminDbName,

jstests/auth/sbe_plan_cache_user_roles.js

@@ -6,13 +6,13 @@
  *   assumes_standalone_mongod,
  * ]
  */
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const mongod = MongoRunner.runMongod();
 const dbName = "test";
 const db = mongod.getDB(dbName);
 
-const sbeEnabled = checkSBEEnabled(db);
+const sbeEnabled = checkSbeFullyEnabled(db);
 
 // Create two users, each with different roles.
 assert.commandWorked(

jstests/change_streams/shard_collection_event.js

@@ -1,7 +1,6 @@
 /**
  * Test that change streams returns shardCollection events with the options specified on the
  * original user command.
- * TODO SERVER-81138 remove multiversion_incompatible and fix comparison with 7.0 binaries
  *  @tags: [
  *    requires_fcv_60,
  *    requires_sharding,
@@ -9,7 +8,6 @@
  *    change_stream_does_not_expect_txns,
  *    assumes_unsharded_collection,
  *    assumes_read_preference_unchanged,
- *    multiversion_incompatible,
  * ]
  */
 
@@ -42,6 +40,19 @@ function getCollectionUuid(coll) {
     return collInfo.info.uuid;
 }
 
+function assertChangeStreamEventEqMultiversionCompatible(actualEvent, expectedEvent) {
+    // SERVER-83104: Remove 'numInitialChunks' check once 8.0 becomes last LTS.
+    if ('numInitialChunks' in actualEvent.operationDescription) {
+        delete actualEvent.operationDescription.numInitialChunks;
+    }
+
+    // SERVER-83104: Remove 'capped' check once 8.0 becomes last LTS.
+    if (!('capped' in actualEvent.operationDescription)) {
+        delete expectedEvent.operationDescription.capped;
+    }
+    return assertChangeStreamEventEq(actualEvent, expectedEvent);
+}
+
 function assertNextChangeEvent(cursor, expectedEvent) {
     let events = test.getNextChanges(cursor, 1);
     while (events.length > 0) {
@@ -60,7 +71,7 @@ function assertNextChangeEvent(cursor, expectedEvent) {
     assert(event.wallTime instanceof Date);
     delete event.wallTime;
     expectedEvent.collectionUUID = getCollectionUuid(collName);
-    assertChangeStreamEventEq(event, expectedEvent);
+    assertChangeStreamEventEqMultiversionCompatible(event, expectedEvent);
 }
 
 function runTest(startChangeStream) {
@@ -92,7 +103,7 @@ function runTest(startChangeStream) {
         }
 
         const shardEvent = events[0];
-        assertChangeStreamEventEq(shardEvent, expectedOutput);
+        assertChangeStreamEventEqMultiversionCompatible(shardEvent, expectedOutput);
 
         // Insert a document before starting the next change stream so that we can validate the
         // resuming behavior.
@@ -181,16 +192,7 @@ function runTest(startChangeStream) {
                 "unique": false,
                 "presplitHashedZones": false,
                 "capped": false,
-                "collation": {
-                    "locale": "simple",
-                    "caseLevel": false,
-                    "caseFirst": "off",
-                    "strength": 3,
-                    "numericOrdering": false,
-                    "alternate": "non-ignorable",
-                    "maxVariable": "punct",
-                    "normalization": false
-                }
+                "collation": {"locale": "simple"}
             }
         })
 }

jstests/concurrency/fsm_workloads/batched_multi_deletes_with_write_conflicts.js

@@ -10,7 +10,7 @@
  *  requires_fcv_61,
  * ]
  */
-import {getPlanStage, getPlanStages} from "jstests/libs/analyze_plan.js";
+import {checkNWouldDelete, getPlanStage, getPlanStages} from "jstests/libs/analyze_plan.js";
 
 export const $config = (function() {
     // 'data' is passed (copied) to each of the worker threads.
@@ -43,10 +43,18 @@ export const $config = (function() {
             if (clusterTopology === '') {
                 assert(getPlanStage(expl, "BATCHED_DELETE"), tojson(expl));
             } else {
+                // Normally, we target both shards; however, in rare cases when the balancer is
+                // running, we may only target a single shard if during the last refresh, a shard no
+                // longer owns a chunk it used to, which makes targeting more specific.
                 const shardNames = Object.keys(clusterTopology.shards);
                 const stages = getPlanStages(expl, "BATCHED_DELETE");
-                assert.eq(stages.length, shardNames.length, tojson(expl));
+                assert.gte(stages.length, 1, tojson(expl));
+                assert.lte(stages.length, shardNames.length, tojson(expl));
             }
+
+            // Verify we wouldn't delete anything, since no ObjectId() would match the
+            // predicate {$gte: 0}.
+            checkNWouldDelete(expl, 0);
             assert.commandWorked(db[collName].remove({}));
         }
 

jstests/concurrency/fsm_workloads/collection_defragmentation.js

@@ -119,11 +119,8 @@ export const $config = (function() {
 
             // Choose a random starting point to look for mergeable chunks to make it less likely
             // that each thread tries to move the same chunk.
-            let index = Random.randInt(chunks.length);
-            for (let i = 0; i < chunks.length; i++) {
-                if (index === chunks.length - 1) {
-                    index = 0;
-                }
+            let index = Random.randInt(chunks.length - 1);
+            for (let i = 0; i < chunks.length - 1; i++) {
                 if (chunks[index].shard === chunks[index + 1].shard &&
                     defragmentationUtil.getZoneForRange(connCache.mongos[0],
                                                         randomColl.getFullName(),
@@ -143,6 +140,10 @@ export const $config = (function() {
                     }
                     return;
                 }
+                index++;
+                if (index >= chunks.length - 1) {
+                    index = 0;
+                }
             }
         },
 

jstests/concurrency/fsm_workloads/random_moveChunk_timeseries_insert_many.js

@@ -0,0 +1,152 @@
+/**
+ * Tests insertMany into a time-series collection during a chunk migration. This test is not
+ * checking results but is meant to run in sanitizers to ensure the exception handling in the
+ * time-series insert many code is correct.
+ * @tags: [
+ *  requires_sharding,
+ *  assumes_balancer_off,
+ *  requires_non_retryable_writes,
+ *  does_not_support_transactions,
+ * ]
+ */
+import {extendWorkload} from "jstests/concurrency/fsm_libs/extend_workload.js";
+import {ChunkHelper} from "jstests/concurrency/fsm_workload_helpers/chunks.js";
+import {
+    $config as $baseConfig
+} from 'jstests/concurrency/fsm_workloads/sharded_moveChunk_partitioned.js';
+import {findChunksUtil} from "jstests/sharding/libs/find_chunks_util.js";
+
+export const $config = extendWorkload($baseConfig, function($config, $super) {
+    // A random non-round start value was chosen so that we can verify the rounding behavior that
+    // occurs while routing on mongos.
+    $config.data.startTime = 1021;
+
+    // One minute.
+    $config.data.increment = 1000 * 60;
+
+    // This should generate documents for a span of one month.
+    $config.data.numInitialDocs = 60 * 24 * 30;
+
+    $config.data.bucketPrefix = "system.buckets.";
+
+    $config.data.metaField = 'm';
+    $config.data.timeField = 't';
+
+    $config.threadCount = 10;
+    $config.iterations = 40;
+    $config.startState = "init";
+
+    /**
+     * Perform insertMany with ordered:false that may target multiple buckets across multiple chunks
+     */
+    $config.states.insert = function insert(db, collName, connCache) {
+        var docs = [];
+        for (let i = 0; i < 10; i++) {
+            // Generate a random timestamp between 'startTime' and largest timestamp we inserted.
+            const timer =
+                this.startTime + Math.floor(Random.rand() * this.numInitialDocs * this.increment);
+            const doc = {
+                _id: new ObjectId(),
+                [this.metaField]: 0,
+                [this.timeField]: new Date(timer),
+            };
+            docs.push(doc);
+        }
+
+        // Perform unordered insertMany. When this is done concurrent with chunk migrations we may
+        // get an exception in mongod when only a subset of documents have been (partially)
+        // processed. This will be retried by mongos and no error is observed for the user. This
+        // test is meant to be run with sanitizers to ensure correctness.
+        assert.commandWorked(db[collName].insertMany(docs, {ordered: false}));
+    };
+
+    /**
+     * Moves a random chunk in the target collection.
+     */
+    $config.states.moveChunk = function moveChunk(db, collName, connCache) {
+        const configDB = db.getSiblingDB('config');
+        const ns = db[this.bucketPrefix + collName].getFullName();
+        const chunks = findChunksUtil.findChunksByNs(configDB, ns).toArray();
+        const chunkToMove = chunks[this.tid];
+        const fromShard = chunkToMove.shard;
+
+        // Choose a random shard to move the chunk to.
+        const shardNames = Object.keys(connCache.shards);
+        const destinationShards = shardNames.filter(function(shard) {
+            if (shard !== fromShard) {
+                return shard;
+            }
+        });
+        const toShard = destinationShards[Random.randInt(destinationShards.length)];
+        const waitForDelete = false;
+        ChunkHelper.moveChunk(db,
+                              this.bucketPrefix + collName,
+                              [chunkToMove.min, chunkToMove.max],
+                              toShard,
+                              waitForDelete);
+    };
+
+    $config.states.init = function init(db, collName, connCache) {};
+
+    $config.transitions = {
+        init: {insert: 1},
+        insert: {insert: 7, moveChunk: 1},
+        moveChunk: {insert: 1, moveChunk: 0}
+    };
+
+    $config.setup = function setup(db, collName, cluster) {
+        db[collName].drop();
+
+        assert.commandWorked(db.createCollection(
+            collName, {timeseries: {metaField: this.metaField, timeField: this.timeField}}));
+        cluster.shardCollection(db[collName], {t: 1}, false);
+
+        const bulk = db[collName].initializeUnorderedBulkOp();
+
+        let currentTimeStamp = this.startTime;
+        for (let i = 0; i < this.numInitialDocs; ++i) {
+            currentTimeStamp += this.increment;
+
+            const metaVal = 0;
+            const doc = {
+                _id: new ObjectId(),
+                [this.metaField]: metaVal,
+                [this.timeField]: new Date(currentTimeStamp),
+                // use an invalid tid to not count these documents when we validate at teardown
+                tid: -1,
+            };
+            bulk.insert(doc);
+        }
+
+        let res = bulk.execute();
+        assert.commandWorked(res);
+        assert.eq(this.numInitialDocs, res.nInserted);
+
+        // Pick 'this.threadCount - 1' split points so that we have 'this.threadCount' chunks.
+        const chunkRange = (currentTimeStamp - this.startTime) / this.threadCount;
+        currentTimeStamp = this.startTime;
+        for (let i = 0; i < (this.threadCount - 1); ++i) {
+            currentTimeStamp += chunkRange;
+            assert.commandWorked(ChunkHelper.splitChunkAt(
+                db, this.bucketPrefix + collName, {'control.min.t': new Date(currentTimeStamp)}));
+        }
+
+        // Create an extra chunk on each shard to make sure multi:true operations return correct
+        // metrics in write results.
+        const destinationShards = Object.keys(cluster.getSerializedCluster().shards);
+        for (const destinationShard of destinationShards) {
+            currentTimeStamp += chunkRange;
+            assert.commandWorked(ChunkHelper.splitChunkAt(
+                db, this.bucketPrefix + collName, {'control.min.t': new Date(currentTimeStamp)}));
+
+            ChunkHelper.moveChunk(
+                db,
+                this.bucketPrefix + collName,
+                [{'control.min.t': new Date(currentTimeStamp)}, {'control.min.t': MaxKey}],
+                destinationShard,
+                /*waitForDelete=*/ false);
+        }
+    };
+
+    return $config;
+});

jstests/concurrency/fsm_workloads/write_without_shard_key_with_resharding.js

@@ -18,6 +18,7 @@ import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
 
 export const $config = extendWorkload($baseConfig, function($config, $super) {
     $config.startState = "init";
+    $config.data.partitionSize = 100;
 
     const customShardKeyFieldName = "customShardKey";
 

jstests/core/administrative/validate_db_metadata_command.js

@@ -5,6 +5,9 @@
  *   # The test runs commands that are not allowed with security token: validateDBMetadata.
  *   not_allowed_with_security_token,
  *   no_selinux,
+ * # TODO SERVER-82166 remove "balancer off" tag once PM-2077 is over: migration can cause an index
+ * # to be created before a dropIndex commits causing the metadata validation to fail
+ * assumes_balancer_off,
  * ]
  */
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";

jstests/core/columnstore/column_scan_skip_row_store_projection.js

@@ -24,14 +24,6 @@
 import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
 import {getSingleNodeExplain} from "jstests/libs/analyze_plan.js";
 import {setUpServerForColumnStoreIndexTest} from "jstests/libs/columnstore_util.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
-const columnstoreEnabled =
-    checkSBEEnabled(db, ["featureFlagColumnstoreIndexes"], true /* checkAllNodes */);
-if (!columnstoreEnabled) {
-    jsTestLog("Skipping columnstore index test since the feature flag is not enabled.");
-    quit();
-}
 
 if (!setUpServerForColumnStoreIndexTest(db)) {
     quit();

jstests/core/columnstore/column_store_index_compression.js

@@ -21,14 +21,6 @@ import {setUpServerForColumnStoreIndexTest} from "jstests/libs/columnstore_util.
 import {DiscoverTopology} from "jstests/libs/discover_topology.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
 import {IndexCatalogHelpers} from "jstests/libs/index_catalog_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
-const columnstoreEnabled =
-    checkSBEEnabled(db, ["featureFlagColumnstoreIndexes"], true /* checkAllNodes */);
-if (!columnstoreEnabled) {
-    jsTestLog("Skipping columnstore index test since the feature flag is not enabled.");
-    quit();
-}
 
 if (!setUpServerForColumnStoreIndexTest(db)) {
     quit();

jstests/core/columnstore/columnstore_eligibility.js

@@ -12,15 +12,15 @@
  *   # server parameters are stored in-memory only so are not transferred onto the recipient.
  *   tenant_migration_incompatible,
  *   not_allowed_with_security_token,
- *   # Logic for when a COLUMN_SCAN plan is generated changed slightly as part of enabling more
- *   # queries in SBE in the 7.0 release.
- *   requires_fcv_70,
+ *   # Columnstore indexes never shipped to the customers in older versions, so there is no need to
+ *   # test multiversion suites.
+ *   requires_fcv_72,
  * ]
  */
 import {aggPlanHasStage, planHasStage} from "jstests/libs/analyze_plan.js";
 import {setUpServerForColumnStoreIndexTest} from "jstests/libs/columnstore_util.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";  // TODO SERVER-80226: Remove this import
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 if (!setUpServerForColumnStoreIndexTest(db)) {
     quit();
@@ -170,8 +170,7 @@ explain = coll.explain().aggregate([
     {$group: {_id: "$comments.author", total_views: {$sum: "$comments.views"}}}
 ]);
 
-// TODO SERVER-80226: Remove 'featureFlagSbeFull' used by $unwind Pushdown feature.
-if (checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+if (checkSbeFullyEnabled(db)) {
     // The entire pipeline should be pushed down to SBE.
     assert(planHasStage(db, explain, "GROUP"), explain);
     assert(planHasStage(db, explain, "UNWIND"), explain);

jstests/core/command_let_variables.js

@@ -7,7 +7,7 @@
 //
 import {getPlanStage, planHasStage} from "jstests/libs/analyze_plan.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";  // TODO SERVER-80226: Remove this import
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const testDB = db.getSiblingDB("command_let_variables");
 const coll = testDB.command_let_variables;
@@ -91,10 +91,9 @@ let explain = assert.commandWorked(testDB.runCommand({
     verbosity: "executionStats"
 }));
 if (!isMongos) {
-    // TODO SERVER-80226: Remove 'featureFlagSbeFull' used by $unwind Pushdown feature.
-    if (checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+    if (checkSbeFullyEnabled(testDB)) {
         // $unwind should be pushed down to SBE.
-        assert(planHasStage(db, explain, "UNWIND"), explain);
+        assert(planHasStage(testDB, explain, "UNWIND"), explain);
     } else {
         assert(explain.hasOwnProperty("stages"), explain);
         assert.neq(explain.stages.length, 0, explain);

jstests/core/find_with_resume_after_param.js

@@ -10,7 +10,6 @@
 
 import {assertDropCollection} from "jstests/libs/collection_drop_recreate.js";
 import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
 
 const clustered = db.clusteredColl;
 const nonClustered = db.normalColl;
@@ -28,6 +27,23 @@ const docs = [{_id: 1, a: 1}, {_id: 2, a: 2}, {_id: 3, a: 3}];
 assert.commandWorked(clustered.insertMany(docs));
 assert.commandWorked(nonClustered.insertMany(docs));
 
+// When given the recordId of the last record in the collection, we should receive a null
+// resumeToken.
+const res = assert.commandWorked(db.runCommand({
+    find: nonClusteredName,
+    filter: {},
+    $_requestResumeToken: true,
+    $_resumeAfter: {'$recordId': NumberLong(3)},
+    hint: {$natural: 1}
+}));
+assert(res.hasOwnProperty('cursor'), res);
+const cursor = res['cursor'];
+assert(cursor.hasOwnProperty('postBatchResumeToken'), res);
+
+// Note that we don't perform an exact equality on 'postBatchResumeToken' because depending on
+// the configuration, it may contain additional fields (such as '$initialSyncId').
+assert.eq(cursor.postBatchResumeToken.$recordId, null, res);
+
 function validateFailedResumeAfterInFind({collName, resumeAfterSpec, errorCode, explainFail}) {
     const spec = {
         find: collName,
@@ -94,23 +110,12 @@ function testResumeAfter(validateFunction) {
         errorCode: ErrorCodes.KeyNotFound
     });
 
-    if (checkSBEEnabled(db)) {
-        // This case really means that 'forceClassicEngine' has not been set. It does not mean any
-        // SBE-specific feature flags are turned on.
-        validateFunction({
-            collName: nonClusteredName,
-            resumeAfterSpec: {'$recordId': null},
-            errorCode: ErrorCodes.KeyNotFound
-        });
-    } else {
-        assert.commandWorked(db.runCommand({
-            find: nonClusteredName,
-            filter: {},
-            $_requestResumeToken: true,
-            $_resumeAfter: {'$recordId': null},
-            hint: {$natural: 1}
-        }));
-    }
+    // Confirm $_resumeAfter token will fail with 'KeyNotFound' if given a null recordId.
+    validateFunction({
+        collName: nonClusteredName,
+        resumeAfterSpec: {'$recordId': null},
+        errorCode: ErrorCodes.KeyNotFound
+    });
 
     // Confirm $_resumeAfter will fail to parse if collection does not exist.
     validateFunction({

jstests/core/index/hidden_index.js

@@ -21,11 +21,8 @@ import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recr
 import {setUpServerForColumnStoreIndexTest} from "jstests/libs/columnstore_util.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
 import {IndexCatalogHelpers} from "jstests/libs/index_catalog_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
 
-const columnstoreEnabled =
-    checkSBEEnabled(db, ["featureFlagColumnstoreIndexes"], true /* checkAllNodes */) &&
-    setUpServerForColumnStoreIndexTest(db);
+const columnstoreEnabled = setUpServerForColumnStoreIndexTest(db);
 
 const collName = "hidden_index";
 let coll = assertDropAndRecreateCollection(db, collName);

jstests/core/index/index_filter_commands.js

@@ -54,7 +54,7 @@ import {
     ClusteredCollectionUtil
 } from "jstests/libs/clustered_collections/clustered_collection_util.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
 // Flag indicating if index filter commands are running through the query settings interface.
 var isIndexFiltersToQuerySettings = TestData.isIndexFiltersToQuerySettings || false;
@@ -356,7 +356,7 @@ assert.commandFailed(
 filters = getFilters();
 assert.eq(0, filters.length, tojson(filters));
 
-if (checkSBEEnabled(db)) {
+if (checkSbeRestrictedOrFullyEnabled(db)) {
     //
     // Test that planCacheSetFilter doesn't apply to the inner side of a $lookup.
     //

jstests/core/index/index_filter_commands_invalidate_plan_cache_entries.js

@@ -21,11 +21,11 @@
  * ]
  */
 import {getPlanCacheKeyFromShape} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 // For testing convenience this variable is made an integer "1" if SBE is fully enabled, because the
 // expected amount of plan cache entries differs between the SBE plan cache and the classic one.
-const isSbeEnabled = checkSBEEnabled(db) ? 1 : 0;
+const isSbeEnabled = checkSbeFullyEnabled(db) ? 1 : 0;
 
 const collName = "index_filter_commands_invalidate_plan_cache_entries";
 const coll = db[collName];

jstests/core/index/index_stats.js

@@ -23,7 +23,7 @@
 // ]
 
 import {getAggPlanStage, getPlanStages} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
 var colName = "jstests_index_stats";
 var col = db[colName];
@@ -239,7 +239,7 @@ assert.eq(2,
                  ])
                   .itcount());
 assert.eq(1, getUsageCount("_id_", col), "Expected aggregation to use _id index");
-if (!checkSBEEnabled(db)) {
+if (!checkSbeRestrictedOrFullyEnabled(db)) {
     assert.eq(2,
               getUsageCount("_id_", foreignCollection),
               "Expected each lookup to be tracked as an index use");
@@ -277,7 +277,7 @@ const pipeline = [
 ];
 assert.eq(2, col.aggregate(pipeline).itcount());
 assert.eq(1, getUsageCount("_id_", col), "Expected aggregation to use _id index");
-if (!checkSBEEnabled(db)) {
+if (!checkSbeRestrictedOrFullyEnabled(db)) {
     assert.eq(2,
               getUsageCount("_id_", foreignCollection),
               "Expected each lookup to be tracked as an index use");

jstests/core/index/indexj.js

@@ -12,7 +12,7 @@
 //   requires_fcv_63,
 // ]
 
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const t = db[jsTestName()];
 t.drop();
@@ -57,7 +57,7 @@ assert.commandWorked(t.insert({a: 2, b: 2}));
 // SBE or the classic engine. This is because the classic engine will use a multi-interval index
 // scan whereas SBE will decompose the intervals into a set of single-interval bounds and will end
 // up examining 0 keys.
-const isSBEEnabled = checkSBEEnabled(db);
+const isSBEEnabled = checkSbeFullyEnabled(db);
 let expectedKeys = isSBEEnabled ? 0 : 3;
 let errMsg = function(actualNumKeys) {
     return "Chosen plan examined " + actualNumKeys + " keys";

jstests/core/index/wildcard/wildcard_index_cached_plans.js

@@ -28,9 +28,8 @@ import {
     getWinningPlan,
 } from "jstests/libs/analyze_plan.js";
 import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
-import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.wildcard_cached_plans;
 
@@ -51,7 +50,7 @@ function getCacheEntryForQuery(query) {
     return null;
 }
 
-const isSbeEnabled = checkSBEEnabled(db);
+const isSbeEnabled = checkSbeFullyEnabled(db);
 
 for (const indexSpec of wildcardIndexes) {
     coll.drop();

jstests/core/json_schema/json_schema.js

@@ -8,9 +8,9 @@
  * Tests for JSON Schema document validation.
  */
 import {assertSchemaMatch} from "jstests/libs/assert_schema_match.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
-const isSBEEnabled = checkSBEEnabled(db);
+const isSBEEnabled = checkSbeFullyEnabled(db);
 
 let coll = db.jstests_json_schema;
 coll.drop();

jstests/core/notablescan.js

@@ -16,48 +16,69 @@
 //   tenant_migration_incompatible,
 // ]
 
-let t = db.test_notablescan;
-t.drop();
+import {isClusteredIxscan} from "jstests/libs/analyze_plan.js";
+import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
 
-try {
-    assert.commandWorked(db._adminCommand({setParameter: 1, notablescan: true}));
-    // commented lines are SERVER-2222
-    if (0) {  // SERVER-2222
-        assert.throws(function() {
-            t.find({a: 1}).toArray();
-        });
-    }
-    t.save({a: 1});
-    assert.throws(function() {
-        t.count({a: 1});
-    });
+function checkError(err) {
+    assert.includes(err.toString(), "'notablescan'");
+}
+
+const colName = jsTestName();
+let coll = db.getCollection(colName);
+coll.drop();
+
+assert.commandWorked(db.adminCommand({setParameter: 1, notablescan: true}));
+
+{
     if (0) {
+        // TODO: SERVER-2222 This should actually throw an error as it performs a collection
+        // scan.
         assert.throws(function() {
-            t.find({}).toArray();
+            coll.find({a: 1}).toArray();
         });
     }
-    assert.eq(1, t.find({}).itcount());  // SERVER-274
 
+    coll.insert({a: 1});
     let err = assert.throws(function() {
-        t.find({a: 1}).toArray();
+        coll.count({a: 1});
     });
-    assert.includes(err.toString(), "No indexed plans available, and running with 'notablescan'");
+    checkError(err);
+
+    // TODO: SERVER-2222 This should actually throw an error as it performs a collection scan.
+    assert.eq(1, coll.find({}).itcount());
 
     err = assert.throws(function() {
-        t.find({a: 1}).hint({$natural: 1}).toArray();
+        coll.find({a: 1}).toArray();
+    });
+    checkError(err);
+
+    err = assert.throws(function() {
+        coll.find({a: 1}).hint({$natural: 1}).toArray();
     });
     assert.includes(err.toString(), "$natural");
-    assert.includes(err.toString(), "notablescan");
+    checkError(err);
 
-    t.createIndex({a: 1});
-    assert.eq(0, t.find({a: 1, b: 1}).itcount());
-    assert.eq(1, t.find({a: 1, b: null}).itcount());
-
-    // SERVER-4327
-    assert.eq(0, t.find({a: {$in: []}}).itcount());
-    assert.eq(0, t.find({a: {$in: []}, b: 0}).itcount());
-} finally {
-    // We assume notablescan was false before this test started and restore that
-    // expected value.
-    assert.commandWorked(db._adminCommand({setParameter: 1, notablescan: false}));
+    coll.createIndex({a: 1});
+    assert.eq(0, coll.find({a: 1, b: 1}).itcount());
+    assert.eq(1, coll.find({a: 1, b: null}).itcount());
 }
+
+{  // Run the testcase with a clustered index.
+    assertDropAndRecreateCollection(db, colName, {clusteredIndex: {key: {_id: 1}, unique: true}})
+    coll = db.getCollection(colName);
+    assert.commandWorked(coll.insert({_id: 22}));
+    assert.eq(1, coll.find({_id: 22}).itcount());
+    let plan = coll.find({_id: 22}).explain();
+    // Make sure the plan has a clustered index scan.
+    assert(isClusteredIxscan(db, plan));
+
+    // Make sure the same works with an aggregate.
+    assert.eq(1, coll.aggregate([{$match: {_id: 22}}]).itcount());
+    plan = coll.explain().aggregate([{$match: {_id: 22}}]);
+    // Make sure the plan has a clustered index scan.
+    assert(isClusteredIxscan(db, plan));
+    assert.commandWorked(
+        db.runCommand({aggregate: colName, pipeline: [{$match: {_id: 22}}], cursor: {}}));
+}
+// Set it back to the original value.
+assert.commandWorked(db.adminCommand({setParameter: 1, notablescan: false}));

jstests/core/query/boolean_expression_simplification.js

@@ -9,6 +9,14 @@
 
 import {getPlanStages, getWinningPlanFromExplain} from "jstests/libs/analyze_plan.js";
 
+const parameterName = "internalQueryEnableBooleanExpressionsSimplifier";
+const isSImplifierEnabled =
+    assert.commandWorked(db.adminCommand({getParameter: 1, [parameterName]: 1}))[parameterName];
+if (!isSImplifierEnabled) {
+    jsTest.log("Skipping the Boolean simplier tests, since the simplifier is disabled...");
+    quit();
+}
+
 /**
  * Checks possible representations of an empty filter in query plans, which can be empty object '{}'
  * or missing value.

jstests/core/query/explain/explain_delete.js

@@ -10,6 +10,7 @@
  *  incompatible_with_preimages_by_default,
  *  ]
  */
+import {checkNWouldDelete} from "jstests/libs/analyze_plan.js";
 
 var collName = "jstests_explain_delete";
 var t = db[collName];
@@ -17,33 +18,6 @@ t.drop();
 
 var explain;
 
-/**
- * Verify that the explain command output 'explain' shows a BATCHED_DELETE stage with an
- * nWouldDelete value equal to 'nWouldDelete'.
- */
-function checkNWouldDelete(explain, nWouldDelete) {
-    assert.commandWorked(explain);
-    assert("executionStats" in explain);
-    var executionStats = explain.executionStats;
-    assert("executionStages" in executionStats);
-
-    // If passed through mongos, then BATCHED_DELETE stage(s) should be below the SHARD_WRITE
-    // mongos stage.  Otherwise the BATCHED_DELETE stage is the root stage.
-    var execStages = executionStats.executionStages;
-    if ("SHARD_WRITE" === execStages.stage) {
-        let totalToBeDeletedAcrossAllShards = 0;
-        execStages.shards.forEach(function(shardExplain) {
-            const rootStageName = shardExplain.executionStages.stage;
-            assert(rootStageName === "BATCHED_DELETE", tojson(execStages));
-            totalToBeDeletedAcrossAllShards += shardExplain.executionStages.nWouldDelete;
-        });
-        assert.eq(totalToBeDeletedAcrossAllShards, nWouldDelete, explain);
-    } else {
-        assert(execStages.stage === "BATCHED_DELETE", explain);
-        assert.eq(execStages.nWouldDelete, nWouldDelete, explain);
-    }
-}
-
 // Explain delete against an empty collection.
 assert.commandWorked(db.createCollection(t.getName()));
 explain = db.runCommand({explain: {delete: collName, deletes: [{q: {a: 1}, limit: 0}]}});
@@ -70,4 +44,4 @@ assert.eq(10, t.count());
 // If we run the same thing without the explain, then all 10 docs should be deleted.
 var deleteResult = db.runCommand({delete: collName, deletes: [{q: {a: 1}, limit: 0}]});
 assert.commandWorked(deleteResult);
-assert.eq(0, t.count());
+assert.eq(0, t.count());

jstests/core/query/explode_for_sort_plan_cache.js

@@ -27,9 +27,9 @@ import {
     getPlanStages,
     getWinningPlan
 } from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
-const isSBEEnabled = checkSBEEnabled(db);
+const isSBEEnabled = checkSbeFullyEnabled(db);
 const coll = db.explode_for_sort_plan_cache;
 coll.drop();
 
@@ -68,12 +68,16 @@ function assertIsNotExplodeForSort(query) {
     assert.eq(sortMerges.length, 0, explain);
 }
 
-function assertExplodeForSortCacheParameterizedCorrectly(
-    {query, queryCount, newQuery, newQueryCount, reuseEntry}) {
+function assertQueryParameterizedCorrectly(
+    {query, queryCount, newQuery, newQueryCount, reuseEntry, isExplode = true}) {
     // Clear plan cache to have a fresh test case.
     coll.getPlanCache().clear();
 
-    assertIsExplodeForSort(query);
+    if (isExplode) {
+        assertIsExplodeForSort(query);
+    } else {
+        assertIsNotExplodeForSort(query);
+    }
 
     // Run the query for the first time to create an inactive plan cache entry.
     assert.eq(queryCount, coll.find(query).sort(sortSpec).itcount());
@@ -131,7 +135,7 @@ for (let a = 1; a <= 3; a++) {
 assertIsNotExplodeForSort({a: {$gte: 1, $lte: 1}, b: {$in: [1, 2]}});
 
 // Changing the $eq predicate value should reuse the plan cache and gives correct results.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {a: {$eq: 1}, b: {$in: [1, 2]}},
     queryCount: 3,
     newQuery: {a: {$eq: 2}, b: {$in: [1, 2]}},
@@ -141,7 +145,7 @@ assertExplodeForSortCacheParameterizedCorrectly({
 
 // Changing the $expr-$eq predicate value should not reuse the SBE plan cache because agg expression
 // is not parameterized.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {$and: [{$expr: {$eq: ["$a", 1]}}, {b: {$in: [1, 2]}}]},
     queryCount: 3,
     newQuery: {$and: [{$expr: {$eq: ["$a", 2]}}, {b: {$in: [1, 2]}}]},
@@ -152,7 +156,7 @@ assertExplodeForSortCacheParameterizedCorrectly({
 });
 
 // Rewriting the $in predicate with $or should reuse the plan cache and gives correct results.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {a: {$eq: 1}, b: {$in: [1, 2]}},
     queryCount: 3,
     newQuery: {$and: [{a: {$eq: 2}}, {$or: [{b: {$eq: 1}}, {b: {$eq: 2}}]}]},
@@ -162,7 +166,7 @@ assertExplodeForSortCacheParameterizedCorrectly({
 
 // Changing the $or predicate value should reuse the plan cache and gives correct results, since $or
 // is rewritten as $in.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {$and: [{a: {$eq: 1}}, {$or: [{b: {$eq: 1}}, {b: {$eq: 2}}]}]},
     queryCount: 3,
     newQuery: {$and: [{a: {$eq: 1}}, {$or: [{b: {$eq: 1}}, {b: {$eq: 3}}]}]},
@@ -172,7 +176,7 @@ assertExplodeForSortCacheParameterizedCorrectly({
 
 // Changing the $in predicate values but not size should reuse the plan cache and gives correct
 // results.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {a: {$eq: 1}, b: {$in: [1, 2]}},
     queryCount: 3,
     newQuery: {a: {$eq: 1}, b: {$in: [1, 3]}},
@@ -182,7 +186,7 @@ assertExplodeForSortCacheParameterizedCorrectly({
 
 // Changing the $in predicate size should not reuse the SBE plan cache but still gives correct
 // results.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {a: {$eq: 1}, b: {$in: [1, 2]}},
     queryCount: 3,
     newQuery: {a: {$eq: 1}, b: {$in: [1, 2, 3]}},
@@ -193,14 +197,14 @@ assertExplodeForSortCacheParameterizedCorrectly({
 // Special values in the predicate will not be parameterized hence the SBE plan cache will not be
 // reused.
 for (let specialValue of [null, {x: 1}, [1]]) {
-    assertExplodeForSortCacheParameterizedCorrectly({
+    assertQueryParameterizedCorrectly({
         query: {a: {$eq: 1}, b: {$in: [1, 2]}},
         queryCount: 3,
         newQuery: {a: {$eq: specialValue}, b: {$in: [1, 2]}},
         newQueryCount: 0,
         reuseEntry: !isSBEEnabled,
     });
-    assertExplodeForSortCacheParameterizedCorrectly({
+    assertQueryParameterizedCorrectly({
         query: {a: {$eq: 1}, b: {$in: [1, 2]}},
         queryCount: 3,
         newQuery: {a: {$eq: 1}, b: {$in: [0, specialValue]}},
@@ -210,10 +214,51 @@ for (let specialValue of [null, {x: 1}, [1]]) {
 }
 
 // Regex in $in predicate will not reuse plan cache.
-assertExplodeForSortCacheParameterizedCorrectly({
+assertQueryParameterizedCorrectly({
     query: {a: {$eq: 1}, b: {$in: [1, 2]}},
     queryCount: 3,
     newQuery: {a: {$eq: 1}, b: {$in: [0, /a|regex/]}},
     newQueryCount: 0,
     reuseEntry: false,
 });
+
+const maxScansToExplode = assert.commandWorked(db.adminCommand(
+    {getParameter: 1, internalQueryMaxScansToExplode: 1}))["internalQueryMaxScansToExplode"];
+
+const maxExplodeIn = [];
+for (let i = 0; i < maxScansToExplode; ++i) {
+    maxExplodeIn.push(i);
+}
+const tooLargeToExplodeIn = maxExplodeIn.concat([maxScansToExplode]);
+const evenLargerIn = tooLargeToExplodeIn.concat([maxScansToExplode + 1]);
+
+// Test that an $in with as many elements as the "max scans to explode" parameter uses explode for
+// sort and is parameterized correctly.
+assertQueryParameterizedCorrectly({
+    query: {a: {$eq: 1}, b: {$in: maxExplodeIn}},
+    queryCount: 6,
+    newQuery: {a: {$eq: 2}, b: {$in: maxExplodeIn}},
+    newQueryCount: 12,
+    reuseEntry: true,
+});
+
+// Test that in SBE a query with explode for sort optimization will use different plan cache
+// entry than the same query with too many scans to explode.
+assertQueryParameterizedCorrectly({
+    query: {a: {$eq: 1}, b: {$in: maxExplodeIn}},
+    queryCount: 6,
+    newQuery: {a: {$eq: 1}, b: {$in: tooLargeToExplodeIn}},
+    newQueryCount: 6,
+    reuseEntry: !isSBEEnabled,
+});
+
+// Test that a query with more than "max scans to explode" will use the same plan cache
+// entry for any number of elements in $in.
+assertQueryParameterizedCorrectly({
+    query: {a: {$eq: 1}, b: {$in: tooLargeToExplodeIn}},
+    queryCount: 6,
+    newQuery: {a: {$eq: 1}, b: {$in: evenLargerIn}},
+    newQueryCount: 6,
+    reuseEntry: true,
+    isExplode: false
+});

jstests/core/query/expr/expr_index_use.js

@@ -5,7 +5,7 @@
 // ]
 
 import {getAggPlanStage, getPlanStage, hasRejectedPlans} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.expr_index_use;
 coll.drop();
@@ -50,7 +50,7 @@ assert.commandWorked(coll.insert({w: {z: 2}}));
 assert.commandWorked(coll.createIndex({w: 1}));
 assert.commandWorked(coll.createIndex({"w.z": 1}));
 
-const isSBEEnabled = checkSBEEnabled(db);
+const isSBEEnabled = checkSbeFullyEnabled(db);
 
 /**
  * Executes the expression 'expr' as both a find and an aggregate. Then confirms

jstests/core/query/idhack.js

@@ -5,7 +5,7 @@
 // ]
 // Include helpers for analyzing explain output.
 import {getWinningPlan, isIdhack} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const t = db.idhack;
 t.drop();
@@ -58,7 +58,7 @@ winningPlan = getWinningPlan(explain.queryPlanner);
 assert(!isIdhack(db, winningPlan), winningPlan);
 
 // Covered query returning _id field only can be handled by ID hack.
-const parentStage = checkSBEEnabled(db) ? "PROJECTION_COVERED" : "FETCH";
+const parentStage = checkSbeFullyEnabled(db) ? "PROJECTION_COVERED" : "FETCH";
 explain = t.find(query, {_id: 1}).explain();
 winningPlan = getWinningPlan(explain.queryPlanner);
 assert(isIdhack(db, winningPlan), winningPlan);

jstests/core/query/null_query_semantics.js

@@ -13,7 +13,6 @@
 //
 import {resultsEq} from "jstests/aggregation/extras/utils.js";
 import {setUpServerForColumnStoreIndexTest} from "jstests/libs/columnstore_util.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
 
 function extractAValues(results) {
     return results.map(function(res) {
@@ -787,9 +786,7 @@ const keyPatterns = [
 ];
 
 // Include Columnstore Index only if FF is enabled and collection is not clustered.
-const columnstoreEnabled =
-    checkSBEEnabled(db, ["featureFlagColumnstoreIndexes"], true /* checkAllNodes */);
-if (columnstoreEnabled && setUpServerForColumnStoreIndexTest(db)) {
+if (setUpServerForColumnStoreIndexTest(db)) {
     keyPatterns.push({keyPattern: {"$**": "columnstore"}});
 }
 

jstests/core/query/or_use_clustered_collection.js

@@ -18,11 +18,13 @@ import {
     getWinningPlan
 } from "jstests/libs/analyze_plan.js";
 import {assertDropCollection} from "jstests/libs/collection_drop_recreate.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeRestrictedOrFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.or_use_clustered_collection;
 assertDropCollection(db, coll.getName());
 
+const isSbeGroupEnabled = checkSbeRestrictedOrFullyEnabled(db);
+
 // Create a clustered collection and create indexes.
 assert.commandWorked(
     db.createCollection(coll.getName(), {clusteredIndex: {key: {_id: 1}, unique: true}}));
@@ -155,7 +157,7 @@ function validateQueryPlan({query, expectedStageCount, expectedDocIds, noFetchWi
         }
 
         // Classic engine doesn't have a GROUP stage like SBE for $group.
-        if (group && !checkSBEEnabled(db)) {
+        if (group && !isSbeGroupEnabled) {
             test.additionalStages["GROUP"] = 0;
         }
 

jstests/core/query/plan_cache/cached_plan_trial_does_not_discard_work.js

@@ -19,14 +19,9 @@
 //   tenant_migration_incompatible,
 //   # TODO SERVER-67607: Test plan cache with CQF enabled.
 //   cqf_experimental_incompatible,
+//   featureFlagSbeFull,
 // ]
 import {getLatestProfilerEntry} from "jstests/libs/profiler.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
-if (!checkSBEEnabled(db)) {
-    jsTestLog("Skipping test because SBE is disabled");
-    quit();
-}
 
 const testDb = db.getSiblingDB(jsTestName());
 assert.commandWorked(testDb.dropDatabase());

jstests/core/query/plan_cache/collation_plan_cache.js

@@ -18,7 +18,7 @@
 //   tenant_migration_incompatible,
 // ]
 import {getPlanCacheKeyFromExplain} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 var coll = db.collation_plan_cache;
 coll.drop();
@@ -54,7 +54,7 @@ assert.commandWorked(
 var shapes = coll.aggregate([{$planCacheStats: {}}]).toArray();
 assert.eq(1, shapes.length, 'unexpected cache size after running query');
 
-const isSbeEnabled = checkSBEEnabled(db);
+const isSbeEnabled = checkSbeFullyEnabled(db);
 if (!isSbeEnabled) {
     assert.eq(shapes[0].createdFromQuery.query, {a: 'foo', b: 5}, shapes);
     assert.eq(shapes[0].createdFromQuery.sort, {}, shapes);

jstests/core/query/plan_cache/match_or_duplicate_predicate_plan_cache.js

@@ -0,0 +1,50 @@
+/**
+ * Test a rooted $or query with duplicate predicates which reproduces SERVER-84338, a bug in the
+ * MatchExpression $or->$in rewrite which produced incorrect plans in the SBE plan cache.
+ *
+ * @tags: [
+ *   # The test runs commands that are not allowed with security token: planCacheClear.
+ *   not_allowed_with_signed_security_token,
+ *   # This test attempts to perform queries and introspect/manipulate the server's plan cache
+ *   # entries. The former operation may be routed to a secondary in the replica set, whereas the
+ *   # latter must be routed to the primary.
+ *   assumes_read_concern_unchanged,
+ *   assumes_read_preference_unchanged,
+ *   does_not_support_stepdowns,
+ *   # If all chunks are moved off of a shard, it can cause the plan cache to miss commands.
+ *   assumes_balancer_off,
+ *   assumes_unsharded_collection,
+ *   # Plan cache state is node-local and will not get migrated alongside tenant data.
+ *   tenant_migration_incompatible,
+ * ]
+ */
+
+const coll = db.server84338;
+coll.drop();
+
+const docs = [
+    {_id: 1, foo: 1},
+    {_id: 2, foo: 2},
+];
+assert.commandWorked(coll.insert(docs));
+
+const predicate = {
+    $or: [{_id: 1, foo: 1}, {_id: 1, foo: 999}]
+};
+const predicate2 = {
+    $or: [{_id: 2, foo: 2}, {_id: 2, foo: 999}]
+};
+
+// This query places an entry in the SBE plan cache.
+assert.eq([{_id: 1, foo: 1}], coll.find(predicate).toArray());
+// This query reuses this entry from the cache.
+assert.eq([{_id: 2, foo: 2}], coll.find(predicate2).toArray());
+
+coll.getPlanCache().clear();
+
+assert.eq(
+    [{_id: null, s: 1}],
+    coll.aggregate([{$match: predicate}, {$group: {_id: null, s: {$sum: "$foo"}}}]).toArray());
+assert.eq(
+    [{_id: null, s: 2}],
+    coll.aggregate([{$match: predicate2}, {$group: {_id: null, s: {$sum: "$foo"}}}]).toArray());

jstests/core/query/plan_cache/plan_cache_clear.js

@@ -24,7 +24,7 @@
 
 import {getPlanCacheKeyFromPipeline, getPlanCacheKeyFromShape} from "jstests/libs/analyze_plan.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.jstests_plan_cache_clear;
 coll.drop();
@@ -183,7 +183,7 @@ const nonExistentColl = db.plan_cache_clear_nonexistent;
 nonExistentColl.drop();
 assert.commandWorked(nonExistentColl.runCommand('planCacheClear'));
 
-if (checkSBEEnabled(db)) {
+if (checkSbeFullyEnabled(db)) {
     // Plan cache commands should work against the main collection only, not foreignColl
     // collections, when $lookup is pushed down into SBE.
     const foreignColl = db.plan_cache_clear_foreign;

jstests/core/query/plan_cache/plan_cache_list_plans.js

@@ -36,12 +36,12 @@ import {
 } from "jstests/libs/analyze_plan.js";
 import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
 import {QuerySettingsUtils} from "jstests/libs/query_settings_utils.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 let coll = db.jstests_plan_cache_list_plans;
 coll.drop();
 
-const isSbeEnabled = checkSBEEnabled(db);
+const isSbeEnabled = checkSbeFullyEnabled(db);
 
 function dumpPlanCacheState() {
     return coll.aggregate([{$planCacheStats: {}}]).toArray();

jstests/core/query/plan_cache/plan_cache_list_shapes.js

@@ -16,9 +16,9 @@
 //   # TODO SERVER-67607: Test plan cache with CQF enabled.
 //   cqf_experimental_incompatible,
 // ]
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
-if (checkSBEEnabled(db)) {
+if (checkSbeFullyEnabled(db)) {
     jsTest.log("Skipping test because SBE is enabled.");
     quit();
 }

jstests/core/query/plan_cache/plan_cache_sbe.js

@@ -24,11 +24,11 @@
  *   assumes_no_implicit_index_creation,
  * ]
  */
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
 const coll = db.plan_cache_sbe;
 coll.drop();
-const isSbeEnabled = checkSBEEnabled(db);
+const isSbeEnabled = checkSbeFullyEnabled(db);
 
 assert.commandWorked(coll.insert({a: 1, b: 1}));
 

jstests/core/query/plan_cache/plan_cache_sbe_hinted.js

@@ -13,15 +13,9 @@
  *   assumes_standalone_mongod,
  *   # TODO SERVER-67607: Test plan cache with CQF enabled.
  *   cqf_experimental_incompatible,
+ *   featureFlagSbeFull,
  * ]
  */
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
-if (!checkSBEEnabled(db)) {
-    jsTest.log("Skip running the test because SBE is not enabled");
-    quit();
-}
-
 const coll = db.plan_cache_sbe;
 coll.drop();
 

jstests/core/query/plan_cache/plan_cache_shell_helpers.js

@@ -17,9 +17,9 @@
 // ]
 import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
 import {getPlanCacheKeyFromShape} from "jstests/libs/analyze_plan.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
+import {checkSbeFullyEnabled} from "jstests/libs/sbe_util.js";
 
-const isSbeEnabled = checkSBEEnabled(db);
+const isSbeEnabled = checkSbeFullyEnabled(db);
 var coll = db.jstests_plan_cache_shell_helpers;
 coll.drop();
 

jstests/core/query/plan_cache/plan_cache_stats_shard_and_host.js

@@ -10,7 +10,6 @@
 //   tenant_migration_incompatible,
 //   # TODO SERVER-67607: Test plan cache with CQF enabled.
 //   cqf_experimental_incompatible,
-//   requires_fcv_71,
 // ]
 import {getPlanCacheKeyFromExplain} from "jstests/libs/analyze_plan.js";
 import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";

jstests/core/query/project/projection_semantics.js

@@ -12,15 +12,10 @@
  *   # server parameters are stored in-memory only so are not transferred onto the recipient.
  *   tenant_migration_incompatible,
  *   not_allowed_with_security_token,
+ *   featureFlagSbeFull,
  * ]
  */
 import {setUpServerForColumnStoreIndexTest} from "jstests/libs/columnstore_util.js";
-import {checkSBEEnabled} from "jstests/libs/sbe_util.js";
-
-if (!checkSBEEnabled(db, ["featureFlagColumnstoreIndexes"])) {
-    jsTestLog("Skipping test since columnstore Indexes are not enabled");
-    quit();
-}
 
 if (!setUpServerForColumnStoreIndexTest(db)) {
     quit();