SERVER-78122 Address bucket OID collisions with buckets pending compression

(cherry picked from commit 26a4bfd5bd)

README.md

@@ -1,6 +1,6 @@
 # ![Logo](docs/leaf.svg) MongoDB README
 
-Welcome to MongoDB!
+Welcome to MongoDB 6.3!
 
 ## Components
 

buildscripts/burn_in_tests.py

@@ -42,7 +42,7 @@ DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required"
 ENTERPRISE_MODULE_PATH = "src/mongo/db/modules/enterprise"
 DEFAULT_REPO_LOCATIONS = [".", f"./{ENTERPRISE_MODULE_PATH}"]
 REPEAT_SUITES = 2
-EVERGREEN_FILE = "etc/evergreen.yml"
+DEFAULT_EVG_PROJECT_FILE = "etc/evergreen.yml"
 # The executor_file and suite_files defaults are required to make the suite resolver work
 # correctly.
 SELECTOR_FILE = "etc/burn_in_tests.yml"
@@ -632,11 +632,14 @@ class BurnInOrchestrator:
     help="The revision in the mongo repo that changes will be compared against if specified.")
 @click.option("--install-dir", "install_dir", type=str,
               help="Path to bin directory of a testable installation")
+@click.option("--evg-project-file", "evg_project_file", default=DEFAULT_EVG_PROJECT_FILE,
+              help="Evergreen project config file")
 @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED)
 def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
          repeat_tests_min: Optional[int], repeat_tests_max: Optional[int],
          repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool,
-         origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool) -> None:
+         origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool,
+         evg_project_file: Optional[str]) -> None:
     """
     Run new or changed tests in repeated mode to validate their stability.
 
@@ -667,6 +670,9 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
     :param resmoke_args: Arguments to pass through to resmoke.
     :param verbose: Log extra debug information.
     :param origin_rev: The revision that local changes will be compared against.
+    :param install_dir: Path to bin directory of a testable installation.
+    :param use_yaml: Output discovered tasks in YAML. Tests will not be run.
+    :param evg_project_file: Evergreen project config file.
     """
     _configure_logging(verbose)
 
@@ -676,7 +682,7 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
                                  repeat_tests_num=repeat_tests_num)  # yapf: disable
 
     repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
-    evg_conf = parse_evergreen_file(EVERGREEN_FILE)
+    evg_conf = parse_evergreen_file(evg_project_file)
 
     change_detector = LocalFileChangeDetector(origin_rev)
     executor = LocalBurnInExecutor(resmoke_args, repeat_config)

buildscripts/combine_reports.py

@@ -25,7 +25,7 @@ def report_exit(combined_test_report):
     """Return report exit code.
 
     The exit code of this script is based on the following:
-        0:  All tests have status "pass", or only non-dynamic tests have status "silentfail".
+        0:  All tests have status "pass".
         31: At least one test has status "fail" or "timeout".
     Note: A test can be considered dynamic if its name contains a ":" character.
     """

buildscripts/debugsymb_mapper.py

@@ -157,19 +157,20 @@ class Mapper:
 
     default_web_service_base_url: str = "https://symbolizer-service.server-tig.prod.corp.mongodb.com"
     default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache')
-    selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug')
+    selected_binaries = ('mongos', 'mongod', 'mongo')
     default_client_credentials_scope = "servertig-symbolizer-fullaccess"
     default_client_credentials_user_name = "client-user"
     default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json')
 
-    def __init__(self, evg_version: str, evg_variant: str, client_id: str, client_secret: str,
-                 cache_dir: str = None, web_service_base_url: str = None,
+    def __init__(self, evg_version: str, evg_variant: str, is_san_variant: bool, client_id: str,
+                 client_secret: str, cache_dir: str = None, web_service_base_url: str = None,
                  logger: logging.Logger = None):
         """
         Initialize instance.
 
         :param evg_version: Evergreen version ID.
         :param evg_variant: Evergreen build variant name.
+        :param is_san_variant: Whether build variant is sanitizer build.
         :param client_id: Client id for Okta Oauth.
         :param client_secret: Secret key for Okta Oauth.
         :param cache_dir: Full path to cache directory as a string.
@@ -178,6 +179,7 @@ class Mapper:
         """
         self.evg_version = evg_version
         self.evg_variant = evg_variant
+        self.is_san_variant = is_san_variant
         self.cache_dir = cache_dir or self.default_cache_dir
         self.web_service_base_url = web_service_base_url or self.default_web_service_base_url
 
@@ -263,11 +265,13 @@ class Mapper:
 
         urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant)
 
-        download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None)
         binaries_url = urlinfo.urls.get("Binaries", "")
-
-        if not download_symbols_url:
-            download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None)
+        if self.is_san_variant:
+            # Sanitizer builds are not stripped and contain debug symbols
+            download_symbols_url = binaries_url
+        else:
+            download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz") or urlinfo.urls.get(
+                "mongo-debugsymbols.zip")
 
         if not download_symbols_url:
             self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s",
@@ -319,23 +323,17 @@ class Mapper:
 
         extractor = CmdOutputExtractor()
 
-        debug_symbols_path = self.download(self.debug_symbols_url)
-        debug_symbols_unpacked_path = self.unpack(debug_symbols_path)
-
         binaries_path = self.download(self.url)
         binaries_unpacked_path = self.unpack(binaries_path)
 
-        # we need to analyze two directories: main binary folder inside debug-symbols and
+        # we need to analyze two directories: main binary folder and
         # shared libraries folder inside binaries.
         # main binary folder holds main binaries, like mongos, mongod, mongo ...
         # shared libraries folder holds shared libraries, tons of them.
         # some build variants do not contain shared libraries.
 
-        debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test')
         binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test')
 
-        self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s",
-                         os.listdir(debug_symbols_unpacked_path))
         self.logger.info("INSIDE unpacked binaries/dist-test: %s",
                          os.listdir(binaries_unpacked_path))
 
@@ -352,19 +350,19 @@ class Mapper:
 
         # start with main binary folder
         for binary in self.selected_binaries:
-            full_bin_path = os.path.join(debug_symbols_unpacked_path,
+            full_bin_path = os.path.join(binaries_unpacked_path,
                                          self.path_options.main_binary_folder_name, binary)
 
             if not os.path.exists(full_bin_path):
                 self.logger.error("Could not find binary at %s", full_bin_path)
-                return
+                continue
 
             build_id_output = extractor.get_build_id(full_bin_path)
 
             if not build_id_output.build_id:
                 self.logger.error("Build ID couldn't be extracted. \nReadELF output %s",
                                   build_id_output.cmd_output)
-                return
+                continue
             else:
                 self.logger.info("Extracted build ID: %s", build_id_output.build_id)
 
@@ -397,14 +395,14 @@ class Mapper:
 
             if not os.path.exists(sofile_path):
                 self.logger.error("Could not find binary at %s", sofile_path)
-                return
+                continue
 
             build_id_output = extractor.get_build_id(sofile_path)
 
             if not build_id_output.build_id:
                 self.logger.error("Build ID couldn't be extracted. \nReadELF out %s",
                                   build_id_output.cmd_output)
-                return
+                continue
             else:
                 self.logger.info("Extracted build ID: %s", build_id_output.build_id)
 
@@ -426,6 +424,7 @@ class Mapper:
 
         # mappings is a generator, we iterate over to generate mappings on the go
         for mapping in mappings:
+            self.logger.info("Creating mapping %s", mapping)
             response = self.http_client.post('/'.join((self.web_service_base_url, 'add')),
                                              json=mapping)
             if response.status_code != 200:
@@ -440,11 +439,12 @@ def make_argument_parser(parser=None, **kwargs):
     if parser is None:
         parser = argparse.ArgumentParser(**kwargs)
 
-    parser.add_argument('--version')
-    parser.add_argument('--client-id')
-    parser.add_argument('--client-secret')
-    parser.add_argument('--variant')
-    parser.add_argument('--web-service-base-url', default="")
+    parser.add_argument("--version")
+    parser.add_argument("--client-id")
+    parser.add_argument("--client-secret")
+    parser.add_argument("--variant")
+    parser.add_argument("--is-san-variant", action="store_true")
+    parser.add_argument("--web-service-base-url", default="")
     return parser
 
 
@@ -452,7 +452,8 @@ def main(options):
     """Execute mapper here. Main entry point."""
 
     mapper = Mapper(evg_version=options.version, evg_variant=options.variant,
-                    client_id=options.client_id, client_secret=options.client_secret,
+                    is_san_variant=options.is_san_variant, client_id=options.client_id,
+                    client_secret=options.client_secret,
                     web_service_base_url=options.web_service_base_url)
 
     # when used as a context manager, mapper instance automatically cleans files/folders after finishing its job.

buildscripts/evergreen_task_timeout.py

@@ -28,20 +28,21 @@ DEFAULT_TIMEOUT_OVERRIDES = "etc/evergreen_timeouts.yml"
 DEFAULT_EVERGREEN_CONFIG = "etc/evergreen.yml"
 DEFAULT_EVERGREEN_AUTH_CONFIG = "~/.evergreen.yml"
 COMMIT_QUEUE_ALIAS = "__commit_queue"
-UNITTEST_TASK = "run_unittests"
 IGNORED_SUITES = {
-    "integration_tests_replset", "integration_tests_replset_ssl_auth", "integration_tests_sharded",
-    "integration_tests_standalone", "integration_tests_standalone_audit", "mongos_test",
-    "server_selection_json_test"
+    "integration_tests_replset",
+    "integration_tests_replset_ssl_auth",
+    "integration_tests_sharded",
+    "integration_tests_standalone",
+    "integration_tests_standalone_audit",
+    "mongos_test",
+    "server_selection_json_test",
+    "sdam_json_test",
 }
 HISTORY_LOOKBACK = timedelta(weeks=2)
 
-COMMIT_QUEUE_TIMEOUT = timedelta(minutes=40)
+COMMIT_QUEUE_TIMEOUT = timedelta(minutes=20)
 DEFAULT_REQUIRED_BUILD_TIMEOUT = timedelta(hours=1, minutes=20)
 DEFAULT_NON_REQUIRED_BUILD_TIMEOUT = timedelta(hours=2)
-# 2x the longest "run tests" phase for unittests as of c9bf1dbc9cc46e497b2f12b2d6685ef7348b0726,
-# which is 5 mins 47 secs, excluding outliers below
-UNITTESTS_TIMEOUT = timedelta(minutes=12)
 
 
 class TimeoutOverride(BaseModel):
@@ -225,11 +226,6 @@ class TaskTimeoutOrchestrator:
             LOGGER.info("Overriding configured timeout", exec_timeout_secs=override.total_seconds())
             determined_timeout = override
 
-        elif task_name == UNITTEST_TASK and override is None:
-            LOGGER.info("Overriding unittest timeout",
-                        exec_timeout_secs=UNITTESTS_TIMEOUT.total_seconds())
-            determined_timeout = UNITTESTS_TIMEOUT
-
         elif _is_required_build_variant(
                 variant) and determined_timeout > DEFAULT_REQUIRED_BUILD_TIMEOUT:
             LOGGER.info("Overriding required-builder timeout",

buildscripts/gdb/mongo.py

@@ -14,7 +14,7 @@ import gdb
 
 if not gdb:
     sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent))
-    from buildscripts.gdb.mongo_printers import absl_get_nodes, get_unique_ptr
+    from buildscripts.gdb.mongo_printers import absl_get_nodes, get_unique_ptr, get_unique_ptr_bytes
 
 
 def detect_toolchain(progspace):
@@ -249,7 +249,7 @@ def get_decorations(obj):
         type_name = type_name[0:type_name.rindex(">")]
         type_name = type_name[type_name.index("constructAt<"):].replace("constructAt<", "")
         # get_unique_ptr should be loaded from 'mongo_printers.py'.
-        decoration_data = get_unique_ptr(decorable["_decorations"]["_decorationData"])
+        decoration_data = get_unique_ptr_bytes(decorable["_decorations"]["_decorationData"])
 
         if type_name.endswith('*'):
             type_name = type_name[0:len(type_name) - 1]

buildscripts/gdb/mongo_printers.py

@@ -22,9 +22,20 @@ if sys.version_info[0] < 3:
         "MongoDB gdb extensions only support Python 3. Your GDB was compiled against Python 2")
 
 
+def get_unique_ptr_bytes(obj):
+    """Read the value of a libstdc++ std::unique_ptr.
+
+    Returns a gdb.Value where its type resolves to `unsigned char*`. The caller must take care to
+    cast the returned value themselves. This function is particularly useful in the context of
+    mongo::Decorable<> types which store the decorations as a slab of memory with
+    std::unique_ptr<unsigned char[]>. In all other cases get_unique_ptr() can be preferred.
+    """
+    return obj.cast(gdb.lookup_type('std::_Head_base<0, unsigned char*, false>'))['_M_head_impl']
+
+
 def get_unique_ptr(obj):
     """Read the value of a libstdc++ std::unique_ptr."""
-    return obj.cast(gdb.lookup_type('std::_Head_base<0, unsigned char*, false>'))['_M_head_impl']
+    return get_unique_ptr_bytes(obj).cast(obj.type.template_argument(0).pointer())
 
 
 ###################################################################################################
@@ -303,7 +314,7 @@ class DecorablePrinter(object):
 
     def children(self):
         """Children."""
-        decoration_data = get_unique_ptr(self.val["_decorations"]["_decorationData"])
+        decoration_data = get_unique_ptr_bytes(self.val["_decorations"]["_decorationData"])
 
         for index in range(self.count):
             descriptor = self.start[index]

buildscripts/generate_compile_expansions_shared_cache.py

@@ -61,7 +61,7 @@ def generate_scons_cache_expansions():
         if sys.platform.startswith("win"):
             shared_mount_root = 'X:\\'
         else:
-            shared_mount_root = '/efs'
+            shared_mount_root = '/efs/scons'
         default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache")
         expansions["scons_cache_path"] = default_cache_path
         expansions[

buildscripts/generate_version_expansions.py

@@ -61,8 +61,8 @@ def generate_version_expansions():
             raise ValueError("Unable to parse version from stdin and no version.json provided")
 
     if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
+        expansions["suffix"] = "v6.3-latest"
+        expansions["src_suffix"] = "v6.3-latest"
         expansions["is_release"] = "false"
     else:
         expansions["suffix"] = version_line

buildscripts/package_test.py

@@ -21,6 +21,7 @@ import requests
 from docker.client import DockerClient
 from docker.models.containers import Container
 from docker.models.images import Image
+
 from simple_report import Report, Result
 
 root = logging.getLogger()
@@ -508,7 +509,9 @@ if args.command == "release":
 
         mongosh_package = get_mongosh_package(arch, test_os)
         if mongosh_package:
-            urls.append(mongosh_package)
+            # The mongosh package doesn't work on Ubuntu 16.04
+            if test_os != "ubuntu1604":
+                urls.append(mongosh_package)
         else:
             logging.error("Could not find mongosh package for %s and %s", arch, test_os)
             sys.exit(1)

buildscripts/package_test_internal.py

@@ -91,14 +91,14 @@ def run_zypper_test(packages: List[str]):
     run_and_log("zypper -n --no-gpg-checks install {}".format(' '.join(packages)))
 
 
-def run_mongo_query(query, should_fail=False, tries=60, interval=1.0):
+def run_mongo_query(shell, query, should_fail=False, tries=60, interval=1.0):
     # type: (str, bool, int, float) -> Optional[subprocess.CompletedProcess[bytes]]
     assert tries >= 1
 
     exec_result = None  # type: Union[subprocess.CompletedProcess[bytes], None]
     current_try = 1
 
-    command = "mongosh --eval '{}'".format(query)
+    command = "{} --eval '{}'".format(shell, query)
 
     # Keep trying the query until we either get a successful return code or we
     # run out of tries.
@@ -248,6 +248,11 @@ def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs:
     else:
         test_args["systemd_path"] = "/bin"
 
+    if not pathlib.Path("/usr/bin/mongosh").exists():
+        test_args["mongo_shell"] = "/usr/bin/mongo"
+    else:
+        test_args["mongo_shell"] = "/usr/bin/mongosh"
+
     return test_args
 
 
@@ -308,7 +313,7 @@ def test_start():
     run_and_log("systemctl enable mongod.service")
     run_and_log("systemctl start mongod.service")
 
-    run_mongo_query("db.smoke.insertOne({answer: 42})")
+    run_mongo_query(test_args["mongo_shell"], "db.smoke.insertOne({answer: 42})")
     run_and_log("systemctl is-active mongod.service")
 
 
@@ -404,7 +409,7 @@ def test_restart():
     run_and_log("systemctl restart mongod.service")
 
     logging.debug("Waiting up to 60 seconds for mongod to restart...")
-    run_mongo_query("db.smoke.insertOne({answer: 42})")
+    run_mongo_query(test_args["mongo_shell"], "db.smoke.insertOne({answer: 42})")
 
     run_and_log("systemctl is-active mongod.service")
 
@@ -415,7 +420,7 @@ def test_stop():
     run_and_log("systemctl stop mongod.service")
 
     logging.debug("Waiting up to 60 seconds for mongod to finish shutting down...")
-    run_mongo_query("db.smoke.insertOne({answer: 42})", should_fail=True)
+    run_mongo_query(test_args["mongo_shell"], "db.smoke.insertOne({answer: 42})", should_fail=True)
 
     run_and_log("systemctl is-active mongod.service", end_on_error=False)
 

buildscripts/promote_silent_failures.py

@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-"""Convert silent test failures into non-silent failures.
-
-Any test files with at least 2 executions in the report.json file that have a "silentfail" status,
-this script will change the outputted report to have a "fail" status instead.
-"""
-
-import collections
-import json
-import optparse
-import os
-import sys
-
-# Get relative imports to work when the package is not installed on the PYTHONPATH.
-if __name__ == "__main__" and __package__ is None:
-    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-    from buildscripts.resmokelib.testing import report
-
-
-def read_json_file(json_file):
-    """Return contents of a JSON file."""
-    with open(json_file) as json_data:
-        return json.load(json_data)
-
-
-def main():
-    """Execute Main program."""
-
-    usage = "usage: %prog [options] report.json"
-    parser = optparse.OptionParser(usage=usage)
-    parser.add_option(
-        "-o", "--output-file", dest="outfile", default="-",
-        help=("If '-', then the report file is written to stdout."
-              " Any other value is treated as the output file name. By default,"
-              " output is written to stdout."))
-
-    (options, args) = parser.parse_args()
-
-    if len(args) != 1:
-        parser.error("Requires a single report.json file.")
-
-    report_file_json = read_json_file(args[0])
-    test_report = report.TestReport.from_dict(report_file_json)
-
-    # Count number of "silentfail" per test file.
-    status_dict = collections.defaultdict(int)
-    for test_info in test_report.test_infos:
-        if test_info.evergreen_status == "silentfail":
-            status_dict[test_info.test_id] += 1
-
-    # For test files with more than 1 "silentfail", convert status to "fail".
-    for test_info in test_report.test_infos:
-        if status_dict[test_info.test_id] >= 2:
-            test_info.evergreen_status = "fail"
-
-    result_report = test_report.as_dict()
-    if options.outfile != "-":
-        with open(options.outfile, "w") as fp:
-            json.dump(result_report, fp)
-    else:
-        print(json.dumps(result_report))
-
-
-if __name__ == "__main__":
-    main()

buildscripts/resmokeconfig/suites/change_streams_multitenant_passthrough.yml

@@ -7,6 +7,8 @@ selector:
   ##
   # TODO SERVER-68341: Implement enable/disable command for mongoQ in the serverless.
   - jstests/change_streams/projection_fakes_internal_event.js
+  # TODO SERVER-69959: Implement a majority-committed insert listener.
+  - jstests/change_streams/only_wake_getmore_for_relevant_changes.js
 
   ##
   # TODO SERVER-70760: This test creates its own sharded cluster and uses transaction. The support

buildscripts/resmokeconfig/suites/change_streams_multitenant_sharded_collections_passthrough.yml

@@ -4,6 +4,8 @@ selector:
   roots:
   - jstests/change_streams/**/*.js
   exclude_files:
+  # TODO SERVER-69959: Implement a majority-committed insert listener.
+  - jstests/change_streams/only_wake_getmore_for_relevant_changes.js
   # TODO SERVER-68341: Implement enable/disable command for mongoQ in the serverless.
   - jstests/change_streams/projection_fakes_internal_event.js
   # TODO SERVER-68557 This test list databases that does not work in the sharded-cluster. This test

buildscripts/resmokeconfig/suites/parallel.yml

@@ -25,5 +25,3 @@ executor:
     mongod_options:
       set_parameters:
         enableTestCommands: 1
-        # Fully enable column scan for CSI tests. This can't be done at runtime because it interferes with other tests.
-        internalQueryColumnScanMinNumColumnFilters: 0

buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml

@@ -3,8 +3,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
-  - jstests/fle2/**/*.js
-  - src/mongo/db/modules/*/jstests/fle2/**/*.js
   exclude_files:
   # Transactions do not support retryability of individual operations.
   # TODO: Remove this once it is supported (SERVER-33952).

buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml

@@ -3,8 +3,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
-  - jstests/fle2/**/*.js
-  - src/mongo/db/modules/*/jstests/fle2/**/*.js
   exclude_files:
   # Transactions do not support retryability of individual operations.
   # TODO: Remove this once it is supported (SERVER-33952).

buildscripts/resmokelib/config.py

@@ -90,7 +90,6 @@ DEFAULTS = {
     "repeat_tests_min": None,
     "repeat_tests_secs": None,
     "replay_file": None,
-    "report_failure_status": "fail",
     "report_file": None,
     "run_all_feature_flag_tests": False,
     "run_no_feature_flag_tests": False,
@@ -169,7 +168,6 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [
     "num_repeat_tests_max",
     "num_repeat_tests_min",
     "time_repeat_tests_secs",
-    "report_failure_status",
 ])
 
 
@@ -236,7 +234,6 @@ class SuiteOptions(_SuiteOptions):
                     REPEAT_TESTS_MAX,
                     REPEAT_TESTS_MIN,
                     REPEAT_TESTS_SECS,
-                    REPORT_FAILURE_STATUS,
                 ])))
 
         options = self._asdict()
@@ -445,9 +442,6 @@ REPEAT_TESTS_MIN = None
 # If set, then each test is repeated the specified time (seconds) inside the suites.
 REPEAT_TESTS_SECS = None
 
-# Controls if the test failure status should be reported as failed or be silently ignored.
-REPORT_FAILURE_STATUS = None
-
 # If set, then resmoke.py will write out a report file with the status of each test that ran.
 REPORT_FILE = None
 

buildscripts/resmokelib/configure_resmoke.py

@@ -331,7 +331,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py
     _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max")
     _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min")
     _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs")
-    _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
     _config.REPORT_FILE = config.pop("report_file")
     _config.SERVICE_EXECUTOR = config.pop("service_executor")
     _config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config")

buildscripts/resmokelib/run/__init__.py

@@ -490,25 +490,12 @@ class TestRunnerEvg(TestRunner):
     additional options for running unreliable tests in Evergreen.
     """
 
-    UNRELIABLE_TAG = _TagInfo(
-        tag_name="unreliable",
-        evergreen_aware=True,
-        suite_options=config.SuiteOptions.ALL_INHERITED._replace(  # type: ignore
-            report_failure_status="silentfail"))
-
     RESOURCE_INTENSIVE_TAG = _TagInfo(
         tag_name="resource_intensive",
         evergreen_aware=False,
         suite_options=config.SuiteOptions.ALL_INHERITED._replace(  # type: ignore
             num_jobs=1))
 
-    RETRY_ON_FAILURE_TAG = _TagInfo(
-        tag_name="retry_on_failure",
-        evergreen_aware=True,
-        suite_options=config.SuiteOptions.ALL_INHERITED._replace(  # type: ignore
-            fail_fast=False, num_repeat_suites=2, num_repeat_tests=1,
-            report_failure_status="silentfail"))
-
     @staticmethod
     def _make_evergreen_aware_tags(tag_name):
         """Return a list of resmoke.py tags.
@@ -543,29 +530,8 @@ class TestRunnerEvg(TestRunner):
 
         combinations = []
 
-        if config.EVERGREEN_PATCH_BUILD:
-            combinations.append(("unreliable and resource intensive",
-                                 ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True))))
-            combinations.append(("unreliable and not resource intensive",
-                                 ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False))))
-            combinations.append(("reliable and resource intensive",
-                                 ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True))))
-            combinations.append(("reliable and not resource intensive",
-                                 ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
-                                                                False))))
-        else:
-            combinations.append(("retry on failure and resource intensive",
-                                 ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
-                                                                     True))))
-            combinations.append(("retry on failure and not resource intensive",
-                                 ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
-                                                                     False))))
-            combinations.append(("run once and resource intensive",
-                                 ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
-                                                                      True))))
-            combinations.append(("run once and not resource intensive",
-                                 ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
-                                                                      False))))
+        combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)]))
+        combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)]))
 
         return combinations
 
@@ -998,13 +964,6 @@ class RunPlugin(PluginInterface):
                                       metavar="CEDAR_REPORT",
                                       help="Writes a JSON file with performance test results.")
 
-        internal_options.add_argument(
-            "--reportFailureStatus", action="store", dest="report_failure_status",
-            choices=("fail", "silentfail"), metavar="STATUS",
-            help="Controls if the test failure status should be reported as failed"
-            " or be silently ignored (STATUS=silentfail). Dynamic test failures will"
-            " never be silently ignored. Defaults to STATUS=%(default)s.")
-
         internal_options.add_argument(
             "--reportFile", dest="report_file", metavar="REPORT",
             help="Writes a JSON file with test status and timing information.")

buildscripts/resmokelib/testing/fixtures/standalone.py

@@ -256,6 +256,15 @@ class MongodLauncher(object):
         if "shardsvr" in mongod_options and "orphanCleanupDelaySecs" not in suite_set_parameters:
             suite_set_parameters["orphanCleanupDelaySecs"] = 1
 
+        # receiveChunkWaitForRangeDeleterTimeoutMS controls the amount of time an incoming migration
+        # will wait for an intersecting range with data in it to be cleared up before failing. The
+        # default is 10 seconds, but in some slower variants this is not enough time for the range
+        # deleter to finish so we increase it here to 90 seconds. Setting a value for this parameter
+        # in the .yml file overrides this.
+        if (("shardsvr" in mongod_options or "configsvr" in mongod_options)
+                and "receiveChunkWaitForRangeDeleterTimeoutMS" not in suite_set_parameters):
+            suite_set_parameters["receiveChunkWaitForRangeDeleterTimeoutMS"] = 90000
+
         # The LogicalSessionCache does automatic background refreshes in the server. This is
         # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off
         # background refreshing for tests. Set in the .yml file to override this.

buildscripts/resmokelib/testing/hooks/tenant_migration.py

@@ -234,6 +234,7 @@ class _TenantMigrationThread(threading.Thread):
     WAIT_SECS_RANGES = [[0.05, 0.1], [0.1, 0.5], [1, 5], [5, 15]]
     POLL_INTERVAL_SECS = 0.1
 
+    MIGRATION_ABORTED_ERR_CODE = 325
     NO_SUCH_MIGRATION_ERR_CODE = 327
     INTERNAL_ERR_CODE = 1
     INVALID_SYNC_SOURCE_ERR_CODE = 119
@@ -364,6 +365,10 @@ class _TenantMigrationThread(threading.Thread):
         return abort_reason["code"] == self.INTERNAL_ERR_CODE and abort_reason[
             "errmsg"] == "simulate a tenant migration error"
 
+    def _is_recipient_failover_abort_reason(self, abort_reason):
+        return abort_reason["code"] == self.MIGRATION_ABORTED_ERR_CODE and abort_reason[
+            "errmsg"].find("Recipient failover happened during migration")
+
     def _create_migration_opts(self, donor_rs_index, recipient_rs_index):
         donor_rs = self._tenant_migration_fixture.get_replset(donor_rs_index)
         recipient_rs = self._tenant_migration_fixture.get_replset(recipient_rs_index)
@@ -426,7 +431,11 @@ class _TenantMigrationThread(threading.Thread):
                 return True
 
             abort_reason = res["abortReason"]
-            if self._is_fail_point_abort_reason(abort_reason):
+            if self._is_recipient_failover_abort_reason(abort_reason):
+                self.logger.info("Tenant migration '%s' aborted due to recipient failover: %s",
+                                 migration_opts.migration_id, str(res))
+                return False
+            elif self._is_fail_point_abort_reason(abort_reason):
                 self.logger.info(
                     "Tenant migration '%s' with donor replica set '%s' aborted due to failpoint: " +
                     "%s.", migration_opts.migration_id, migration_opts.get_donor_name(), str(res))

buildscripts/resmokelib/testing/report.py

@@ -218,12 +218,7 @@ class TestReport(unittest.TestResult):
 
             test_info = self.find_test_info(test)
             test_info.status = "fail"
-            if test_info.dynamic:
-                # Dynamic tests are used for data consistency checks, so the failures are never
-                # silenced.
-                test_info.evergreen_status = "fail"
-            else:
-                test_info.evergreen_status = self.suite_options.report_failure_status
+            test_info.evergreen_status = "fail"
             test_info.return_code = test.return_code
 
     def setFailure(self, test, return_code=1):
@@ -235,12 +230,7 @@ class TestReport(unittest.TestResult):
                 raise ValueError("stopTest was not called on %s" % (test.basename()))
 
             test_info.status = "fail"
-            if test_info.dynamic:
-                # Dynamic tests are used for data consistency checks, so the failures are never
-                # silenced.
-                test_info.evergreen_status = "fail"
-            else:
-                test_info.evergreen_status = self.suite_options.report_failure_status
+            test_info.evergreen_status = "fail"
             test_info.return_code = return_code
 
         # Recompute number of success, failures, and errors.

buildscripts/resmokelib/testing/symbolizer_service.py

@@ -57,6 +57,15 @@ class ResmokeSymbolizerConfig(NamedTuple):
         """
         return sys.platform == "win32" or sys.platform == "cygwin"
 
+    @staticmethod
+    def is_macos() -> bool:
+        """
+        Whether we are on MacOS.
+
+        :return: True if on MacOS.
+        """
+        return sys.platform == "darwin"
+
 
 class ResmokeSymbolizer:
     """Symbolize stacktraces inside test logs."""
@@ -135,6 +144,10 @@ class ResmokeSymbolizer:
             test.logger.info("Running on Windows, skipping symbolization")
             return False
 
+        if self.config.is_macos():
+            test.logger.info("Running on MacOS, skipping symbolization")
+            return False
+
         return True
 
     def get_stacktrace_dir(self, test: TestCase) -> Optional[str]:

buildscripts/tests/resmokelib/test_parser.py

@@ -231,7 +231,6 @@ class TestLocalCommandLine(unittest.TestCase):
         cmdline = to_local_args([
             "run",
             "--suites=my_suite",
-            "--reportFailureStatus=fail",
             "--reportFile=report.json",
             "--perfReportFile=perf.json",
             "--storageEngine=my_storage_engine",

buildscripts/tests/resmokelib/testing/test_symbolizer_service.py

@@ -15,6 +15,7 @@ def mock_resmoke_symbolizer_config():
     config_mock.client_id = "client_id"
     config_mock.client_secret = "client_secret"
     config_mock.is_windows.return_value = False
+    config_mock.is_macos.return_value = False
     return config_mock
 
 
@@ -85,6 +86,11 @@ class TestResmokeSymbolizer(unittest.TestCase):
         ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
         self.assertFalse(ret)
 
+    def test_should_not_symbolize_if_on_macos(self):
+        self.config_mock.is_macos.return_value = True
+        ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
+        self.assertFalse(ret)
+
     def test_should_symbolize_return_true(self):
         ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
         self.assertTrue(ret)

buildscripts/timeouts/timeout_service.py

@@ -5,8 +5,7 @@ import inject
 import structlog
 from buildscripts.resmoke_proxy.resmoke_proxy import ResmokeProxyService
 from buildscripts.timeouts.timeout import TimeoutEstimate
-from buildscripts.util.teststats import HistoricTaskData
-from evergreen import EvergreenApi
+from buildscripts.util.teststats import HistoricTaskData, normalize_test_name
 
 LOGGER = structlog.get_logger(__name__)
 CLEAN_EVERY_N_HOOK = "CleanEveryN"
@@ -53,7 +52,10 @@ class TimeoutService:
         if not historic_stats:
             return TimeoutEstimate.no_timeouts()
 
-        test_set = set(self.resmoke_proxy.list_tests(timeout_params.suite_name))
+        test_set = {
+            normalize_test_name(test)
+            for test in self.resmoke_proxy.list_tests(timeout_params.suite_name)
+        }
         test_runtimes = [
             stat for stat in historic_stats.get_tests_runtimes() if stat.test_name in test_set
         ]

etc/asan.denylist

@@ -0,0 +1,3 @@
+#SERVER-74122
+#third party should not be subject to sanitizer
+src:src/third_party/IntelRDFPMathLib20U1/TESTS/readtest.c

etc/backports_required_for_multiversion_tests.yml

@@ -30,6 +30,8 @@ last-continuous:
     ticket: SERVER-53335
   - test_file: jstests/replsets/replSetGetStatus_member_wall_times.js
     ticket: SERVER-54909
+  - test_file: jstests/sharding/migration_server_status.js
+    ticket: SERVER-56186
   - test_file: jstests/sharding/cwwc_conflict_add_shard.js
     ticket: SERVER-56800
   - test_file: jstests/sharding/reconfig_fails_no_cwwc_set_sharding.js
@@ -254,12 +256,16 @@ last-continuous:
     ticket: SERVER-67723
   - test_file: jstests/sharding/collection_uuid_shard_capped_collection.js
     ticket: SERVER-67885
+  - test_file: jstests/sharding/prepare_transaction_then_migrate.js
+    ticket: SERVER-68361
   - test_file: jstests/core/txns/txn_ops_allowed_on_buckets_coll.js
     ticket: SERVER-68556
   - test_file: jstests/core/txns/no_writes_to_system_collections_in_txn.js
     ticket: SERVER-68556
   - test_file: jstests/sharding/resharding_temp_ns_routing_info_unsharded.js
     ticket: SERVER-68628
+  - test_file: jstests/sharding/move_chunk_concurrent_cloning.js
+    ticket: SERVER-68648
   - test_file: jstests/sharding/move_chunk_interrupt_postimage.js
     ticket: SERVER-68728
   - test_file: jstests/sharding/resharding_critical_section_metrics.js
@@ -272,26 +278,22 @@ last-continuous:
     ticket: SERVER-69861
   - test_file: jstests/aggregation/expressions/switch_errors.js
     ticket: SERVER-70190
-  - test_file: jstests/core/cover_null_queries.js
-    ticket: SERVER-70436
-  - test_file: jstests/sharding/prepare_transaction_then_migrate.js
-    ticket: SERVER-68361
-  - test_file: jstests/sharding/migration_server_status.js
-    ticket: SERVER-56186
-  - test_file: jstests/replsets/apply_ops_strips_hash.js
-    ticket: SERVER-70841
   - test_file: jstests/replsets/wtimeout_too_large.js
     ticket: SERVER-70360
-  - test_file: jstests/sharding/all_collection_stats.js
-    ticket: SERVER-71477
+  - test_file: jstests/core/cover_null_queries.js
+    ticket: SERVER-70436
   - test_file: jstests/sharding/timeseries_sharding_admin_commands.js
     ticket: SERVER-70790
+  - test_file: jstests/replsets/apply_ops_strips_hash.js
+    ticket: SERVER-70841
+  - test_file: jstests/sharding/all_collection_stats.js
+    ticket: SERVER-71477
+  - test_file: jstests/sharding/hidden_index.js
+    ticket: SERVER-71638
   - test_file: jstests/sharding/invalid_system_views_sharded_collection.js
     ticket: SERVER-71667
   - test_file: jstests/sharding/drop_collection.js
     ticket: SERVER-71689
-  - test_file: jstests/sharding/hidden_index.js
-    ticket: SERVER-71638
   - test_file: jstests/core/query/project/project_with_collation.js
     ticket: SERVER-72416
   - test_file: jstests/core/collation.js
@@ -300,12 +302,46 @@ last-continuous:
     ticket: SERVER-72416
   - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/oplog_rollover.js
     ticket: SERVER-72422
-  - test_file: jstests/sharding/move_chunk_concurrent_cloning.js
-    ticket: SERVER-68648
   - test_file: jstests/sharding/timeseries_cluster_indexstats.js
     ticket: SERVER-72620
   - test_file: jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js
     ticket: SERVER-73110
+  - test_file: jstests/core/doc_validation_with_now_variable.js
+    ticket: SERVER-73125
+  - test_file: jstests/core/timeseries/timeseries_groupby_reorder.js
+    ticket: SERVER-73822
+  - test_file: jstests/sharding/query/collation_targeting_inherited.js
+    ticket: SERVER-72224
+  - test_file: jstests/core/clustered/clustered_collection_collation.js
+    ticket: SERVER-72224
+  - test_file: jstests/sharding/internal_txns/incomplete_transaction_history_during_migration.js
+    ticket: SERVER-73938
+  - test_file: jstests/sharding/shard_keys_with_dollar_sign.js
+    ticket: SERVER-74124
+  - test_file: jstests/core/query/sbe_plan_cache_autoparameterize_ixscan.js
+    ticket: SERVER-74262
+  - test_file: jstests/sharding/query/group_plan_cache_sharded.js
+    ticket: SERVER-74245
+  - test_file: jstests/core/field_name_validation.js
+    ticket: SERVER-75517
+  - test_file: jstests/core/clustered/clustered_collection_hint.js
+    ticket: SERVER-73482
+  - test_file: jstests/core/clustered/clustered_collection_collation.js
+    ticket: SERVER-67105
+  - test_file: jstests/core/clustered/clustered_collection_bounded_scan.js
+    ticket: SERVER-67105
+  - test_file: jstests/noPassthrough/clustered_collection_sorted_scan.js
+    ticket: SERVER-76102
+  - test_file: jstests/replsets/session_cache_refresh_write_error_fail.js
+    ticket: SERVER-73229
+  - test_file: jstests/noPassthrough/ttl_operation_metrics_multi_dbs.js
+    ticket: SERVER-75277
+  - test_file: jstests/core/map_reduce_subplanning.js
+    ticket: SERVER-74131
+  - test_file: jstests/core/timeseries/timeseries_filter_extended_range.js
+    ticket: SERVER-69952
+  - test_file: jstests/sharding/refine_collection_shard_key_basic.js
+    ticket: SERVER-76394
   suites: null
 last-lts:
   all:
@@ -405,6 +441,8 @@ last-lts:
     ticket: SERVER-55305
   - test_file: jstests/replsets/rollback_with_coalesced_txn_table_updates_from_vectored_inserts.js
     ticket: SERVER-55305
+  - test_file: jstests/sharding/migration_server_status.js
+    ticket: SERVER-56186
   - test_file: jstests/sharding/time_zone_info_mongos.js
     ticket: SERVER-56371
   - test_file: jstests/concurrency/fsm_workloads/findAndModify_flip_location.js
@@ -637,12 +675,16 @@ last-lts:
     ticket: SERVER-67723
   - test_file: jstests/sharding/collection_uuid_shard_capped_collection.js
     ticket: SERVER-67885
+  - test_file: jstests/sharding/prepare_transaction_then_migrate.js
+    ticket: SERVER-68361
   - test_file: jstests/core/txns/txn_ops_allowed_on_buckets_coll.js
     ticket: SERVER-68556
   - test_file: jstests/core/txns/no_writes_to_system_collections_in_txn.js
     ticket: SERVER-68556
   - test_file: jstests/sharding/resharding_temp_ns_routing_info_unsharded.js
     ticket: SERVER-68628
+  - test_file: jstests/sharding/move_chunk_concurrent_cloning.js
+    ticket: SERVER-68648
   - test_file: jstests/sharding/move_chunk_interrupt_postimage.js
     ticket: SERVER-68728
   - test_file: jstests/replsets/tenant_migration_recipient_forget_migration.js
@@ -653,26 +695,22 @@ last-lts:
     ticket: SERVER-69861
   - test_file: jstests/aggregation/expressions/switch_errors.js
     ticket: SERVER-70190
-  - test_file: jstests/core/cover_null_queries.js
-    ticket: SERVER-70436
-  - test_file: jstests/sharding/prepare_transaction_then_migrate.js
-    ticket: SERVER-68361
-  - test_file: jstests/sharding/migration_server_status.js
-    ticket: SERVER-56186
-  - test_file: jstests/replsets/apply_ops_strips_hash.js
-    ticket: SERVER-70841
   - test_file: jstests/replsets/wtimeout_too_large.js
     ticket: SERVER-70360
-  - test_file: jstests/sharding/all_collection_stats.js
-    ticket: SERVER-71477
+  - test_file: jstests/core/cover_null_queries.js
+    ticket: SERVER-70436
   - test_file: jstests/sharding/timeseries_sharding_admin_commands.js
     ticket: SERVER-70790
+  - test_file: jstests/replsets/apply_ops_strips_hash.js
+    ticket: SERVER-70841
+  - test_file: jstests/sharding/all_collection_stats.js
+    ticket: SERVER-71477
+  - test_file: jstests/sharding/hidden_index.js
+    ticket: SERVER-71638
   - test_file: jstests/sharding/invalid_system_views_sharded_collection.js
     ticket: SERVER-71667
   - test_file: jstests/sharding/drop_collection.js
     ticket: SERVER-71689
-  - test_file: jstests/sharding/hidden_index.js
-    ticket: SERVER-71638
   - test_file: jstests/core/query/project/project_with_collation.js
     ticket: SERVER-72416
   - test_file: jstests/core/collation.js
@@ -681,10 +719,40 @@ last-lts:
     ticket: SERVER-72416
   - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/oplog_rollover.js
     ticket: SERVER-72422
-  - test_file: jstests/sharding/move_chunk_concurrent_cloning.js
-    ticket: SERVER-68648
   - test_file: jstests/sharding/timeseries_cluster_indexstats.js
     ticket: SERVER-72620
   - test_file: jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js
     ticket: SERVER-73110
+  - test_file: jstests/core/doc_validation_with_now_variable.js
+    ticket: SERVER-73125
+  - test_file: jstests/core/timeseries/timeseries_groupby_reorder.js
+    ticket: SERVER-73822
+  - test_file: jstests/sharding/query/collation_targeting_inherited.js
+    ticket: SERVER-72224
+  - test_file: jstests/core/clustered/clustered_collection_collation.js
+    ticket: SERVER-72224
+  - test_file: jstests/sharding/internal_txns/incomplete_transaction_history_during_migration.js
+    ticket: SERVER-73938
+  - test_file: jstests/sharding/shard_keys_with_dollar_sign.js
+    ticket: SERVER-74124
+  - test_file: jstests/core/field_name_validation.js
+    ticket: SERVER-75517
+  - test_file: jstests/core/clustered/clustered_collection_hint.js
+    ticket: SERVER-73482
+  - test_file: jstests/core/clustered/clustered_collection_collation.js
+    ticket: SERVER-67105
+  - test_file: jstests/core/clustered/clustered_collection_bounded_scan.js
+    ticket: SERVER-67105
+  - test_file: jstests/noPassthrough/clustered_collection_sorted_scan.js
+    ticket: SERVER-76102
+  - test_file: jstests/replsets/session_cache_refresh_write_error_fail.js
+    ticket: SERVER-73229
+  - test_file: jstests/noPassthrough/ttl_operation_metrics_multi_dbs.js
+    ticket: SERVER-75277
+  - test_file: jstests/core/map_reduce_subplanning.js
+    ticket: SERVER-74131
+  - test_file: jstests/core/timeseries/timeseries_filter_extended_range.js
+    ticket: SERVER-69952
+  - test_file: jstests/sharding/refine_collection_shard_key_basic.js
+    ticket: SERVER-76394
   suites: null

etc/evergreen.yml

@@ -190,12 +190,12 @@ variables:
     distros:
     - windows-vsCurrent-large
   - name: burn_in_tests_gen
-  - name: .aggfuzzer .common
+  - name: .aggfuzzer .common !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: buildscripts_test
   - name: causally_consistent_jscore_txns_passthrough
-  - name: .encrypt !.aggregation !.gcm
+  - name: .encrypt !.aggregation !.gcm !.feature_flag_guarded
   - name: external_auth
   - name: external_auth_aws
   - name: external_auth_windows
@@ -251,7 +251,7 @@ variables:
   - name: compile_build_tools_next_TG
     distros:
     - rhel70
-  - name: .aggfuzzer
+  - name: .aggfuzzer !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: auth_gen
@@ -316,7 +316,7 @@ buildvariants:
     large_distro_name: rhel80-medium
     compile_variant: *linux_x86_dynamic_debug_compile_variant_name
   tasks:
-  - name: .aggregation !.encrypt
+  - name: .aggregation !.encrypt !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: .causally_consistent !.wo_snapshot
   - name: .change_streams !.secondary_reads
@@ -346,40 +346,6 @@ buildvariants:
   - name: sharding_gen
   - name: sharding_opportunistic_secondary_targeting_gen
 
-- <<: *linux-64-debug-required-template
-  name: &linux-64-debug-wtdevelop linux-64-debug-wtdevelop
-  display_name: "~ Linux DEBUG WiredTiger develop"
-  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
-  modules:
-  - wtdevelop
-  depends_on:
-  - name: archive_dist_test_debug
-    variant: &linux_x86_dynamic_debug_wtdevelop_compile_variant_name linux-x86-dynamic-debug-wtdevelop-compile
-  - name: version_gen
-    variant: generate-tasks-for-version
-    # This is added because of EVG-18211.
-    # Without this we are adding extra dependencies on evergreen and it is causing strain
-    omit_generated_tasks: true
-  expansions:
-    use_wt_develop: true
-    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
-    large_distro_name: rhel80-medium
-    max_sub_suites: "5"
-    target_resmoke_time: "15"
-    compile_flags: >-
-      --dbg=on
-      --opt=on
-      -j$(grep -c ^processor /proc/cpuinfo)
-      --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
-      --enable-free-mon=on
-      --enable-http-client=on
-      --link-model=dynamic
-    scons_cache_mode: all
-    scons_cache_scope: shared
-    num_scons_link_jobs_available: 0.99
-    test_flags: --excludeWithAnyTags=requires_http_client
-    compile_variant: *linux_x86_dynamic_debug_wtdevelop_compile_variant_name
-
 - name: &tla-plus tla-plus
   display_name: "TLA+"
   run_on:
@@ -430,8 +396,8 @@ buildvariants:
     distros:
     - rhel80-xlarge
   - name: test_api_version_compatibility
-  - name: .aggfuzzer !.multiversion
-  - name: .aggregation !.multiversion
+  - name: .aggfuzzer !.multiversion !.feature_flag_guarded
+  - name: .aggregation !.multiversion !.feature_flag_guarded
   - name: audit
   - name: .auth !.multiversion
   - name: .causally_consistent !.sharding
@@ -658,8 +624,8 @@ buildvariants:
       distros:
       - rhel80-xlarge
     - name: test_api_version_compatibility
-    - name: .aggfuzzer !.multiversion
-    - name: .aggregation !.multiversion
+    - name: .aggfuzzer !.multiversion !.feature_flag_guarded
+    - name: .aggregation !.multiversion !.feature_flag_guarded
     - name: audit
     - name: .auth !.multiversion
     - name: .causally_consistent !.sharding
@@ -805,7 +771,7 @@ buildvariants:
   - name: compile_build_tools_next_TG
     distros:
     - windows-vsCurrent-xlarge
-  - name: .aggregation !.auth !.encrypt
+  - name: .aggregation !.auth !.encrypt !.feature_flag_guarded
   - name: aggregation_expression_multiversion_fuzzer_gen
   - name: aggregation_expression_optimization_fuzzer_gen
   - name: auth_gen
@@ -1022,8 +988,8 @@ buildvariants:
   - name: compile_test_and_package_serial_TG
     distros:
     - windows-vsCurrent-large
-  - name: .aggfuzzer
-  - name: .aggregation !.auth !.encrypt !.unwind
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.auth !.encrypt !.unwind !.feature_flag_guarded
   - name: auth_gen
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -1036,7 +1002,6 @@ buildvariants:
   - name: disk_wiredtiger
   - name: .jscore .common !.auth
   - name: json_schema
-  - name: powercycle_gen
   - name: .query_fuzzer
   - name: .read_write_concern
   - name: replica_sets_gen
@@ -1116,7 +1081,7 @@ buildvariants:
   tasks:
   - name: compile_test_and_package_serial_TG
   - name: compile_build_tools_next_TG
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: auth_gen
   - name: causally_consistent_jscore_txns_passthrough
   - name: disk_wiredtiger
@@ -1129,7 +1094,6 @@ buildvariants:
   - name: .ssl
   - name: .stitch
   - name: unittest_shell_hang_analyzer_gen
-  - name: generate_buildid_to_debug_symbols_mapping
 
 - name: &enterprise-macos-rosetta-2 enterprise-macos-rosetta-2
   display_name: "Enterprise macOS Via Rosetta 2"
@@ -1329,8 +1293,8 @@ buildvariants:
     variant: generate-tasks-for-version
   tasks:
   - name: burn_in_tests_gen
-  - name: .aggfuzzer
-  - name: .aggregation
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
   - name: audit
   - name: .auth
   - name: unittest_shell_hang_analyzer_gen
@@ -1432,8 +1396,8 @@ buildvariants:
     distros:
     - rhel80-xlarge
   - name: test_api_version_compatibility
-  - name: .aggfuzzer !.no_debug_mode
-  - name: .aggregation !.no_debug_mode
+  - name: .aggfuzzer !.feature_flag_guarded !.no_debug_mode
+  - name: .aggregation !.feature_flag_guarded !.no_debug_mode
   - name: audit
   - name: .auth !.no_debug_mode
   - name: .causally_consistent !.sharding
@@ -1902,7 +1866,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel80-xlarge
-  - name: .aggfuzzer
+  - name: .aggfuzzer !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: auth_gen
@@ -2751,8 +2715,8 @@ buildvariants:
   # - name: test_api_version_compatibility # Not relevant for TSAN
   # - name: burn_in_tests_gen # No burn in tests needed
   - name: check_feature_flag_tags
-  # - name: .aggfuzzer # Not passing
-  # - name: .aggregation # Not passing
+  # - name: .aggfuzzer !.feature_flag_guarded # Not passing
+  # - name: .aggregation !.feature_flag_guarded # Not passing
   - name: audit
   # - name: .auth  # Not passing
   # - name: burn_in_tags_gen # No burn in tests needed
@@ -2859,8 +2823,8 @@ buildvariants:
     compile_variant: *enterprise-rhel80-debug-complete-tsan
   tasks:
   - name: compile_test_and_package_serial_TG
-  - name: .aggfuzzer
-  - name: .aggregation
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
   - name: .auth
   - name: resmoke_end2end_tests
   - name: unittest_shell_hang_analyzer_gen
@@ -3006,7 +2970,7 @@ buildvariants:
   - name: compile_test_and_package_parallel_dbtest_stream_TG
     distros:
     - rhel80-build
-  - name: .aggregation !.no_async
+  - name: .aggregation !.no_async !.feature_flag_guarded
   - name: .sharding .auth
   - name: .sharding .causally_consistent !.wo_snapshot
   - name: .concurrency .common !.kill_terminate
@@ -3048,7 +3012,7 @@ buildvariants:
   - name: compile_test_and_package_parallel_dbtest_stream_TG
     distros:
     - rhel80-build
-  - name: .aggregation !.no_async
+  - name: .aggregation !.no_async !.feature_flag_guarded
   - name: .sharding .auth
   - name: .sharding .causally_consistent !.wo_snapshot
   - name: .concurrency .common !.kill_terminate
@@ -3087,7 +3051,7 @@ buildvariants:
   - name: compile_test_and_package_parallel_dbtest_stream_TG
     distros:
     - rhel80-build
-  - name: .aggregation !.no_async
+  - name: .aggregation !.no_async !.feature_flag_guarded
   - name: .sharding .auth
   - name: .sharding .causally_consistent !.wo_snapshot
   - name: .concurrency .common !.kill_terminate
@@ -3131,7 +3095,7 @@ buildvariants:
   - name: compile_test_and_package_parallel_dbtest_stream_TG
     distros:
     - rhel80-build
-  - name: .aggregation !.no_async
+  - name: .aggregation !.no_async !.feature_flag_guarded
   - name: .sharding .auth
   - name: .sharding .causally_consistent !.wo_snapshot
   - name: .concurrency .common !.kill_terminate
@@ -3175,7 +3139,7 @@ buildvariants:
   - name: compile_test_and_package_parallel_dbtest_stream_TG
     distros:
     - rhel80-build
-  - name: .aggregation !.no_async
+  - name: .aggregation !.no_async !.feature_flag_guarded
   - name: .sharding .auth
   - name: .sharding .causally_consistent !.wo_snapshot
   - name: .concurrency .common !.kill_terminate
@@ -3212,7 +3176,7 @@ buildvariants:
     distros:
     - rhel80-build
     patch_only: true
-  - name: .aggregation !.no_async
+  - name: .aggregation !.no_async !.feature_flag_guarded
     patch_only: true
   - name: .sharding .auth
     patch_only: true
@@ -3439,18 +3403,6 @@ buildvariants:
     - windows-vsCurrent-large
   - name: .benchmarks !benchmarks_orphaned
 
-- <<: *enterprise-windows-nopush-template
-  name: &enterprise-windows-wtdevelop enterprise-windows-wtdevelop
-  display_name: "~ Enterprise Windows WiredTiger develop"
-  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
-  modules:
-  - enterprise
-  - wtdevelop
-  expansions:
-    <<: *enterprise-windows-nopush-expansions-template
-    use_wt_develop: true
-    compile_variant: *enterprise-windows-wtdevelop
-
 ### QO & QE Patch-Specific Build Variants ###
 - <<: *enterprise-windows-nopush-template
   name: &windows-compile-query-patch-only windows-compile-query-patch-only

etc/evergreen_nightly.yml

@@ -6,16 +6,362 @@ include:
 - filename: etc/evergreen_yml_components/variants/atlas.yml
 - filename: etc/evergreen_yml_components/variants/misc_release.yml
 ### Comment out when using this file for a LTS or Rapid release branch. ###
-- filename: etc/evergreen_yml_components/variants/ibm.yml
+# - filename: etc/evergreen_yml_components/variants/ibm.yml
 ### Uncomment when using this file for a LTS release branch. ###
 # - filename: etc/evergreen_yml_components/variants/in_memory.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
-# - filename: etc/evergreen_yml_components/variants/sanitizer.yml
+- filename: etc/evergreen_yml_components/variants/sanitizer.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
-# - filename: etc/evergreen_yml_components/variants/ninja.yml
+- filename: etc/evergreen_yml_components/variants/ninja.yml
+- filename: etc/evergreen_yml_components/variants/compile_static_analysis.yml
+### Uncomment when using this file for a LTS or Rapid release branch. ###
+- filename: etc/evergreen_yml_components/variants/classic_engine.yml
 
 
 parameters:
 - key: evergreen_config_file_path
   value: "etc/evergreen_nightly.yml"
   description: "path to this file"
+
+
+variables:
+
+# Common compile variant dependency specifications.
+- &linux_x86_dynamic_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_x86_dynamic_compile_variant_name linux-x86-dynamic-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+
+- &linux_x86_dynamic_debug_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_x86_dynamic_debug_compile_variant_name linux-x86-dynamic-debug-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+
+- &linux_debug_aubsan_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_debug_aubsan_compile_variant_name linux-debug-aubsan-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+
+- &windows_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &windows_compile_variant_name windows-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+
+- &linux_x86_generic_expansions
+  multiversion_platform: rhel80
+  multiversion_edition: enterprise
+  repo_edition: enterprise
+  large_distro_name: rhel80-medium
+  num_scons_link_jobs_available: 0.99
+  compile_variant: *linux_x86_dynamic_compile_variant_name
+
+# If you add anything to san_options, make sure the appropriate changes are
+# also made to SConstruct.
+# and also to the san_options in compile_static_analysis.yml
+- aubsan_options: &aubsan_options
+  >-
+    UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
+    LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1"
+    ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:strict_string_checks=true:detect_invalid_pointer_pairs=1:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
+
+
+buildvariants:
+
+- &linux-64-debug-required-template
+  <<: *linux_x86_dynamic_debug_compile_variant_dependency
+  name: &linux-64-debug-required linux-64-debug-required
+  display_name: "! Linux x86 Shared Library DEBUG"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  run_on:
+  - rhel80-medium
+  expansions:
+    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
+    test_flags: --excludeWithAnyTags=requires_http_client
+    target_resmoke_time: 15
+    max_sub_suites: 5
+    large_distro_name: rhel80-medium
+    compile_variant: *linux_x86_dynamic_debug_compile_variant_name
+  tasks:
+  - name: .aggregation !.encrypt !.feature_flag_guarded
+  - name: .auth !.audit !.multiversion
+  - name: .causally_consistent !.wo_snapshot
+  - name: .change_streams !.secondary_reads
+  - name: .clustered_collections
+  - name: .misc_js
+  - name: disk_wiredtiger
+  - name: free_monitoring
+  - name: .jscore .common
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: query_golden_classic
+  - name: libunwind_tests
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: .ocsp
+  - name: .read_write_concern
+  - name: .replica_sets !.encrypt !.ignore_non_generated_replica_sets_jscore_passthrough !.fcbis
+  - name: replica_sets_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+  - name: replica_sets_max_mirroring_large_txns_format_gen
+  - name: .retry
+  - name: .read_only
+  - name: session_jscore_passthrough
+  - name: sharded_multi_stmt_txn_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot
+  - name: sharding_gen
+  - name: sharding_opportunistic_secondary_targeting_gen
+
+- &enterprise-windows-template
+  <<: *windows_compile_variant_dependency
+  name: &enterprise-windows-required enterprise-windows-required
+  display_name: "! Enterprise Windows"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - windows-vsCurrent-small
+  expansions: &windows_required_expansions
+    compile_variant: *windows_compile_variant_name
+    exe: ".exe"
+    content_type: application/zip
+    python: '/cygdrive/c/python/python37/python.exe'
+    ext: zip
+    multiversion_platform: windows
+    multiversion_edition: enterprise
+    jstestfuzz_num_generated_files: 35
+    target_resmoke_time: 20
+    max_sub_suites: 5
+    large_distro_name: windows-vsCurrent-large
+    push_path: windows
+    push_bucket: downloads.10gen.com
+    push_name: windows
+    push_arch: x86_64-enterprise
+    test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls
+    external_auth_jobs_max: 1
+  tasks:
+  - name: burn_in_tests_gen
+  - name: audit
+  - name: auth_audit_gen
+  - name: buildscripts_test
+  - name: causally_consistent_jscore_txns_passthrough
+    distros:
+    - windows-vsCurrent-large
+  - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
+  - name: external_auth
+  - name: external_auth_aws
+  - name: external_auth_windows
+    distros:
+    - windows-2016-dc
+  - name: .jscore .common !.sharding
+  - name: jsCore_auth
+  - name: jsCore_ese
+  - name: jsCore_txns_large_txns_format
+  - name: .jstestfuzz .common
+  - name: mqlrun
+  - name: noPassthrough_gen
+  - name: noPassthroughWithMongod_gen
+  - name: .replica_sets .common !.ignore_non_generated_replica_sets_jscore_passthrough
+  - name: .replica_sets .multi_oplog !.ignore_non_generated_replica_sets_jscore_passthrough !.gcm
+  - name: replica_sets_jscore_passthrough_gen
+    distros:
+    - windows-vsCurrent-large
+  - name: sasl
+  - name: .sharding .txns
+  - name: sharding_auth_gen
+  - name: sharding_auth_audit_gen
+  - name: sharding_ese_gen
+
+- &enterprise-rhel-80-64-bit-dynamic-template
+  <<: *linux_x86_dynamic_compile_variant_dependency
+  name: &enterprise-rhel-80-64-bit-dynamic-required enterprise-rhel-80-64-bit-dynamic-required
+  display_name: "! Shared Library Enterprise RHEL 8.0"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-small
+  expansions: &enterprise-rhel-80-64-bit-dynamic-expansions
+    <<: *linux_x86_generic_expansions
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    has_packages: false
+    jstestfuzz_num_generated_files: 40
+    jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 10
+    max_sub_suites: 5
+    idle_timeout_factor: 1.5
+    exec_timeout_factor: 1.5
+    large_distro_name: rhel80-medium
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: *linux_x86_dynamic_compile_variant_name
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+  tasks:
+  - name: burn_in_tests_gen
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
+  - name: audit
+  - name: .auth
+  - name: buildscripts_test
+  - name: unittest_shell_hang_analyzer_gen
+  - name: .causally_consistent !.sharding
+  - name: .change_streams
+  - name: .change_stream_fuzzer
+  - name: .misc_js
+  - name: .concurrency !.large !.ubsan !.no_txns !.debug_only
+  - name: .concurrency .large !.ubsan !.no_txns !.debug_only
+    distros:
+    - rhel80-medium
+  - name: .config_fuzzer !.large
+  - name: .config_fuzzer .large
+    distros:
+    - rhel80-medium
+  - name: disk_wiredtiger
+  - name: .encrypt
+  - name: idl_tests
+  - name: initial_sync_fuzzer_gen
+  - name: jsCore
+    distros:
+    - rhel80-xlarge
+  - name: .jscore .common !jsCore
+  - name: jsCore_min_batch_repeat_queries_ese_gsm
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: .jstestfuzz !.flow_control  # Flow control jstestfuzz take longer.
+  - name: libunwind_tests
+  - name: mqlrun
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: multiversion_gen
+  - name: .query_fuzzer
+  - name: .random_multiversion_ds
+  - name: .read_write_concern .large
+    distros:
+    - rhel80-medium
+  - name: .read_write_concern !.large
+  - name: .replica_sets !.encrypt !.auth
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_api_version_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_reconfig_kill_primary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: retryable_writes_jscore_passthrough_gen
+  - name: retryable_writes_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-medium
+  - name: .read_only
+  - name: .rollbackfuzzer
+  - name: sasl
+  - name: search
+  - name: search_auth
+  - name: search_ssl
+  - name: session_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot !.multi_stmt
+  - name: sharding_api_version_jscore_passthrough_gen
+  - name: .sharding .txns
+  - name: .sharding .common
+  - name: .updatefuzzer
+  - name: secondary_reads_passthrough_gen
+  - name: .serverless
+    distros:
+    - rhel80-xlarge
+
+- &rhel80-debug-aubsan-lite-template
+  <<: *linux_debug_aubsan_compile_variant_dependency
+  name: &rhel80-debug-aubsan-lite-required rhel80-debug-aubsan-lite-required
+  display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-build
+  expansions: &aubsan-lite-required-expansions
+    compile_variant: *linux_debug_aubsan_compile_variant_name
+    lang_environment: LANG=C
+    san_options: *aubsan_options
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
+    resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under {A,UB}SAN build.
+    hang_analyzer_dump_core: false
+    max_sub_suites: 3
+    num_scons_link_jobs_available: 0.99
+    large_distro_name: rhel80-build
+  tasks:
+  - name: jsCore
+  - name: jsCore_txns
+
+- <<: *enterprise-rhel-80-64-bit-dynamic-template
+  name: &commit-queue commit-queue
+  display_name: "~ Commit Queue"
+  cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter
+  stepback: false
+  expansions:
+    <<: *linux_x86_generic_expansions
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    has_packages: false
+    compile_flags: >-
+      --ssl
+      MONGO_DISTMOD=rhel80
+      -j$(grep -c ^processor /proc/cpuinfo)
+      --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
+      --link-model=dynamic
+    crypt_task_compile_flags: >-
+      SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic
+      -Wl,--no-gnu-unique"
+      CCFLAGS="-fno-gnu-unique"
+    clang_tidy_toolchain: v4
+    num_scons_unit_cc_jobs_available: 0.2
+    compile_variant: *commit-queue
+  depends_on: []
+  tasks:
+  - name: compile_test_and_package_parallel_core_stream_TG
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: compile_test_and_package_parallel_unittest_stream_TG
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: compile_test_and_package_parallel_dbtest_stream_TG
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: jsCore
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: .lint
+  - name: test_api_version_compatibility
+  - name: validate_commit_message
+  - name: check_feature_flag_tags
+  - name: compile_venv_deps_check

etc/evergreen_timeouts.yml

@@ -10,131 +10,28 @@
 # in alphabetical order.
 
 overrides:
-  amazon-linux2-arm64-compile:
-    - task: server_discovery_and_monitoring_json_test
-      exec_timeout: 30  # 30 min.
-    - task: run_unittests
-      exec_timeout: 24
-
-  enterprise-amazon2-arm64:
-  - task: config_fuzzer_replica_sets_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
-
-  enterprise-amazon2022:
-  - task: config_fuzzer_replica_sets_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
-
-  enterprise-amazon2022-arm64:
-  - task: config_fuzzer_replica_sets_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
-
-  enterprise-rhel-70-64-bit:
-  - task: config_fuzzer_replica_sets_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
 
   enterprise-macos:
   - task: replica_sets_jscore_passthrough
     exec_timeout: 150  # 2.5 hours
-  - task: run_unittests
-    exec_timeout: 60
 
   enterprise-macos-arm64:
   - task: replica_sets_jscore_passthrough
     exec_timeout: 150  # 2.5 hours
-  - task: run_unittests
-    exec_timeout: 60
-
-  enterprise-macos-cxx20:
-  - task: run_unittests
-    exec_timeout: 60
-
-  enterprise-macos-rosetta-2:
-  - task: run_unittests
-    exec_timeout: 60
 
   enterprise-rhel-80-64-bit-coverage:
   - task: replica_sets_jscore_passthrough
     exec_timeout: 150  # 2.5 hours.
 
-  enterprise-rhel-80-64-bit-dynamic:
-  - task: replica_sets_large_txns_format
-    exec_timeout: 120  # 2 hours.
-  - task: config_fuzzer_replica_sets_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
-
-  enterprise-rhel80-debug-tsan:
-  - task: run_unittests
-    exec_timeout: 24
-
-  enterprise-rhel80-debug-complete-tsan:
-  - task: run_unittests
-    exec_timeout: 24
-
-  enterprise-rhel80-debug-complete-v4tsan-experimental:
-  - task: run_unittests
-    exec_timeout: 24
-
-  enterprise-rhel80-debug-v4tsan-experimental:
-  - task: run_unittests
-    exec_timeout: 24
-
-  enterprise-windows:
-  - task: replica_sets_jscore_passthrough
-    exec_timeout: 180  # 3 hours.
-
-  enterprise-windows-all-feature-flags-suggested:
-  - task: replica_sets_jscore_passthrough
-    exec_timeout: 180  # 3 hours.
-
-  linux-64-debug:
-  - task: auth
-    exec_timeout: 60  # 1 hour.
-
-  linux-x86-dynamic-compile-required:
-    - task: server_discovery_and_monitoring_json_test
-      exec_timeout: 30  # 30 min.
-
-  linux-64-debug-repeated-execution:
-  - task: run_unittests
-    exec_timeout: 120  # 2 hours.
-
   macos:
   - task: replica_sets_jscore_passthrough
     exec_timeout: 150  # 2.5 hours
-  - task: run_unittests
-    exec_timeout: 60
-  - task: sharded_collections_jscore_passthrough
-    exec_timeout: 300 # 5 hours
-
-  macos-debug-suggested:
-  - task: run_unittests
-    exec_timeout: 60
 
   macos-arm64:
-  - task: run_unittests
-    exec_timeout: 60
-  - task: sharding_update_v1_oplog_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours
-  - task: sharded_collections_jscore_passthrough
-    exec_timeout: 300 # 5 hours
   - task: replica_sets_jscore_passthrough
     exec_timeout: 150  # 2.5 hours
 
-  rhel80-asan:
-  - task: run_unittests
-    exec_timeout: 24
-
-  rhel80-debug-asan:
-  - task: run_unittests
-    exec_timeout: 24
-
-  rhel80-debug-aubsan-lite:
-  - task: run_unittests
-    exec_timeout: 24
-
   rhel80-debug-ubsan:
-  - task: run_unittests
-    exec_timeout: 24
   - task: update_timeseries_fuzzer
     exec_timeout: 150  # 2.5 hours
 
@@ -142,22 +39,6 @@ overrides:
   - task: replica_sets_jscore_passthrough
     exec_timeout: 180  # 3 hours.
 
-  rhel80-ubsan:
-  - task: run_unittests
-    exec_timeout: 24
-
-  windows:
-  - task: replica_sets
-    exec_timeout: 180  # 3 hours.
-
-  windows-debug-suggested:
-  - task: replica_sets_initsync_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
-  - task: replica_sets_jscore_passthrough
-    exec_timeout: 180  # 3 hours.
-  - task: run_unittests
-    exec_timeout: 30  # 30 minutes.
-
   rhel80-debug-ubsan-all-feature-flags:
   - task: update_timeseries_fuzzer
     exec_timeout: 150  # 2.5 hours

etc/evergreen_yml_components/definitions.yml

@@ -66,7 +66,7 @@ modules:
 - name: enterprise
   repo: git@github.com:10gen/mongo-enterprise-modules.git
   prefix: src/mongo/db/modules
-  branch: master
+  branch: v6.3
 
 - name: wtdevelop
   repo: git@github.com:wiredtiger/wiredtiger.git
@@ -616,7 +616,7 @@ functions:
   "get buildnumber": &get_buildnumber
     command: keyval.inc
     params:
-      key: "${build_variant}_master"
+      key: "${build_variant}_v6.3"
       destination: "builder_num"
 
   "run diskstats": &run_diskstats
@@ -3876,7 +3876,7 @@ tasks:
 ## Standalone fuzzer for checking columnstore index correctness ##
 - <<: *jstestfuzz_template
   name: aggregation_columnstore_fuzzer_gen
-  tags: ["aggfuzzer", "common", "columnstore", "require_npm", "random_name", "sbe_only"]
+  tags: ["aggfuzzer", "common", "columnstore", "require_npm", "random_name", "sbe_only", "feature_flag_guarded"]
   commands:
   - func: "generate resmoke tasks"
     vars:
@@ -4691,7 +4691,9 @@ tasks:
 
 - <<: *task_template
   name: aggregation_column_store_index_passthrough
-  tags: ["aggregation", "sbe_only"]
+  # TODO SERVER-75026 Remove the 'feature_flag_guarded' tag and ensure this suite is run on all
+  # variants.
+  tags: ["aggregation", "sbe_only", "feature_flag_guarded"]
   depends_on:
   - name: aggregation
   commands:
@@ -5515,9 +5517,11 @@ tasks:
 
 - <<: *task_template
   name: jsCore_column_store_indexes
+  # TODO SERVER-75026 Remove the 'feature_flag_guarded' tag and ensure this suite is run on all
+  # variants.
+  tags: ["jscore", "sbe_only", "feature_flag_guarded"]
   depends_on:
   - name: jsCore
-  tags: ["jscore", "sbe_only"]
   commands:
   - func: "do setup"
   - func: "run tests"
@@ -5609,12 +5613,11 @@ tasks:
     vars:
       resmoke_jobs_max: 1
 
-- <<: *task_template
-  name: sharded_collections_jscore_passthrough
+- <<: *gen_task_template
+  name: sharded_collections_jscore_passthrough_gen
   tags: ["sharding", "jscore"]
   commands:
-  - func: "do setup"
-  - func: "run tests"
+  - func: "generate resmoke tasks"
 
 - <<: *gen_task_template
   name: sharded_collections_jscore_multiversion_gen
@@ -7302,7 +7305,7 @@ tasks:
       - "./src/evergreen/packages_publish.sh"
 
 - name: push
-  run_on: rhel80-small
+  run_on: rhel8.7-small
   tags: ["publish"]
   patchable: false
   depends_on:
@@ -7356,13 +7359,28 @@ tasks:
       aws_key_remote: ${repo_aws_key}
       aws_secret_remote: ${repo_aws_secret}
   - func: "f_expansions_write"
-  - func: "set up notary client credentials"
+
+  # login to container registry
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |
+        set -oe
+        podman login --username ${release_tools_container_registry_username} --password ${release_tools_container_registry_password} ${release_tools_container_registry}
+
+  # signing windows artifacts
   - command: subprocess.exec
-    type: test
     params:
       binary: bash
       args:
-      - "./src/evergreen/notary_client_run.sh"
+      - "./src/evergreen/garasign_jsign_sign.sh"
+
+  # signing linux artifacts
+  - command: subprocess.exec
+    params:
+      binary: bash
+      args:
+      - "./src/evergreen/garasign_gpg_sign.sh"
 
   # Put the binaries tarball/zipfile
   - command: s3.put
@@ -7475,7 +7493,7 @@ tasks:
       aws_secret: ${aws_secret}
       permissions: public-read
       build_variants: ["enterprise-windows", "windows"]
-      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi
+      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi
       bucket: build-push-testing
       content_type: application/x-msi
       remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi
@@ -7534,7 +7552,7 @@ tasks:
       aws_secret: ${aws_secret}
       permissions: public-read
       build_variants: ["enterprise-windows", "windows"]
-      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1
+      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha1
       bucket: build-push-testing
       content_type: text/plain
       remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha1
@@ -7592,7 +7610,7 @@ tasks:
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
       build_variants: ["enterprise-windows", "windows"]
-      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256
+      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha256
       bucket: build-push-testing
       permissions: public-read
       remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha256
@@ -7651,7 +7669,7 @@ tasks:
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
       build_variants: ["enterprise-windows", "windows"]
-      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5
+      local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.md5
       bucket: build-push-testing
       permissions: public-read
       content_type: text/plain
@@ -7782,7 +7800,7 @@ tasks:
          'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}}
 
 - name: crypt_push
-  run_on: rhel80-small
+  run_on: rhel8.7-small
   tags: ["publish_crypt"]
   patchable: false
   stepback: false
@@ -7819,13 +7837,18 @@ tasks:
       aws_key_remote: ${repo_aws_key}
       aws_secret_remote: ${repo_aws_secret}
   - func: "f_expansions_write"
-  - func: "set up notary client credentials"
+  # login to container registry
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |
+        set -oe
+        podman login --username ${release_tools_container_registry_username} --password ${release_tools_container_registry_password} ${release_tools_container_registry}
   - command: subprocess.exec
-    type: test
     params:
       binary: bash
       args:
-      - "./src/evergreen/notary_client_crypt_run.sh"
+      - "./src/evergreen/garasign_gpg_crypt_sign.sh"
   # Put the crypt tarball/zipfile
   - command: s3.put
     params:

etc/evergreen_yml_components/variants/atlas.yml

@@ -41,7 +41,7 @@ buildvariants:
   - name: compile_build_tools_next_TG
     distros:
     - rhel70
-  - name: .aggfuzzer
+  - name: .aggfuzzer !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: auth_gen
@@ -50,7 +50,7 @@ buildvariants:
   - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .common
   - name: libunwind_tests

etc/evergreen_yml_components/variants/classic_engine.yml

@@ -0,0 +1,298 @@
+# Build variants for testing the classic engine.
+
+variables:
+- &linux_x86_dynamic_compile_variant_dependency
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: &linux_x86_dynamic_compile_variant_name linux-x86-dynamic-compile-required
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+
+- &linux_x86_generic_expansions
+  multiversion_platform: rhel80
+  multiversion_edition: enterprise
+  repo_edition: enterprise
+  large_distro_name: rhel80-medium
+  num_scons_link_jobs_available: 0.99
+  compile_variant: *linux_x86_dynamic_compile_variant_name
+
+- &enterprise-rhel-80-64-bit-dynamic-expansions
+    <<: *linux_x86_generic_expansions
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    has_packages: false
+    jstestfuzz_num_generated_files: 40
+    jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 10
+    max_sub_suites: 5
+    idle_timeout_factor: 1.5
+    exec_timeout_factor: 1.5
+    large_distro_name: rhel80-medium
+
+# If you add anything to san_options, make sure the appropriate changes are
+# also made to SConstruct.
+# and also to the san_options in compile_static_analysis.yml
+- aubsan_options: &aubsan_options
+  >-
+    UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
+    LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1"
+    ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:strict_string_checks=true:detect_invalid_pointer_pairs=1:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
+
+buildvariants:
+- &enterprise-rhel-80-64-bit-dynamic-classic-engine
+  <<: *linux_x86_dynamic_compile_variant_dependency
+  name: enterprise-rhel-80-64-bit-dynamic-classic-engine
+  display_name: "Shared Library Enterprise RHEL 8.0 (Classic Engine)"
+  cron: "0 0 * * 0"  # once a week (Sunday midnight UTC)
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-small
+  stepback: false
+  expansions:
+    <<: *enterprise-rhel-80-64-bit-dynamic-expansions
+    jstestfuzz_num_generated_files: 40
+    jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 10
+    max_sub_suites: 5
+    test_flags: >-
+      --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
+    large_distro_name: rhel80-medium
+  depends_on:
+  - name: archive_dist_test_debug
+    variant: *linux_x86_dynamic_compile_variant_name
+  - name: version_gen
+    variant: generate-tasks-for-version
+    # This is added because of EVG-18211.
+    # Without this we are adding extra dependencies on evergreen and it is causing strain
+    omit_generated_tasks: true
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+  tasks:
+  - name: .aggfuzzer !.sbe_only
+  - name: .aggregation !.sbe_only
+  - name: .auth
+  - name: .causally_consistent !.sharding
+  - name: .change_stream_fuzzer
+  - name: .change_streams
+  - name: .concurrency !.large !.ubsan !.no_txns !.debug_only !.compute_mode
+  - name: .concurrency .large !.ubsan !.no_txns !.debug_only !.compute_mode
+    distros:
+    - rhel80-medium
+  - name: .encrypt
+  - name: .jscore .common !jsCore !.sbe_only
+  - name: .jstestfuzz !.flow_control
+  - name: .misc_js
+  - name: .multi_shard
+  - name: .query_fuzzer
+  - name: .random_multiversion_ds
+  - name: .read_only
+  - name: .read_write_concern !.large
+  - name: .read_write_concern .large
+    distros:
+    - rhel80-medium
+  - name: .replica_sets !.encrypt !.auth
+    distros:
+    - rhel80-xlarge
+  - name: .rollbackfuzzer
+  - name: .sharding .common
+  - name: .sharding .jscore !.wo_snapshot !.multi_stmt
+  - name: .sharding .txns
+  - name: .serverless
+    distros:
+    - rhel80-xlarge
+  - name: .updatefuzzer
+  - name: audit
+  - name: burn_in_tests_gen
+  - name: check_feature_flag_tags
+  - name: check_for_todos
+  - name: disk_wiredtiger
+  - name: initial_sync_fuzzer_gen
+  - name: jsCore
+    distros:
+    - rhel80-xlarge
+  - name: jsCore_min_batch_repeat_queries_ese_gsm
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: lint_fuzzer_sanity_patch
+  - name: mqlrun
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: multiversion_gen
+  - name: .multiversion_sanity_check
+  - name: replica_sets_api_version_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_reconfig_kill_primary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: retryable_writes_jscore_passthrough_gen
+  - name: retryable_writes_jscore_stepdown_passthrough_gen
+  - name: sasl
+  - name: search
+  - name: search_auth
+  - name: search_ssl
+  - name: secondary_reads_passthrough_gen
+  - name: session_jscore_passthrough
+  - name: sharding_api_version_jscore_passthrough_gen
+  - name: test_api_version_compatibility
+  - name: unittest_shell_hang_analyzer_gen
+
+- name: &rhel80-debug-asan-classic-engine rhel80-debug-asan-classic-engine
+  display_name: "* ASAN Enterprise RHEL 8.0 DEBUG (Classic Engine)"
+  cron: "0 0 * * 2"  # once a week (Tuesday midnight UTC)
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-build
+  stepback: false
+  expansions:
+    additional_package_targets: >-
+      archive-mongocryptd
+      archive-mongocryptd-debug
+    lang_environment: LANG=C
+    san_options: *aubsan_options
+    compile_flags: >-
+      --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars
+      --dbg=on
+      --opt=on
+      --allocator=system
+      --sanitize=address
+      --ssl
+      --ocsp-stapling=off
+      --enable-free-mon=on
+      -j$(grep -c ^processor /proc/cpuinfo)
+    compile_variant: *rhel80-debug-asan-classic-engine
+    test_flags: >-
+      --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
+      --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling
+    multiversion_platform: rhel80
+    multiversion_edition: enterprise
+    resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under ASAN build.
+    hang_analyzer_dump_core: false
+    scons_cache_scope: shared
+    exec_timeout_secs: 14400 # 3 hour timeout
+    separate_debug: off
+    large_distro_name: rhel80-build
+  tasks:
+  - name: compile_test_benchmark_and_package_serial_TG
+  - name: .aggregation !.sbe_only
+  - name: .auth
+  - name: audit
+  - name: .benchmarks
+  - name: .causally_consistent !.wo_snapshot
+  - name: .change_streams
+  - name: .misc_js
+  - name: .concurrency !.ubsan !.no_txns !.kill_terminate !.compute_mode
+  - name: .encrypt
+  - name: free_monitoring
+  - name: external_auth
+  - name: external_auth_aws
+  - name: initial_sync_fuzzer_gen
+  - name: .integration !.standalone !.audit
+  - name: .jscore .common !.sbe_only
+  - name: jsCore_min_batch_repeat_queries_ese_gsm
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: .logical_session_cache
+  - name: .multi_shard .common
+  - name: .query_fuzzer
+  - name: .read_write_concern
+  - name: replica_sets_large_txns_format_jscore_passthrough
+  - name: .replica_sets !.multi_oplog
+  - name: .replica_sets .encrypt
+  - name: .resharding_fuzzer
+  - name: .retry
+  - name: .read_only
+  - name: .rollbackfuzzer
+  - name: .updatefuzzer
+  - name: sasl
+  - name: secondary_reads_passthrough_gen
+  - name: session_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot
+  - name: .sharding .common !.csrs !.jstestfuzz
+  - name: .watchdog
+  - name: .stitch
+  - name: .serverless
+  - name: unittest_shell_hang_analyzer_gen
+  - name: .updatefuzzer
+  - name: server_discovery_and_monitoring_json_test_TG
+  - name: server_selection_json_test_TG
+
+- name: &rhel80-debug-ubsan-classic-engine rhel80-debug-ubsan-classic-engine
+  display_name: "* UBSAN Enterprise RHEL 8.0 DEBUG (Classic Engine)"
+  cron: "0 0 * * 4"  # once a week (Thursday midnight UTC)
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-build
+  stepback: false
+  expansions:
+    additional_package_targets: >-
+      archive-mongocryptd
+      archive-mongocryptd-debug
+    lang_environment: LANG=C
+    san_options: *aubsan_options
+    compile_variant: *rhel80-debug-ubsan-classic-engine
+    compile_flags: >-
+      --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars
+      --dbg=on
+      --opt=on
+      --sanitize=undefined
+      --ssl
+      --ocsp-stapling=off
+      --enable-free-mon=on
+      -j$(grep -c ^processor /proc/cpuinfo)
+    test_flags: >-
+      --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
+      --excludeWithAnyTags=requires_ocsp_stapling
+    multiversion_platform: rhel80
+    multiversion_edition: enterprise
+    resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under UBSAN build.
+    scons_cache_scope: shared
+    separate_debug: off
+    large_distro_name: rhel80-build
+  tasks:
+  - name: compile_test_benchmark_and_package_serial_TG
+  - name: .aggregation !.sbe_only
+  - name: .auth
+  - name: audit
+  - name: .benchmarks
+  - name: .causally_consistent !.wo_snapshot
+  - name: .change_streams
+  - name: .misc_js
+  - name: .concurrency !.no_txns !.repl !.kill_terminate !.compute_mode
+  - name: disk_wiredtiger
+  - name: .encrypt
+  - name: free_monitoring
+  - name: initial_sync_fuzzer_gen
+  - name: .integration !.audit
+  - name: .jscore .common !.sbe_only
+  - name: jsCore_min_batch_repeat_queries_ese_gsm
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: .logical_session_cache .one_sec
+  - name: .multi_shard .common
+  - name: .read_write_concern
+  - name: replica_sets_large_txns_format_jscore_passthrough
+  - name: .replica_sets !.multi_oplog
+  - name: .replica_sets .encrypt
+  - name: .resharding_fuzzer
+  - name: .retry
+  - name: .rollbackfuzzer
+  - name: .read_only
+  - name: sasl
+  - name: secondary_reads_passthrough_gen
+  - name: session_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot
+  - name: .sharding .common !.csrs !.jstestfuzz
+  - name: .stitch
+  - name: .updatefuzzer
+  - name: .serverless
+  - name: watchdog_wiredtiger
+  - name: server_discovery_and_monitoring_json_test_TG
+  - name: server_selection_json_test_TG

etc/evergreen_yml_components/variants/in_memory.yml

@@ -29,8 +29,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel80-xlarge
-  - name: .aggfuzzer .common
-  - name: .aggregation !.unwind !.encrypt
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: .aggregation !.unwind !.encrypt !.feature_flag_guarded
   - name: audit
   - name: .auth !.multiversion
   - name: .causally_consistent !.wo_snapshot !.durable_history
@@ -108,7 +108,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - windows-vsCurrent-large
-  - name: .aggfuzzer .common
+  - name: .aggfuzzer .common !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: causally_consistent_jscore_txns_passthrough

etc/evergreen_yml_components/variants/misc_release.yml

@@ -32,8 +32,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - amazon2-build
-  - name: .aggfuzzer .common
-  - name: aggregation
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -98,8 +98,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - amazon2-build
-  - name: .aggfuzzer .common
-  - name: aggregation
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: audit
   - name: .auth !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
@@ -153,7 +153,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - amazon2-arm64-large
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -220,8 +220,8 @@ buildvariants:
     distros:
     - amazon2-arm64-large
   - name: test_api_version_compatibility
-  - name: .aggfuzzer
-  - name: .aggregation
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
   - name: audit
   - name: .auth
   - name: .causally_consistent !.sharding
@@ -246,7 +246,7 @@ buildvariants:
   - name: jsCore
     distros:
     - amazon2-arm64-large
-  - name: .jscore .common !jsCore
+  - name: .jscore .common !jsCore !.feature_flag_guarded
   - name: jsCore_min_batch_repeat_queries_ese_gsm
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -386,8 +386,8 @@ buildvariants:
     distros:
     - amazon2022-large
   - name: test_api_version_compatibility
-  - name: .aggfuzzer !.multiversion
-  - name: .aggregation
+  - name: .aggfuzzer !.multiversion !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
   - name: audit
   - name: .auth !.multiversion
   - name: .causally_consistent !.sharding
@@ -448,8 +448,6 @@ buildvariants:
     distros:
     - amazon2022-large
   - name: .publish_crypt
-    distros:
-    - amazon2022-large
   - name: secondary_reads_passthrough_gen
   - name: server_discovery_and_monitoring_json_test_TG
   - name: .serverless
@@ -488,7 +486,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - amazon2022-arm64-large
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -551,8 +549,8 @@ buildvariants:
     distros:
     - amazon2022-arm64-large
   - name: test_api_version_compatibility
-  - name: .aggfuzzer !.multiversion
-  - name: .aggregation
+  - name: .aggfuzzer !.feature_flag_guarded !.multiversion
+  - name: .aggregation !.feature_flag_guarded
   - name: audit
   - name: .auth !.multiversion
   - name: .causally_consistent !.sharding
@@ -577,7 +575,7 @@ buildvariants:
   - name: jsCore
     distros:
     - amazon2022-arm64-large
-  - name: .jscore .common !jsCore
+  - name: .jscore .common !jsCore !.feature_flag_guarded
   - name: jsCore_min_batch_repeat_queries_ese_gsm
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -659,8 +657,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - debian10-build
-  - name: .aggfuzzer .common
-  - name: aggregation
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: aggregation_auth
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
@@ -669,7 +667,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: multiversion_gen
@@ -725,14 +723,14 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - debian10-build
-  - name: .aggfuzzer .common
+  - name: .aggfuzzer .common !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
   - name: .jepsen_docker
     distros:
     - debian10-large
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -777,8 +775,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - debian11-large
-  - name: .aggfuzzer .common
-  - name: aggregation
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: aggregation_auth
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
@@ -787,7 +785,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: multiversion_gen
@@ -841,14 +839,11 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - debian11-large
-  - name: .aggfuzzer .common
+  - name: .aggfuzzer .common !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
-  - name: .jepsen_docker
-    distros:
-    - debian11-large
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -892,7 +887,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel70
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -993,7 +988,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel80-build
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -1061,8 +1056,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel80-large
-  - name: .aggfuzzer
-  - name: .aggregation
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
   - name: audit
   - name: .auth
   - name: unittest_shell_hang_analyzer_gen
@@ -1170,7 +1165,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel82-arm64-large
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -1231,7 +1226,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel82-arm64-large
-  - name: .aggfuzzer !.multiversion
+  - name: .aggfuzzer !.multiversion !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: auth_gen
@@ -1239,7 +1234,7 @@ buildvariants:
   - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
@@ -1288,7 +1283,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel90-build
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -1345,7 +1340,7 @@ buildvariants:
   - name: compile_build_tools_next_TG
     distros:
     - rhel90-large
-  - name: .aggfuzzer !.multiversion
+  - name: .aggfuzzer !.feature_flag_guarded !.multiversion
   - name: audit
   - name: auth_audit_gen
   - name: auth_gen
@@ -1434,8 +1429,6 @@ buildvariants:
     distros:
     - ubuntu2204-arm64-small
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-rhel90-arm64
@@ -1475,7 +1468,7 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - rhel90-arm64-large
-  - name: .aggfuzzer !.multiversion
+  - name: .aggfuzzer !.multiversion !.feature_flag_guarded
   - name: audit
   - name: auth_audit_gen
   - name: auth_gen
@@ -1505,8 +1498,6 @@ buildvariants:
     distros:
     - ubuntu2204-arm64-small
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: suse12
@@ -1542,8 +1533,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - suse12-sp5-large
-  - name: .aggfuzzer .common
-  - name: aggregation
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -1554,7 +1545,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: multiversion_gen
   - name: .logical_session_cache .one_sec
@@ -1610,11 +1601,11 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - suse12-sp5-large
-  - name: .aggfuzzer .common
+  - name: .aggfuzzer .common !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -1666,11 +1657,11 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - suse15-build
-  - name: .aggfuzzer .common !.multiversion
+  - name: .aggfuzzer .common !.multiversion !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -1715,8 +1706,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - suse15-build
-  - name: .aggfuzzer .common !.multiversion
-  - name: aggregation
+  - name: .aggfuzzer .common !.multiversion !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -1724,7 +1715,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: multiversion_gen
@@ -1772,8 +1763,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - ubuntu1804-build
-  - name: .aggfuzzer .common
-  - name: aggregation
+  - name: .aggfuzzer .common !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: .misc_js
   - name: .concurrency .common
@@ -1856,7 +1847,7 @@ buildvariants:
     - ubuntu1804-build
   - name: .crypt
   - name: .publish_crypt
-  - name: .aggfuzzer .common
+  - name: .aggfuzzer .common !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
@@ -1865,7 +1856,7 @@ buildvariants:
   - name: .jepsen
     distros:
     - ubuntu1804-build
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_auth
   - name: .jstestfuzz .common
   - name: libunwind_tests
@@ -1924,7 +1915,7 @@ buildvariants:
     compile_variant: enterprise-ubuntu1804-arm64
   tasks:
   - name: compile_test_and_package_serial_no_unittests_TG
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: aggregation_wildcard_fuzzer_gen
   - name: .auth !.audit !.multiversion !.jscore
   - name: causally_consistent_jscore_txns_passthrough
@@ -1932,7 +1923,7 @@ buildvariants:
   - name: .concurrency .common
   - name: concurrency_replication_causal_consistency_gen
   - name: fle
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_gen
@@ -2018,8 +2009,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - ubuntu2204-large
-  - name: .aggfuzzer .common !.multiversion
-  - name: aggregation
+  - name: .aggfuzzer .common !.multiversion !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: .misc_js
   - name: .concurrency .common
@@ -2073,8 +2064,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - ubuntu2004-large
-  - name: .aggfuzzer .common !.multiversion
-  - name: aggregation
+  - name: .aggfuzzer .common !.multiversion !.feature_flag_guarded
+  - name: aggregation !.feature_flag_guarded
   - name: .auth !.audit !.multiversion
   - name: .misc_js
   - name: .concurrency .common
@@ -2142,13 +2133,13 @@ buildvariants:
     - ubuntu2004-large
   - name: .crypt
   - name: .publish_crypt
-  - name: .aggfuzzer .common !.multiversion
+  - name: .aggfuzzer .common !.multiversion !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_auth
   - name: .jstestfuzz .common
   - name: libunwind_tests
@@ -2208,13 +2199,13 @@ buildvariants:
     - ubuntu2204-large
   - name: .crypt
   - name: .publish_crypt
-  - name: .aggfuzzer .common !.multiversion
+  - name: .aggfuzzer .common !.multiversion !.feature_flag_guarded
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_auth
   - name: .jstestfuzz .common
   - name: libunwind_tests
@@ -2268,7 +2259,7 @@ buildvariants:
     compile_variant: enterprise-ubuntu2004-arm64
   tasks:
   - name: compile_test_and_package_serial_no_unittests_TG
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: aggregation_wildcard_fuzzer_gen
   - name: .auth !.audit !.multiversion !.jscore
   - name: causally_consistent_jscore_txns_passthrough
@@ -2276,7 +2267,7 @@ buildvariants:
   - name: .concurrency .common
   - name: concurrency_replication_causal_consistency_gen
   - name: fle
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_gen
@@ -2360,7 +2351,7 @@ buildvariants:
     compile_variant: enterprise-ubuntu2204-arm64
   tasks:
   - name: compile_test_and_package_serial_no_unittests_TG
-  - name: aggregation
+  - name: aggregation !.feature_flag_guarded
   - name: aggregation_wildcard_fuzzer_gen
   - name: .auth !.audit !.multiversion !.jscore
   - name: causally_consistent_jscore_txns_passthrough
@@ -2368,7 +2359,7 @@ buildvariants:
   - name: .concurrency .common
   - name: concurrency_replication_causal_consistency_gen
   - name: fle
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_gen
@@ -2455,8 +2446,8 @@ buildvariants:
   - name: compile_test_and_package_serial_no_unittests_TG
     distros:
     - windows-vsCurrent-large
-  - name: .aggfuzzer
-  - name: .aggregation !.auth !.encrypt !.unwind
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.auth !.encrypt !.unwind !.feature_flag_guarded
   - name: auth_gen
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -2468,7 +2459,7 @@ buildvariants:
   - name: .concurrency .common
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: json_schema
   - name: .jstestfuzz !.initsync !.flow_control !.stepdowns
   - name: multiversion_gen
@@ -2529,8 +2520,8 @@ buildvariants:
   - name: compile_test_and_package_serial_TG
     distros:
     - windows-vsCurrent-large
-  - name: .aggfuzzer
-  - name: .aggregation !.auth !.encrypt !.unwind
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.auth !.encrypt !.unwind !.feature_flag_guarded
   - name: auth_gen
   - name: causally_consistent_jscore_txns_passthrough
   - name: .misc_js
@@ -2545,12 +2536,11 @@ buildvariants:
     - windows-vsCurrent-large
   - name: .publish_crypt
   - name: disk_wiredtiger
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: json_schema
   - name: .jstestfuzz !.initsync !.flow_control !.stepdowns
   - name: multiversion_gen
   - name: multiversion_auth_gen
-  - name: powercycle_gen
   - name: .query_fuzzer
   - name: .read_write_concern
   - name: replica_sets_gen
@@ -2588,7 +2578,7 @@ buildvariants:
   tasks:
   - name: compile_test_and_package_serial_TG
   - name: compile_build_tools_next_TG
-  - name: .aggregation !.auth !.encrypt !.unwind
+  - name: .aggregation !.auth !.encrypt !.unwind !.feature_flag_guarded
   - name: auth_gen
   - name: .causally_consistent !.sharding
   - name: .change_streams !.secondary_reads
@@ -2597,7 +2587,7 @@ buildvariants:
   - name: disk_wiredtiger
   - name: free_monitoring
   - name: initial_sync_fuzzer_gen
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .causal
   - name: .jstestfuzz .interrupt
@@ -2642,7 +2632,7 @@ buildvariants:
   tasks:
   - name: compile_test_and_package_serial_TG
   - name: compile_build_tools_next_TG
-  - name: .aggregation !.auth !.encrypt !.unwind
+  - name: .aggregation !.auth !.encrypt !.unwind !.feature_flag_guarded
   - name: auth_gen
   - name: .causally_consistent !.sharding
   - name: .change_streams !.secondary_reads
@@ -2651,7 +2641,7 @@ buildvariants:
   - name: disk_wiredtiger
   - name: free_monitoring
   - name: initial_sync_fuzzer_gen
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .causal
   - name: .jstestfuzz .interrupt
@@ -2708,7 +2698,7 @@ buildvariants:
   - name: causally_consistent_jscore_txns_passthrough
   # TODO: SERVER-66945 Re-enable ESE on enterprise macos
   # - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: mqlrun
@@ -2752,7 +2742,7 @@ buildvariants:
   - name: auth_audit_gen
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: mqlrun

etc/evergreen_yml_components/variants/sanitizer.yml

@@ -32,7 +32,7 @@ buildvariants:
     compile_variant: rhel80-asan
   tasks:
     - name: compile_test_and_package_serial_TG
-    - name: .aggfuzzer .common
+    - name: .aggfuzzer .common !.feature_flag_guarded
     - name: free_monitoring
     - name: .jstestfuzz !.initsync
 
@@ -76,7 +76,7 @@ buildvariants:
     compile_variant: rhel80-debug-asan
   tasks:
   - name: compile_test_benchmark_and_package_serial_TG
-  - name: .aggregation
+  - name: .aggregation !.feature_flag_guarded
   - name: .auth
   - name: audit
   - name: .benchmarks
@@ -158,7 +158,7 @@ buildvariants:
     compile_variant: rhel80-debug-ubsan
   tasks:
   - name: compile_test_benchmark_and_package_serial_TG
-  - name: .aggregation
+  - name: .aggregation !.feature_flag_guarded
   - name: .auth
   - name: audit
   - name: .benchmarks

etc/evergreen_yml_components/variants/task_generation.yml

@@ -11,5 +11,9 @@ buildvariants:
   - rhel80-medium
   tasks:
   - name: version_gen
+    distros:
+    - ubuntu2004-medium
   - name: version_burn_in_gen
+    distros:
+    - ubuntu2004-medium
   - name: version_expansions_gen

etc/perf.yml

@@ -1,882 +0,0 @@
-command_type: system
-stepback: false
-
-## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
-parameters:
-  - key: patch_compile_flags
-    description: "Additional SCons flags to be applied during scons compile invocations in this patch"
-
-variables:
-  ###
-  # Leave this section uncommented to enable compile.
-  _real_remote_file: &_remote_file
-      ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}.tar.gz
-  _real_compile: &_compile
-    - variant: linux-wt-standalone
-      name: compile
-  _real_expansions: &_expansion_updates
-      []
-  ###
-
-  ###
-  # **Or**: Leave this section uncommented to bypass/skip compile.
-  # This file ↓ came from a microbenchmarks waterfall run.
-  # https://evergreen.mongodb.com/version/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551
-  # Artifacts eventually expire. If this fails, grab the compile artifacts url and update this.
-#  _skip_remote_file: &_remote_file
-#      /perf/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551/c0a8cbe58cc46253a94130d2cb64cdd8089b3551/linux/mongodb-performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551.tar.gz
-#  _skip_compile: &_compile
-#      []
-#  _skip_expansions: &_expansion_updates
-#    - key: mdb_binary_for_server
-#      value: https://mciuploads.s3.amazonaws.com/perf/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551/c0a8cbe58cc46253a94130d2cb64cdd8089b3551/linux/mongodb-performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551.tar.gz
-#    - key: mdb_binary_for_client
-#      value: https://mciuploads.s3.amazonaws.com/perf/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551/c0a8cbe58cc46253a94130d2cb64cdd8089b3551/linux/mongodb-performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551.tar.gz
-  ###
-
-  _src_dir: &src_dir src/mongo
-  _modules: &modules
-    - enterprise
-    - mongo-tools
-    - dsi
-    - genny
-    - workloads
-    - linkbench
-    - linkbench2
-    - mongo-perf
-    - YCSB
-    - benchmarks
-    - py-tpcc
-
-modules:
-  ###
-  # Same in every DSI project. Ensure that this block is synchronized with
-  # evergreen-dsitest.yml, atlas/system_perf_atlas.yml, and src/dsi/onboarding.py
-  # (search update-repos-here) in this repo, and etc/system_perf.yml and
-  # etc/perf.yml in mongodb/mongo
-  - name: dsi
-    repo: git@github.com:10gen/dsi.git
-    prefix: ${workdir}/src
-    branch: microbenchmarks-stable
-  - name: genny
-    repo: git@github.com:10gen/genny.git
-    prefix: ${workdir}/src
-    branch: microbenchmarks-stable
-  - name: workloads
-    repo: git@github.com:10gen/workloads.git
-    prefix: ${workdir}/src
-    branch: master
-  - name: linkbench
-    repo: git@github.com:10gen/linkbench.git
-    prefix: ${workdir}/src
-    branch: master
-  - name: linkbench2
-    repo: git@github.com:10gen/linkbench2.git
-    prefix: ${workdir}/src
-    branch: master
-  - name: mongo-perf
-    repo: git@github.com:mongodb/mongo-perf.git
-    prefix: ${workdir}/src
-    branch: master
-  - name: YCSB
-    repo: git@github.com:mongodb-labs/YCSB.git
-    prefix: ${workdir}/src
-    branch: production
-  - name: benchmarks
-    repo: git@github.com:mongodb-labs/benchmarks.git
-    prefix: ${workdir}/src
-    branch: master
-  - name: py-tpcc
-    repo: git@github.com:mongodb-labs/py-tpcc.git
-    prefix: ${workdir}/src
-    branch: production
-
-  ###
-#  - name: mongo
-#    repo: git@github.com:mongodb/mongo.git
-#    prefix: ${workdir}/src
-#    branch: master
-  - name: enterprise
-    repo: git@github.com:10gen/mongo-enterprise-modules.git
-    prefix: src/mongo/db/modules
-    branch: master
-  - name: mongo-tools
-    repo: git@github.com:mongodb/mongo-tools.git
-    prefix: mongo-tools/src/github.com/mongodb
-    branch: master
-
-
-###
-# Same in every DSI project
-pre:
-  - func: f_other_pre_ops
-  - func: f_dsi_pre_run
-post:
-  - func: f_dsi_post_run
-  - func: f_other_post_ops
-timeout:
-  - func: f_dsi_timeout
-  - func: f_other_timeout
-###
-
-functions:
-  ###
-  # Same in every DSI project
-  f_dsi_pre_run:
-    - command: manifest.load
-    - command: expansions.update
-      params:
-        updates: *_expansion_updates
-  f_run_dsi_workload:
-    - command: git.get_project
-      params:
-        directory: *src_dir
-        revisions:
-          dsi: ${dsi_rev}
-          genny: ${genny_rev}
-          linkbench: ${linkbench_rev}
-          linkbench2: ${linkbench2_rev}
-          workloads: ${workloads_rev}
-          mongo-perf: ${mongo-perf_rev}
-          YCSB: ${YCSB_rev}
-          benchmarks: ${benchmarks_rev}
-          py-tpcc: ${py-tpcc_rev}
-    - command: expansions.write
-      params:
-        file: ./expansions.yml
-    - command: shell.exec
-      params:
-        script: ./src/dsi/run-dsi run_workload
-    - command: shell.exec
-      type: system
-      params:
-        script: ./src/dsi/run-dsi determine_failure -m SYSTEM
-    - command: shell.exec
-      type: setup
-      params:
-        script: ./src/dsi/run-dsi determine_failure -m SETUP
-    - command: shell.exec
-      type: test
-      params:
-        script: ./src/dsi/run-dsi determine_failure -m TEST
-  f_dsi_post_run:
-    - command: shell.exec
-      params:
-        script: ./src/dsi/run-dsi post_run
-    - command: perf.send
-      params:
-        file: ./build/CedarReports/cedar_report.json
-        aws_key: ${terraform_key}
-        aws_secret: ${terraform_secret}
-        bucket: genny-metrics
-        region: us-east-1
-        prefix: ${task_id}_${execution}
-    - command: attach.results
-      params:
-        file_location: ./build/EvergreenResultsJson/results.json
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: ./build/Artifacts/DSIArtifacts.tgz
-        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: application/x-gzip
-        display_name: DSI Artifacts - Execution ${execution}
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: ./build/Documentation/index.html
-        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html
-        bucket: mciuploads
-        permissions: public-read
-        content_type: text/html
-        display_name: Documentation
-  f_dsi_timeout:
-    - command: shell.exec
-      params:
-        script: ./src/dsi/run-dsi on_timeout
-  ###
-
-  f_other_post_ops:
-      - command: shell.exec
-        params:
-          working_dir: src
-          script: |
-            # removes files from the (local) scons cache when it's over a
-            # threshold, to the $prune_ratio percentage. Ideally override
-            # these default values in the distro config in evergreen.
-
-            if [ -d "${scons_cache_path}" ]; then
-                /opt/mongodbtoolchain/v4/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
-            fi
-  f_other_pre_ops:
-    - &f_other_pre_ops
-      command: shell.exec
-      params:
-        silent: true
-        script: |
-          for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
-              pkill -9 "$PS"
-          done
-  f_other_timeout:
-    # Can't be empty so just `echo`.
-    - command: shell.exec
-      params: {script: "echo"}
-
-  ###
-  # Compile
-  compile mongodb:
-    # We create a virtual environment with the Python dependencies for compiling the server
-    # installed.
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-
-          mkdir -p mongodb/bin
-
-          /opt/mongodbtoolchain/v4/bin/virtualenv --python /opt/mongodbtoolchain/v4/bin/python3 "${workdir}/compile_venv"
-          source "${workdir}/compile_venv/bin/activate"
-
-          python -m pip install -r etc/pip/compile-requirements.txt
-    - command: expansions.write
-      params:
-         file: expansions.yml
-         redacted: true
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-
-          source "${workdir}/compile_venv/bin/activate"
-
-          # We get the raw version string (r1.2.3-45-gabcdef) from git
-          export MONGO_VERSION=$(git describe --abbrev=7)
-
-          # If this is a patch build, we add the patch version id to the version string so we know
-          # this build was a patch, and which evergreen task it came from
-          if [ "${is_patch|false}" = "true" ]; then
-            MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
-          fi
-
-          # This script handles sanitizing the version string for use during SCons build
-          # and when pushing artifacts up to S3.
-          IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
-            buildscripts/generate_version_expansions.py --out version_expansions.yml
-    - command: expansions.update
-      params:
-        file: src/version_expansions.yml
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-
-          # This script handles whether the SCons cache should be used
-          source "${workdir}/compile_venv/bin/activate"
-          SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
-            IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
-            python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
-    - command: expansions.update
-      params:
-        file: src/compile_expansions.yml
-    - command: shell.exec
-      params:
-        working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
-        script: |
-          set -o verbose
-          set -o errexit
-
-          # make sure newlines in the scripts are handled correctly by windows
-          if [ "Windows_NT" = "$OS" ]; then
-            set -o igncr
-          fi;
-
-          # set_goenv provides set_goenv(), print_ldflags() and print_tags() used below
-          . ./set_goenv.sh
-          GOROOT="" set_goenv || exit
-          go version
-
-          build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
-          if [ "${build_mongoreplay}" = "true" ]; then
-              build_tools="$build_tools mongoreplay"
-          fi
-          for i in $build_tools; do
-              go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
-              "../../../../../mongodb/bin/$i${exe|}" --version
-          done
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-          source "${workdir}/compile_venv/bin/activate"
-          python ./buildscripts/idl/gen_all_feature_flag_list.py
-          mkdir -p mongodb/feature_flags
-          cp ./all_feature_flags.txt mongodb/feature_flags
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-          source "${workdir}/compile_venv/bin/activate"
-          python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args install-core install-jstestshell SPLIT_DWARF=0 MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
-          mkdir -p mongodb/jstests/hooks
-          if [ -d jstests/hooks ]
-          then
-            echo "Fetching JS test DB correctness checks from directory jstests"
-            cp -a jstests/* mongodb/jstests
-
-            echo "Now adding our own special run_validate_collections.js wrapper"
-            mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
-
-            cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
-            print("NOTE: run_validate_collections.js will skip the oplog!");
-            TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
-            load('jstests/hooks/run_validate_collections.actual.js');
-          EOF
-          fi
-          tar czf mongodb${compile_variant|}.tar.gz mongodb
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/mongodb${compile_variant|}.tar.gz
-        remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}.tar.gz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: mongodb${compile_variant|}.tar.gz
-  ###
-
-  ## Schedule Tasks ##
-  f_schedule_tasks:
-    - command: git.get_project
-      params:
-        directory: *src_dir
-        revisions:
-          dsi: ${dsi_rev}
-          genny: ${genny_rev}
-          linkbench: ${linkbench_rev}
-          linkbench2: ${linkbench2_rev}
-          workloads: ${workloads_rev}
-          mongo-perf: ${mongo-perf_rev}
-          YCSB: ${YCSB_rev}
-          benchmarks: ${benchmarks_rev}
-          py-tpcc: ${py-tpcc_rev}
-    - command: expansions.write
-      params:
-        file: ./expansions.yml
-    - command: shell.exec
-      params:
-        script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks}
-    - command: generate.tasks
-      params:
-        files:
-          - build/TaskJSON/Tasks.json
-
-
-tasks:
-  ###
-  # Same in every DSI project
-  - name: schedule_global_auto_tasks
-    priority: 5
-    commands:
-      - func: f_schedule_tasks
-        vars:
-          tasks: all_tasks
-  - name: schedule_variant_auto_tasks
-    priority: 5
-    commands:
-      - func: f_schedule_tasks
-        vars:
-          tasks: variant_tasks
-  - name: schedule_patch_auto_tasks
-    priority: 5
-    commands:
-      - func: f_schedule_tasks
-        vars:
-          tasks: patch_tasks
-  - name: smoke_test
-    priority: 5
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: short
-  - name: smoke_test_ssl
-    priority: 5
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: short
-          mongodb_setup: replica-ssl
-          infrastructure_provisioning: replica
-  - name: smoke_test_standalone_auth
-    priority: 5
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: short
-          mongodb_setup: standalone-auth
-          infrastructure_provisioning: single
-  - name: smoke_test_replset_auth
-    priority: 5
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: short
-          mongodb_setup: replica-auth
-          infrastructure_provisioning: replica
-  - name: smoke_test_shard_lite_auth
-    priority: 5
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: short
-          mongodb_setup: shard-lite-auth
-          infrastructure_provisioning: shard-lite
-  - name: dsi_integ_test_run_command_simple
-    priority: 5
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: run_command_simple
-  ###
-
-  - name: compile
-    commands:
-      - command: manifest.load
-      - command: git.get_project
-        params:
-          directory: src
-          revisions:
-            enterprise: ${enterprise_rev}
-            mongo-tools: ${mongo-tools_rev}
-      - func: "compile mongodb"
-
-
-  - name: genny_execution_UserAcquisition
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: auto_genny_workload
-          auto_workload_path: ./src/genny/dist/etc/genny/workloads/execution/UserAcquisition.yml
-  - name: genny_scale_InsertRemove
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: auto_genny_workload
-          auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/InsertRemove.yml
-  - name: query_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: query,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: query_read_commands_large_dataset
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: query_large_dataset,
-             include_filter_2: regression,
-             exclude_filter: none,
-             threads: "1 4",
-             read_cmd: 'true',
-             share_dataset: 'true'}
-  - name: big_collection
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: query,
-             include_filter_2: getmore,
-             exclude_filter: none,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: views-query
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: query_identityview,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: views-aggregation
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: aggregation_identityview,
-             include_filter_2: regression,
-             exclude_filter: none,
-             threads: "1",
-             read_cmd: 'true'}
-  - name: where_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: where,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: update_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: update,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: insert_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: insert,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: wildcard-index-read_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: wildcard_read,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: wildcard-index-write_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: wildcard_write,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: geo_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: geo,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: misc_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: command multi remove mixed,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: misc_custom_filter_default_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          mongodb_setup: microbenchmarks_standalone_custom_filter_default
-          test_control_params: |
-            {include_filter_1: command multi remove mixed,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: misc_custom_filter_slow_or_sample_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          mongodb_setup: microbenchmarks_standalone_custom_filter_slow_or_sample
-          test_control_params: |
-            {include_filter_1: command multi remove mixed,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: misc_custom_filter_complex_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          mongodb_setup: microbenchmarks_standalone_custom_filter_complex
-          test_control_params: |
-            {include_filter_1: command multi remove mixed,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: misc_custom_filter_whole_doc_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          mongodb_setup: microbenchmarks_standalone_custom_filter_whole_doc
-          test_control_params: |
-            {include_filter_1: command multi remove mixed,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: misc_slowms_everything_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          mongodb_setup: microbenchmarks_standalone_slowms_everything
-          test_control_params: |
-            {include_filter_1: command multi remove mixed,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: singleThreaded_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: single_threaded,
-             include_filter_2: core regression,
-             exclude_filter: none,
-             threads: "1",
-             read_cmd: 'true'}
-  - name: aggregation_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: aggregation,
-             include_filter_2: regression,
-             exclude_filter: js,
-             threads: "1",
-             read_cmd: 'true'}
-  - name: aggregation_read_commands_large_dataset
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: aggregation_large_dataset,
-             include_filter_2: regression,
-             exclude_filter: js,
-             threads: "1 4",
-             read_cmd: 'true',
-             share_dataset: 'true'}
-  - name: agg-query-comparison_read_commands
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: agg_query_comparison,
-             include_filter_2: core regression,
-             exclude_filter: single_threaded,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: pipeline-updates
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: pipeline-updates,
-             include_filter_2: regression,
-             exclude_filter: none,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-  - name: javascript
-    depends_on: *_compile
-    commands:
-      - func: f_run_dsi_workload
-        vars:
-          test_control: microbenchmarks
-          test_control_params: |
-            {include_filter_1: js,
-             include_filter_2: aggregation,
-             exclude_filter: none,
-             threads: "1 2 4 8",
-             read_cmd: 'true'}
-
-
-microbenchmark-buildvariants:
-  _linux-wt-standalone: &linux-wt-standalone
-    name: linux-wt-standalone
-    display_name: Standalone Linux inMemory
-    cron: "0 */4 * * *" # Every 4 hours starting at midnight
-    modules: *modules
-    expansions:
-      # We are explicitly tracking the rhel62 variant compile options from evergreen.yml for
-      # microbenchmarks, since they run on the centos6 boxes.  If we can get proper artifacts directly
-      # from that project, we should do that and remove the compile tasks.
-      compile_flags: >-
-        --ssl
-        --separate-debug MONGO_DISTMOD=rhel62
-        -j$(grep -c ^processor /proc/cpuinfo)
-        --release
-        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
-      mongod_exec_wrapper: &exec_wrapper "numactl --physcpubind=4,5,6,7 -i 1"
-      perf_exec_wrapper: &perf_wrapper "numactl --physcpubind=1,2,3 -i 0"
-      use_scons_cache: true
-      platform: linux
-      infrastructure_provisioning: microbenchmarks
-      mongodb_setup: microbenchmarks_standalone
-      canaries: none
-      storageEngine: inMemory
-      project_dir: &project_dir perf
-    run_on:
-    - "centos6-perf"
-    tasks:
-    - name: compile
-      distros:
-      - rhel62-large
-    - name: big_collection
-    - name: genny_scale_InsertRemove
-    - name: genny_execution_UserAcquisition
-    - name: aggregation_read_commands
-    - name: aggregation_read_commands_large_dataset
-    - name: agg-query-comparison_read_commands
-    - name: query_read_commands
-    - name: query_read_commands_large_dataset
-    - name: views-aggregation
-    - name: views-query
-    - name: where_read_commands
-    - name: update_read_commands
-    - name: insert_read_commands
-    - name: wildcard-index-read_read_commands
-    - name: wildcard-index-write_read_commands
-    - name: geo_read_commands
-    - name: misc_read_commands
-    - name: misc_custom_filter_default_read_commands
-    - name: misc_custom_filter_slow_or_sample_read_commands
-    - name: misc_custom_filter_complex_read_commands
-    - name: misc_custom_filter_whole_doc_read_commands
-    - name: misc_slowms_everything_read_commands
-    - name: singleThreaded_read_commands
-    - name: pipeline-updates
-    - name: javascript
-
-  _linux-wt-repl: &linux-wt-repl
-    name: linux-wt-repl
-    display_name: 1-Node ReplSet Linux inMemory
-    cron: "0 */4 * * *" # Every 4 hours starting at midnight
-    modules: *modules
-    expansions:
-      mongod_exec_wrapper: *exec_wrapper
-      perf_exec_wrapper: *perf_wrapper
-      platform: linux
-      infrastructure_provisioning: microbenchmarks
-      mongodb_setup: microbenchmarks_replica
-      canaries: none
-      storageEngine: inMemory
-      project_dir: *project_dir
-    run_on:
-    - "centos6-perf"
-    tasks:
-    - name: genny_scale_InsertRemove
-    - name: update_read_commands
-    - name: insert_read_commands
-    - name: misc_read_commands
-    - name: singleThreaded_read_commands
-    - name: wildcard-index-write_read_commands
-    - name: pipeline-updates
-
-buildvariants:
-  - *linux-wt-standalone
-
-  - *linux-wt-repl
-
-  - <<: *linux-wt-standalone
-    name: linux-wt-standalone-all-feature-flags
-    display_name: Standalone Linux inMemory (all feature flags)
-    cron: "0 0 * * *" # Every day starting at 00:00
-    expansions:
-      mongodb_setup: microbenchmarks_standalone-all-feature-flags
-
-  - <<: *linux-wt-standalone
-    name: linux-wt-standalone-classic-query-engine
-    display_name: Standalone Linux inMemory (Classic Query Engine)
-    # Will make it less frequent when the current SBE perf improvement is finished (SERVER-69799).
-    cron: "0 0 * * 0,2,3,4,5" # Run it every day except Saturday and Monday.
-    expansions:
-      mongodb_setup: microbenchmarks_standalone-classic-query-engine
-
-  - <<: *linux-wt-standalone
-    name: linux-wt-standalone-sbe
-    display_name: Standalone Linux inMemory (SBE)
-    # Will make it less frequent when the current SBE perf improvement is finished (SERVER-69799).
-    cron: "0 0 * * 0,2,3,4,5" # Run it every day except Saturday and Monday.
-    expansions:
-      mongodb_setup: microbenchmarks_standalone-sbe
-
-  - <<: *linux-wt-repl
-    name: linux-wt-repl-all-feature-flags
-    display_name: 1-Node ReplSet Linux inMemory (all feature flags)
-    cron: "0 0 * * *" # Every day starting at 00:00
-    expansions:
-      mongodb_setup: microbenchmarks_replica-all-feature-flags

etc/pip/components/idl.req

@@ -1,3 +1,2 @@
 unittest-xml-reporting >= 2.2.0, <= 3.0.4
-typing <= 3.7.4.3
 packaging <= 21.3

etc/pip/components/lint.req

@@ -5,7 +5,6 @@ mypy ~= 0.942
 pydocstyle == 6.1.1
 pylint == 2.7.2
 structlog ~= 19.2.0
-typing <= 3.7.4.3
 yamllint == 1.15.0
 yapf == 0.26.0
 evergreen-lint == 0.1.3

evergreen/external_auth_aws_setup.sh

@@ -17,10 +17,6 @@ cat << EOF > aws_e2e_setup.json
 
     "iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}",
     "iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}",
-    "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}",
-
-    "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}",
-    "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}",
-    "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}"
+    "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}"
 }
 EOF

evergreen/functions/task_timeout_determine.sh

@@ -25,6 +25,12 @@ if [[ -n "${burn_in_bypass}" ]]; then
   build_variant_for_timeout=${burn_in_bypass}
 fi
 
+if [[ -n "${alias}" ]]; then
+  evg_alias=${alias}
+else
+  evg_alias="evg-alias-absent"
+fi
+
 activate_venv
 PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_factor \
   --install-dir "${install_dir}" \
@@ -32,7 +38,7 @@ PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_facto
   --suite-name ${suite_name} \
   --project ${project} \
   --build-variant $build_variant_for_timeout \
-  --evg-alias '${alias}' \
+  --evg-alias $evg_alias \
   --timeout ${timeout_secs} \
   --exec-timeout ${exec_timeout_secs} \
   --evg-api-config ./.evergreen.yml \

evergreen/garasign_gpg_crypt_sign.sh

@@ -0,0 +1,31 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+ext="${ext:-tgz}"
+
+crypt_file_name=mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext}
+mv "mongo_crypt_shared_v1.$ext" $crypt_file_name
+
+# generating checksums
+shasum -a 1 $crypt_file_name | tee $crypt_file_name.sha1
+shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256
+md5sum $crypt_file_name | tee $crypt_file_name.md5
+
+# signing crypt linux artifact with gpg
+cat << EOF >> gpg_signing_commands.sh
+gpgloader # loading gpg keys.
+gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name
+EOF
+
+podman run \
+  -e GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60} \
+  -e GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60} \
+  --rm \
+  -v $(pwd):$(pwd) -w $(pwd) \
+  ${garasign_gpg_image} \
+  /bin/bash -c "$(cat ./gpg_signing_commands.sh)"

evergreen/garasign_gpg_sign.sh

@@ -0,0 +1,63 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+set -o errexit
+set -o verbose
+
+cd src
+
+long_ext=${ext}
+if [ "$long_ext" == "tgz" ]; then
+  long_ext="tar.gz"
+fi
+
+mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext}
+mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true
+mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true
+mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true
+mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true
+
+# generating checksums
+function gen_checksums() {
+  if [ -e $1 ]; then
+    shasum -a 1 $1 | tee $1.sha1
+    shasum -a 256 $1 | tee $1.sha256
+    md5sum $1 | tee $1.md5
+  else
+    echo "$1 does not exist. Skipping checksum generation"
+  fi
+}
+
+gen_checksums mongodb-$push_name-$push_arch-$suffix.$ext
+gen_checksums mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext
+gen_checksums mongodb-src-$src_suffix.$long_ext
+gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext
+
+# signing linux artifacts with gpg
+cat << 'EOF' > gpg_signing_commands.sh
+gpgloader # loading gpg keys.
+function sign(){
+  if [ -e $1 ]
+  then
+    gpg --yes -v --armor -o $1.sig --detach-sign $1
+  else
+    echo "$1 does not exist. Skipping signing"
+  fi
+}
+
+EOF
+
+cat << EOF >> gpg_signing_commands.sh
+sign mongodb-$push_name-$push_arch-$suffix.$ext
+sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext
+sign mongodb-src-$src_suffix.$long_ext
+sign mongodb-cryptd-$push_name-$push_arch-$suffix.$ext
+EOF
+
+podman run \
+  -e GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60} \
+  -e GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60} \
+  --rm \
+  -v $(pwd):$(pwd) -w $(pwd) \
+  ${garasign_gpg_image} \
+  /bin/bash -c "$(cat ./gpg_signing_commands.sh)"

evergreen/garasign_jsign_sign.sh

@@ -0,0 +1,42 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+set -o errexit
+set -o verbose
+
+cd src
+
+msi_filename=mongodb-${push_name}-${push_arch}-${suffix}.msi
+/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename $msi_filename || true
+
+# signing windows artifacts with jsign
+cat << 'EOF' > jsign_signing_commands.sh
+function sign(){
+  if [ -e $1 ]
+  then
+    jsign -a mongo-authenticode-2021 --replace --tsaurl http://timestamp.digicert.com -d SHA-256 $1
+  else
+    echo "$1 does not exist. Skipping signing"
+  fi
+}
+EOF
+cat << EOF >> jsign_signing_commands.sh
+sign $msi_filename
+EOF
+
+podman run \
+  -e GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username} \
+  -e GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password} \
+  --rm \
+  -v $(pwd):$(pwd) -w $(pwd) \
+  ${garasign_jsign_image} \
+  /bin/bash -c "$(cat ./jsign_signing_commands.sh)"
+
+# generating checksums
+if [ -e $msi_filename ]; then
+  shasum -a 1 $msi_filename | tee $msi_filename.sha1
+  shasum -a 256 $msi_filename | tee $msi_filename.sha256
+  md5sum $msi_filename | tee $msi_filename.md5
+else
+  echo "$msi_filename does not exist. Skipping checksum generation"
+fi

evergreen/generate_buildid_debug_symbols_mapping.sh

@@ -6,10 +6,16 @@ cd src
 set -o errexit
 set -o verbose
 
+is_san_variant_arg=""
+if [[ -n "${san_options}" ]]; then
+  is_san_variant_arg="--is-san-variant"
+fi
+
 activate_venv
 
 $python buildscripts/debugsymb_mapper.py \
   --version "${version_id}" \
   --client-id "${symbolizer_client_id}" \
   --client-secret "${symbolizer_client_secret}" \
-  --variant "${build_variant}"
+  --variant "${build_variant}" \
+  $is_san_variant_arg

evergreen/jepsen_docker/setup.sh

@@ -1,38 +1,5 @@
 set -euo pipefail
 
-if [ ! $(which docker) ]; then
-  sudo apt-get update
-  sudo apt-get install -yq \
-    apt-transport-https \
-    ca-certificates \
-    curl \
-    gnupg \
-    lsb-release
-
-  if [ ! -f "/usr/share/keyrings/docker-archive-keyring.gpg" ]; then
-    curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-  fi
-
-  set +e
-  if ! grep "https://download.docker.com/linux/debian" "/etc/apt/sources.list.d/docker.list"; then
-    echo \
-      "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
-            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-  fi
-  set -e
-
-  sudo apt-get update
-  sudo apt-get install -yq docker-ce docker-ce-cli containerd.io
-  sudo docker run hello-world
-fi
-
-if [ ! $(which docker-compose) ]; then
-  sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose
-  sudo chmod +x /usr/bin/docker-compose
-fi
-
-sudo chmod 777 /var/run/docker.sock
-
 git clone --branch=evergreen-master git@github.com:10gen/jepsen.git jepsen
 cp -rf src/dist-test jepsen/docker/node
 # place the mongodb jepsen test adjacent to the control node's Dockerfile.

evergreen/notary_client_crypt_run.sh

@@ -1,20 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-. ./notary_env.sh
-
-set -o errexit
-set -o verbose
-
-ext="${ext:-tgz}"
-
-mv "mongo_crypt_shared_v1.$ext" mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext}
-
-/usr/local/bin/notary-client.py \
-  --key-name "server-6.0" \
-  --auth-token-file ${workdir}/src/signing_auth_token \
-  --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" \
-  --notary-url http://notary-service.build.10gen.cc:5000 \
-  mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext}

evergreen/notary_client_run.sh

@@ -1,23 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-. ./notary_env.sh
-
-set -o errexit
-set -o verbose
-
-long_ext=${ext}
-if [ "$long_ext" == "tgz" ]; then
-  long_ext="tar.gz"
-fi
-
-mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext}
-mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true
-mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true
-mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true
-mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true
-/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-${push_name}-${push_arch}-${suffix}.msi || true
-
-/usr/local/bin/notary-client.py --key-name "server-6.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} mongodb-${push_name}-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${long_ext} mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext}

evergreen/prelude_mongo_task_generator.sh

@@ -1,6 +1,6 @@
 function setup_mongo_task_generator {
   if [ ! -f mongo-task-generator ]; then
-    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.2/mongo-task-generator --output mongo-task-generator
+    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.5/mongo-task-generator --output mongo-task-generator
     chmod +x mongo-task-generator
   fi
 }

evergreen/prelude_venv.sh

@@ -22,8 +22,24 @@ function activate_venv {
 
   if [ "Windows_NT" = "$OS" ]; then
     export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
+  elif [ "$(uname)" = "Darwin" ]; then
+    #SERVER-75626 After activating the virtual environment under the mocos host. the PYTHONPATH setting
+    #is incorrect, and the site-packages directory of the virtual environment cannot be found in the sys.path.
+    python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+    export PYTHONPATH="${workdir}/venv/lib/python${python_version}/site-packages:${PYTHONPATH}:${workdir}/src"
   else
-    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
+    python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+    site_packages="${workdir}/venv/lib/python${python_version}/site-packages"
+    python -c "import sys; print(sys.path)"
+
+    # Check if site_packages is already in sys.path
+    in_sys_path=$($python -c "import sys; print('$site_packages' in sys.path)")
+    if [ "$in_sys_path" = "False" ]; then
+      export PYTHONPATH="${site_packages}:${PYTHONPATH}:${workdir}/src"
+    else
+      export PYTHONPATH="$PYTHONPATH:${workdir}/src"
+    fi
+    python -c "import sys; print(sys.path)"
   fi
 
   echo "python set to $(which $python)"

jstests/aggregation/sources/lookup/lookup_query_stats.js

@@ -252,7 +252,12 @@ let testQueryExecutorStatsWithCollectionScan = function(params) {
         checkExplainOutputForAllVerbosityLevels(
             localColl,
             fromColl,
-            {totalDocsExamined: 20, totalKeysExamined: 0, collectionScans: 4, indexesUsed: []},
+            {
+                totalDocsExamined: localDocCount * foreignDocCount,
+                totalKeysExamined: 0,
+                collectionScans: localDocCount,
+                indexesUsed: []
+            },
             {allowDiskUse: false},
             params.withUnwind);
     }

jstests/auth/impersonation-deny.js

@@ -8,9 +8,8 @@ function testMongod(mongod, systemuserpwd = undefined) {
     const admin = mongod.getDB('admin');
     admin.createUser({user: 'admin', pwd: 'admin', roles: ['root']});
 
-    function assertUnauthorized(cmd, msg) {
-        const errmsg =
-            assert.commandFailedWithCode(admin.runCommand(cmd), ErrorCodes.Unauthorized).errmsg;
+    function assertError(cmd, msg, code) {
+        const errmsg = assert.commandFailedWithCode(admin.runCommand(cmd), code).errmsg;
         assert(errmsg.includes(msg), "Error message is missing '" + msg + "': " + errmsg);
     }
 
@@ -18,7 +17,8 @@ function testMongod(mongod, systemuserpwd = undefined) {
 
     // Localhost authbypass is disabled, and we haven't logged in,
     // so normal auth-required commands should fail.
-    assertUnauthorized({usersInfo: 1}, 'Command usersInfo requires authentication');
+    assertError(
+        {usersInfo: 1}, 'Command usersInfo requires authentication', ErrorCodes.Unauthorized);
 
     // Hello command requires no auth, so it works fine.
     assert.commandWorked(admin.runCommand({hello: 1}));
@@ -33,7 +33,8 @@ function testMongod(mongod, systemuserpwd = undefined) {
             "$impersonatedRoles": [{role: 'root', db: 'admin'}],
         }
     };
-    assertUnauthorized(kImpersonatedHello, 'Unauthorized use of impersonation metadata');
+    assertError(
+        kImpersonatedHello, 'Unauthorized use of impersonation metadata', ErrorCodes.Unauthorized);
 
     // TODO SERVER-72448: Remove
     const kImpersonatedHelloLegacy = {
@@ -43,11 +44,41 @@ function testMongod(mongod, systemuserpwd = undefined) {
             "$impersonatedRoles": [{role: 'root', db: 'admin'}],
         }
     };
-    assertUnauthorized(kImpersonatedHelloLegacy, 'Unauthorized use of impersonation metadata');
+    assertError(kImpersonatedHelloLegacy,
+                'Unauthorized use of impersonation metadata',
+                ErrorCodes.Unauthorized);
+
+    // TODO SERVER-72448: Remove, checks that both legacy and new impersonation metadata fields
+    // cannot be set simultaneously.
+    const kImpersonatedHelloBoth = {
+        hello: 1,
+        "$audit": {
+            "$impersonatedUser": {user: 'admin', db: 'admin'},
+            "$impersonatedUsers": [{user: 'admin', db: 'admin'}],
+            "$impersonatedRoles": [{role: 'root', db: 'admin'}],
+        }
+    };
+    assertError(kImpersonatedHelloBoth,
+                'Cannot specify both $impersonatedUser and $impersonatedUsers',
+                ErrorCodes.BadValue);
+
+    // TODO SERVER-72448: Remove, checks that the legacy impersonation metadata field can only
+    // contain at most 1 field if specified.
+    const kImpersonatedHelloLegacyMultiple = {
+        hello: 1,
+        "$audit": {
+            "$impersonatedUsers": [{user: 'admin', db: 'admin'}, {user: 'test', db: 'pwd'}],
+            "$impersonatedRoles": [{role: 'root', db: 'admin'}],
+        }
+    };
+    assertError(kImpersonatedHelloLegacyMultiple,
+                'Can only impersonate up to one user per connection',
+                ErrorCodes.BadValue);
 
     // Try as admin (root role), should still fail.
     admin.auth('admin', 'admin');
-    assertUnauthorized(kImpersonatedHello, 'Unauthorized use of impersonation metadata');
+    assertError(
+        kImpersonatedHello, 'Unauthorized use of impersonation metadata', ErrorCodes.Unauthorized);
     admin.logout();
 
     if (systemuserpwd !== undefined) {

jstests/auth/lib/commands_lib.js

@@ -5354,6 +5354,36 @@ export const authCommandsLib = {
               {runOnDb: secondDbName, roles: {}}
           ]
         },
+        {
+          testname: "s_moveRange",
+          command: {moveRange: "test.x", min: {x:1}, toShard:"a"},
+          skipUnlessSharded: true,
+          testcases: [
+              {
+                runOnDb: adminDbName,
+                roles: roles_clusterManager,
+                privileges: [{resource: {db: "test", collection: "x"}, actions: ["moveChunk"]}],
+                expectFail: true
+              },
+              {runOnDb: firstDbName, roles: {}},
+              {runOnDb: secondDbName, roles: {}}
+          ]
+        },
+        {
+          testname: "d_moveRange",
+          command: {_shardsvrMoveRange: "test.x", fromShard: "a", toShard: "b", min: {}, max: {}, maxChunkSizeBytes: 1024},
+          skipSharded: true,
+          testcases: [
+              {
+                runOnDb: adminDbName,
+                roles: {__system: 1},
+                privileges: [{resource: {cluster: true}, actions: ["internal"]}],
+                expectFail: true
+              },
+              {runOnDb: firstDbName, roles: {}},
+              {runOnDb: secondDbName, roles: {}}
+          ]
+        },
         {
           testname: "movePrimary",
           command: {movePrimary: "x"},

jstests/auth/sbe_plan_cache_user_roles.js

@@ -0,0 +1,56 @@
+/**
+ * Test $$USER_ROLES works correctly with the SBE plan cache. The same query should return the
+ * updated user role info when a different user logs in.
+ * @tags: [
+ *   featureFlagUserRoles,
+ *   # Multiple servers can mess up the plan cache list.
+ *   assumes_standalone_mongod,
+ * ]
+ */
+(function() {
+"use strict";
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+const mongod = MongoRunner.runMongod();
+const dbName = "test";
+const db = mongod.getDB(dbName);
+
+// Create two users, each with different roles.
+assert.commandWorked(
+    db.runCommand({createUser: "user1", pwd: "pwd", roles: [{role: "read", db: dbName}]}));
+assert.commandWorked(
+    db.runCommand({createUser: "user2", pwd: "pwd", roles: [{role: "readWrite", db: dbName}]}));
+
+const coll = db.sbe_plan_cache_user_roles;
+coll.drop();
+
+const verifyPlanCache = function(role) {
+    if (checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+        const caches = coll.getPlanCache().list();
+        assert.eq(1, caches.length, caches);
+        assert.eq(caches[0].cachedPlan.stages.includes(role), false, caches);
+    }
+};
+
+assert.commandWorked(coll.insert({_id: 1}));
+
+// While logged in as user1, we should see user1's roles.
+db.auth("user1", "pwd");
+let results = coll.find({}, {roles: "$$USER_ROLES"}).toArray();
+assert.eq(results.length, 1);
+assert.eq(results[0].roles, [{_id: "test.read", role: "read", db: "test"}]);
+// It can take two executions of a query for a plan to get cached.
+coll.find({}, {roles: "$$USER_ROLES"}).toArray();
+verifyPlanCache("test.read");
+db.logout();
+
+// While logged in as user2, we should see user2's roles.
+db.auth("user2", "pwd");
+results = coll.find({}, {roles: "$$USER_ROLES"}).toArray();
+assert.eq(results.length, 1);
+assert.eq(results[0].roles, [{_id: "test.readWrite", role: "readWrite", db: "test"}]);
+verifyPlanCache("test.readWrite");
+db.logout();
+
+MongoRunner.stopMongod(mongod);
+})();

jstests/concurrency/fsm_workloads/random_DDL_operations.js

@@ -10,6 +10,8 @@
  *  ]
  */
 
+load('jstests/libs/feature_flag_util.js');
+
 const dbPrefix = jsTestName() + '_DB_';
 const dbCount = 2;
 const collPrefix = 'sharded_coll_';
@@ -62,6 +64,10 @@ var $config = (function() {
                 ]);
         },
         movePrimary: function(db, collName, connCache) {
+            if (this.skipMovePrimary) {
+                return;
+            }
+
             db = getRandomDb(db);
             const shardId = getRandomShard(connCache);
 
@@ -91,6 +97,10 @@ var $config = (function() {
     };
 
     let setup = function(db, collName, connCache) {
+        // TODO (SERVER-71309): Remove once 7.0 becomes last LTS. Prevent non-resilient movePrimary
+        // operations from being executed in multiversion suites.
+        this.skipMovePrimary = !FeatureFlagUtil.isEnabled(db.getMongo(), 'ResilientMovePrimary');
+
         for (var i = 0; i < dbCount; i++) {
             const dbName = dbPrefix + i;
             const newDb = db.getSiblingDB(dbName);
@@ -116,7 +126,6 @@ var $config = (function() {
         startState: 'create',
         states: states,
         transitions: transitions,
-        data: {},
         setup: setup,
         teardown: teardown,
         passConnectionCache: true

jstests/concurrency/fsm_workloads/rename_sharded_collection.js

@@ -16,10 +16,6 @@
  *   does_not_support_add_remove_shards,
  *   # This test just performs rename operations that can't be executed in transactions
  *   does_not_support_transactions,
- *   # Can be removed once PM-1965-Milestone-1 is completed.
- *
- *   # TODO SERVER-73385 reenable when fixed.
- *   assumes_balancer_off,
  *  ]
  */
 

jstests/concurrency/fsm_workloads/server_status_with_time_out_cursors.js

@@ -4,7 +4,11 @@
  * Run serverStatus() while running a large number of queries which are expected to reach maxTimeMS
  * and time out.
  *
- * @tags: [catches_command_failures]
+ * @tags: [
+ *     catches_command_failures,
+ *     # This test leaks cursors causing range deletions to hang waiting for ongoing queries
+ *     assumes_balancer_off,
+ * ]
  */
 load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongos
 

jstests/core/administrative/profile/profile_delete.js

@@ -6,6 +6,8 @@
 //   does_not_support_stepdowns,
 //   requires_non_retryable_writes,
 //   requires_profiling,
+//   # Uses $where operator
+//   requires_scripting,
 // ]
 
 // Confirms that profiled delete execution contains all expected metrics with proper values.

jstests/core/clustered/clustered_collection_collation.js

@@ -35,6 +35,9 @@ const incompatibleCollation = {
     locale: "fr_CA",
     strength: 2
 };
+const simpleCollation = {
+    locale: "simple",
+};
 
 assert.commandWorked(db.createCollection(
     collatedName, {clusteredIndex: {key: {_id: 1}, unique: true}, collation: defaultCollation}));
@@ -141,7 +144,7 @@ const verifyNoTightBoundsAndFindsN = function(coll, expected, predicate, queryCo
     const max = res.queryPlanner.winningPlan.maxRecord;
     assert.neq(null, min, "No min bound");
     assert.neq(null, max, "No max bound");
-    assert.neq(min, max, "COLLSCAN bounds are equal");
+    assert(min !== max, "COLLSCAN bounds are equal");
     assert.eq(expected, coll.find(predicate).count(), "Didn't find the expected records");
 };
 
@@ -150,33 +153,77 @@ const testBounds = function(coll, expected, defaultCollation) {
     verifyHasBoundsAndFindsN(coll, 1, {_id: 5});
     verifyHasBoundsAndFindsN(coll, 1, {_id: {int: 5}});
     verifyHasBoundsAndFindsN(coll, 1, {_id: {ints: [5, 10]}});
+    verifyNoTightBoundsAndFindsN(coll, 2, {_id: {$in: [5, {ints: [5, 10]}]}});
 
     // Test non string types with incompatible collations.
     verifyHasBoundsAndFindsN(coll, 1, {_id: 5}, incompatibleCollation);
     verifyHasBoundsAndFindsN(coll, 1, {_id: {int: 5}}, incompatibleCollation);
     verifyHasBoundsAndFindsN(coll, 1, {_id: {ints: [5, 10]}}, incompatibleCollation);
+    verifyNoTightBoundsAndFindsN(
+        coll, 2, {_id: {$in: [5, {ints: [5, 10]}]}}, incompatibleCollation);
 
     // Test strings respect the collation.
     verifyHasBoundsAndFindsN(coll, expected, {_id: "A"});
     verifyHasBoundsAndFindsN(coll, expected, {_id: {str: "A"}});
     verifyHasBoundsAndFindsN(coll, expected, {_id: {strs: ["A", "b"]}});
     verifyHasBoundsAndFindsN(coll, expected, {_id: {strs: ["a", "B"]}});
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", 1]}});
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", "C"]}});
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["", {str: "A"}]}});
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: [{}, {strs: ["A", "b"]}]}});
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: [[], {strs: ["a", "B"]}]}});
 
     // Test strings not in the _id field
     verifyNoBoundsAndFindsN(coll, expected, {data: ["A", "b"]});
     verifyNoBoundsAndFindsN(coll, expected, {data: ["a", "B"]});
 
-    // Test non compatible query collations don't generate bounds
+    // Test non compatible query collations don't generate exact bounds. This means, the bounds
+    // generated are with respect to the KeyString encoding of the data type of the query. For
+    // example, an _id: <string> query will be bounded by min and max values for type 'string', but
+    // not bounded by the exact value of <string>.
     verifyNoTightBoundsAndFindsN(coll, expected, {_id: "A"}, incompatibleCollation);
     verifyNoTightBoundsAndFindsN(coll, expected, {_id: {str: "A"}}, incompatibleCollation);
     verifyNoTightBoundsAndFindsN(coll, expected, {_id: {strs: ["A", "b"]}}, incompatibleCollation);
     verifyNoTightBoundsAndFindsN(coll, expected, {_id: {strs: ["a", "B"]}}, incompatibleCollation);
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", 1]}}, incompatibleCollation);
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", "C"]}}, incompatibleCollation);
+    verifyNoTightBoundsAndFindsN(
+        coll, expected, {_id: {$in: ["", {str: "A"}]}}, incompatibleCollation);
+    verifyNoTightBoundsAndFindsN(
+        coll, expected, {_id: {$in: [{}, {strs: ["A", "b"]}]}}, incompatibleCollation);
+    verifyNoTightBoundsAndFindsN(
+        coll, expected, {_id: {$in: [[], {strs: ["a", "B"]}]}}, incompatibleCollation);
+
+    if (defaultCollation != undefined && defaultCollation.locale != simpleCollation.locale) {
+        // 'Simple' collations are treated differently than non-simple queries since they are the
+        // default 'locale' when a collation is not specified. Test that the 'simple' collation is
+        // not compatible when the clustered collection has a non-simple collation.
+        verifyNoTightBoundsAndFindsN(coll, expected, {_id: "A"}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(coll, expected, {_id: {str: "A"}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(coll, expected, {_id: {strs: ["A", "b"]}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(coll, expected, {_id: {strs: ["a", "B"]}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", 1]}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", "C"]}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(
+            coll, expected, {_id: {$in: ["", {str: "A"}]}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(
+            coll, expected, {_id: {$in: [{}, {strs: ["A", "b"]}]}}, simpleCollation);
+        verifyNoTightBoundsAndFindsN(
+            coll, expected, {_id: {$in: [[], {strs: ["a", "B"]}]}}, simpleCollation);
+    }
 
     // Test compatible query collations generate bounds
     verifyHasBoundsAndFindsN(coll, expected, {_id: "A"}, defaultCollation);
     verifyHasBoundsAndFindsN(coll, expected, {_id: {str: "A"}}, defaultCollation);
     verifyHasBoundsAndFindsN(coll, expected, {_id: {strs: ["A", "b"]}}, defaultCollation);
     verifyHasBoundsAndFindsN(coll, expected, {_id: {strs: ["a", "B"]}}, defaultCollation);
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", 1]}}, defaultCollation);
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["A", "C"]}}, defaultCollation);
+    verifyNoTightBoundsAndFindsN(coll, expected, {_id: {$in: ["", {str: "A"}]}}, defaultCollation);
+    verifyNoTightBoundsAndFindsN(
+        coll, expected, {_id: {$in: [{}, {strs: ["A", "b"]}]}}, defaultCollation);
+    verifyNoTightBoundsAndFindsN(
+        coll, expected, {_id: {$in: [[], {strs: ["a", "B"]}]}}, defaultCollation);
 };
 
 insertDocuments(collated);

jstests/core/columnstore/column_scan_skip_row_store_projection.js

@@ -3,6 +3,9 @@
  * above a columnscan stage.
  *
  * @tags: [
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # explain is not supported in transactions
  *   does_not_support_transactions,
  *   requires_pipeline_optimization,
@@ -10,8 +13,6 @@
  *   assumes_read_concern_unchanged,
  *   # explain will be different in a sharded collection
  *   assumes_unsharded_collection,
- *   # column store row store expression skipping is new in 6.2.
- *   requires_fcv_62,
  *   # Columnstore tests set server parameters to disable columnstore query planning heuristics -
  *   # 1) server parameters are stored in-memory only so are not transferred onto the recipient,
  *   # 2) server parameters may not be set in stepdown passthroughs because it is a command that may

jstests/core/columnstore/column_store_index_compression.js

@@ -6,8 +6,8 @@
  *   requires_collstats,
  *
  *   # Column store indexes are still under a feature flag and require full SBE.
- *   uses_column_store_index,
  *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *
  *   # In passthrough suites, this test makes direct connections to mongod instances that compose
  *   # the passthrough fixture in order to perform additional validation. Tenant migration,

jstests/core/columnstore/column_store_projection.js

@@ -2,7 +2,9 @@
  * Test column stores indexes that use a "columnstoreProjection" or "prefix.$**" notation to limit
  * indexed data to a subset of the document namespace.
  * @tags: [
- *   requires_fcv_63,
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # Runs explain on an aggregate command which is only compatible with readConcern local.
  *   assumes_read_concern_unchanged,
  *   # Columnstore tests set server parameters to disable columnstore query planning heuristics -

jstests/core/columnstore/columnstore_eligibility.js

@@ -1,7 +1,9 @@
 /**
  * Tests the eligibility of certain queries to use a columnstore index.
  * @tags: [
- *   requires_fcv_63,
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # Refusing to run a test that issues an aggregation command with explain because it may return
  *   # incomplete results if interrupted by a stepdown.
  *   does_not_support_stepdowns,

jstests/core/columnstore/columnstore_index.js

@@ -1,7 +1,9 @@
 /**
  * Tests some basic use cases and functionality of a columnstore index.
  * @tags: [
- *   requires_fcv_63,
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *
  *   # Uses $indexStats which is not supported inside a transaction.
  *   does_not_support_transactions,

jstests/core/columnstore/columnstore_index_correctness.js

@@ -1,7 +1,9 @@
 /**
  * Testing of just the query layer's integration for columnar index.
  * @tags: [
- *   requires_fcv_63,
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # Runs explain on an aggregate command which is only compatible with readConcern local.
  *   assumes_read_concern_unchanged,
  *   # Columnstore tests set server parameters to disable columnstore query planning heuristics -

jstests/core/columnstore/columnstore_index_per_path_filters.js

@@ -3,7 +3,9 @@
  * might be pushed down into the column scan stage.
  *
  * @tags: [
- *   requires_fcv_63,
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # Runs explain on an aggregate command which is only compatible with readConcern local.
  *   assumes_read_concern_unchanged,
  *   # Columnstore tests set server parameters to disable columnstore query planning heuristics -

jstests/core/columnstore/columnstore_large_array_index_correctness.js

@@ -1,6 +1,9 @@
 /**
  * Testing of just the query layer's integration for columnar indexes that encode large arrays.
  * @tags: [
+ *   # Column store indexes are still under a feature flag and require full SBE.
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # Columnstore tests set server parameters to disable columnstore query planning heuristics -
  *   # 1) server parameters are stored in-memory only so are not transferred onto the recipient,
  *   # 2) server parameters may not be set in stepdown passthroughs because it is a command that may
@@ -8,8 +11,6 @@
  *   tenant_migration_incompatible,
  *   does_not_support_stepdowns,
  *   not_allowed_with_security_token,
-
- *   requires_fcv_63,
  * ]
  */
 (function() {

jstests/core/columnstore/columnstore_validindex.js

@@ -1,7 +1,8 @@
 /**
  * Tests parsing and validation of columnstore indexes.
  * @tags: [
- *   requires_fcv_63,
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  *   # Uses index building in background.
  *   requires_background_index,
  *   # Columnstore tests set server parameters to disable columnstore query planning heuristics -

jstests/core/doc_validation_with_now_variable.js

@@ -0,0 +1,20 @@
+/**
+ * Tests that insertion with a $$NOW Validator works properly.
+ *
+ * @tags: [assumes_no_implicit_collection_creation_after_drop]
+ */
+(function() {
+"use strict";
+const coll = db.coll_doc_validation_with_now_variable;
+coll.drop();
+// assert.commandWorked(db.createCollection("coll_doc_validation_with_now_variable",
+//                                          {validator: {"$expr": {$gt: ["$ts", "$$NOW"]}}}));
+
+assert.commandWorked(db.createCollection("coll_doc_validation_with_now_variable",
+                                         {validator: {"$expr": {$lt: ["$ts", "$$NOW"]}}}));
+
+assert.commandWorked(coll.insert({"ts": new Date(1589617694938)}));
+
+const result = coll.insert({"ts": new Date(2708791380000)});
+assert.commandFailedWithCode(result, ErrorCodes.DocumentValidationFailure, tojson(result));
+})();

jstests/core/field_name_validation.js

@@ -66,6 +66,9 @@ assert.commandWorked(coll.insert({_id: 2, $valid: 1, $db: 1}));
 assert.commandWorked(coll.insert({_id: 3, $valid: 1, $ref: 1}));
 assert.commandWorked(coll.insert({_id: 4, $valid: 1, $alsoValid: 1}));
 
+// Valid, because _id.$gt is a field name, and not equivalent to {_id: {$gt: 4}}
+assert.commandWorked(coll.insert({"_id.$gt": 4}));
+
 //
 // Update command field name validation.
 //
@@ -93,9 +96,16 @@ assert.writeErrorWithCode(coll.update({"a.b": 1}, {_id: {$invalid: 1}}, {upsert:
                           ErrorCodes.DollarPrefixedFieldName);
 assert.writeErrorWithCode(coll.update({"a.b": 1}, {$set: {_id: {$invalid: 1}}}, {upsert: true}),
                           ErrorCodes.DollarPrefixedFieldName);
+assert.writeErrorWithCode(coll.update({"a.b": 1}, {$set: {"_id.$gt": 1}}, {upsert: true}),
+                          ErrorCodes.DollarPrefixedFieldName);
 assert.writeErrorWithCode(
     coll.update({"a.b": 1}, {$setOnInsert: {_id: {$invalid: 1}}}, {upsert: true}),
     ErrorCodes.DollarPrefixedFieldName);
+assert.writeErrorWithCode(
+    coll.update({"a.b": 1}, {$setOnInsert: {"_id.$invalid": 1}}, {upsert: true}),
+    ErrorCodes.DollarPrefixedFieldName);
+assert.writeErrorWithCode(coll.update({"_id.$gt": 1}, {$set: {a: 1}}, {upsert: true}),
+                          ErrorCodes.DollarPrefixedFieldName);
 
 // Replacement-style updates can contain nested $-prefixed fields.
 assert.commandWorked(coll.update({"a.b": 1}, {a: {$c: 1}}));

jstests/core/function_string_representations.js

@@ -3,6 +3,8 @@
 //   not_allowed_with_security_token,
 //   does_not_support_stepdowns,
 //   uses_map_reduce_with_temp_collections,
+//   # Uses $where operator
+//   requires_scripting,
 // ]
 
 /**

jstests/core/map_reduce_subplanning.js

@@ -0,0 +1,47 @@
+// The test runs commands that are not allowed with security token: mapReduce.
+// @tags: [
+//   not_allowed_with_security_token,
+//   does_not_support_stepdowns,
+//   requires_fastcount,
+//   requires_getmore,
+//   requires_non_retryable_writes,
+//   # This test has statements that do not support non-local read concern.
+//   does_not_support_causal_consistency,
+//   # Uses mapReduce command.
+//   requires_scripting,
+// ]
+
+load("jstests/aggregation/extras/utils.js");  // For resultsEq
+(function() {
+"use strict";
+
+const coll = db.map_reduce_subplanning;
+coll.drop();
+db.getCollection("mrOutput").drop();
+
+coll.createIndex({a: 1, c: 1});
+coll.createIndex({b: 1, c: 1});
+coll.createIndex({a: 1});
+coll.createIndex({b: 1});
+
+assert.commandWorked(coll.insert({a: 2}));
+assert.commandWorked(coll.insert({b: 3}));
+assert.commandWorked(coll.insert({b: 3}));
+assert.commandWorked(coll.insert({a: 2, b: 3}));
+
+assert.commandWorked(coll.mapReduce(
+    function() {
+        if (!this.hasOwnProperty('a')) {
+            emit('a', 0);
+        } else {
+            emit('a', this.a);
+        }
+    },
+    function(key, vals) {
+        return vals.reduce((a, b) => a + b, 0);
+    },
+    {out: {merge: "mrOutput"}, query: {$or: [{a: 2}, {b: 3}]}}));
+
+assert(resultsEq([{"_id": "a", "value": 4}], db.getCollection("mrOutput").find().toArray()),
+       db.getCollection("mrOutput").find().toArray());
+})();

jstests/core/query/count/counta.js

@@ -1,6 +1,8 @@
 // Check that count returns 0 in some exception cases.
 //
-// @tags: [requires_fastcount]
+// @tags: [
+//   requires_fastcount
+// ]
 
 (function() {
 'use strict';

jstests/core/query/js/js2.js

@@ -1,4 +1,6 @@
-// @tags: [requires_non_retryable_writes]
+// @tags: [
+//   requires_non_retryable_writes
+//]
 
 t = db.jstests_js2;
 t.remove({});

jstests/core/query/sbe_plan_cache_autoparameterize_ixscan.js

@@ -0,0 +1,91 @@
+/**
+ * Tests that auto-parameterized index scan plans are correctly stored in the SBE plan cache, and
+ * that they can be correctly recovered from the cache with new parameter values.
+ *
+ * @tags: [
+ *   not_allowed_with_security_token,
+ *   assumes_read_concern_unchanged,
+ *   assumes_read_preference_unchanged,
+ *   assumes_unsharded_collection,
+ *   does_not_support_stepdowns,
+ *   # The SBE plan cache was enabled by default in 6.3.
+ *   requires_fcv_63,
+ *   # Plan cache state is node-local and will not get migrated alongside tenant data.
+ *   tenant_migration_incompatible,
+ *   cqf_incompatible,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/sbe_util.js");      // For 'checkSBEEnabled'.
+load("jstests/libs/analyze_plan.js");  // For 'getQueryHashFromExplain'.
+
+// This test is specifically verifying the behavior of the SBE plan cache, which is only enabled
+// when SBE is enabled.
+if (!checkSBEEnabled(db)) {
+    jsTestLog("Skipping test because SBE is not enabled");
+    return;
+}
+
+const coll = db[jsTestName()];
+coll.drop();
+
+// Set up the collection with an index and a set of documents.
+assert.commandWorked(coll.createIndex({a: 1}));
+assert.commandWorked(coll.insertMany([{_id: 1, a: 1}, {_id: 2, a: 2}, {_id: 3, a: 3}]));
+const filter1 = {
+    a: {$gte: 2, $lte: 2}
+};
+const filter2 = {
+    a: {$gte: 1, $lte: 2}
+};
+const sortPattern = {
+    a: -1
+};
+
+// Create a cache entry using 'filter1'.
+assert.eq(0, coll.getPlanCache().list().length, "Expected 0 cache entries");
+const filter1Result = coll.find(filter1).sort(sortPattern).toArray();
+const expectedFilter1Result = [{_id: 2, a: 2}];
+assert.eq(expectedFilter1Result, filter1Result);
+const cacheEntries = coll.getPlanCache().list();
+assert.eq(1, cacheEntries.length, cacheEntries);
+const cacheEntry = cacheEntries[0];
+
+// Verify that our cache entry is pinned and active.
+assert(cacheEntry.isPinned, cacheEntry);
+assert(cacheEntry.isActive, cacheEntry);
+
+// Capture the results for 'filter2' and verify that it used the same plan cache entry as 'filter1'.
+const cacheResults = coll.find(filter2).sort(sortPattern).toArray();
+const expectedFilter2Result = [{_id: 2, a: 2}, {_id: 1, a: 1}];
+assert.eq(cacheResults, expectedFilter2Result);
+
+// There should still be exactly one plan cache entry.
+assert.eq(1, coll.getPlanCache().list().length, cacheEntries);
+
+// The plan cache key and the query hashes of both queries should match.
+const explain = coll.find(filter2).sort(sortPattern).explain();
+const planCacheKey = cacheEntry.planCacheKey;
+assert.neq(null, planCacheKey, cacheEntry);
+assert.eq(planCacheKey, getPlanCacheKeyFromExplain(explain, db), explain);
+
+const queryHash = cacheEntry.queryHash;
+assert.neq(null, queryHash, cacheEntry);
+assert.eq(queryHash, getQueryHashFromExplain(explain, db), explain);
+
+// Clear the plan cache, and run 'filter2' again. This time, verify that we create a cache entry
+// with the same planCacheKey and queryHash as before.
+coll.getPlanCache().clear();
+assert.eq(0, coll.getPlanCache().list().length, "Expected 0 cache entries");
+const results = coll.find(filter2).sort(sortPattern).toArray();
+const newCacheEntries = coll.getPlanCache().list();
+assert.eq(1, newCacheEntries.length, "Expected 1 cache entry");
+const newCacheEntry = newCacheEntries[0];
+assert.eq(newCacheEntry.planCacheKey, planCacheKey, newCacheEntry);
+assert.eq(newCacheEntry.queryHash, queryHash, newCacheEntry);
+
+// The query should also return the same results as before.
+assert.eq(results, cacheResults);
+}());

jstests/core/timeseries/timeseries_collmod.js

@@ -211,5 +211,8 @@ if (TimeseriesTest.timeseriesScalabilityImprovementsEnabled(db.getMongo())) {
         "timeseries":
             {"bucketMaxSpanSeconds": bucketingValueMax, "bucketRoundingSeconds": bucketingValueMax}
     }));
+
+    // No-op command should succeed with empty time-series options.
+    assert.commandWorked(db.runCommand({"collMod": collName, "timeseries": {}}));
 }
 })();

jstests/core/timeseries/timeseries_delete_hint.js

@@ -73,7 +73,7 @@ const validateDeleteIndex = (docsToInsert,
                 : assert.commandWorked(
                       testDB.runCommand({delete: coll.getName(), deletes: deleteQuery}));
             assert.eq(res["n"], expectedNRemoved);
-            assert.docEq(expectedRemainingDocs, coll.find({}, {_id: 0}).toArray());
+            assert.sameMembers(expectedRemainingDocs, coll.find({}, {_id: 0}).toArray());
             assert(coll.drop());
         },
         docsToInsert,

jstests/core/timeseries/timeseries_filter_extended_range.js

@@ -4,8 +4,13 @@
  *
  * @tags: [
  *   requires_fcv_62,
+ *   # Refusing to run a test that issues an aggregation command with explain because it may
+ *   # return incomplete results if interrupted by a stepdown.
+ *   does_not_support_stepdowns,
  *   # We need a timeseries collection.
  *   requires_timeseries,
+ *   # Explain of a resolved view must be executed by mongos.
+ *   directly_against_shardsvrs_incompatible,
  * ]
  */
 
@@ -50,11 +55,13 @@ function runTest(underflow, overflow, query, results) {
 
     const pipeline = [{$match: query}, {$project: {_id: 0, [timeFieldName]: 1}}];
 
+    const plan = tsColl.explain().aggregate(pipeline);
+
     // Verify agg pipeline. We don't want to go through a plan that encourages a sort order to
     // avoid BUS and index selection, so we sort after gathering the results.
     const aggActuals = tsColl.aggregate(pipeline).toArray();
     aggActuals.sort(cmpTimeFields);
-    assert.docEq(results, aggActuals);
+    assert.docEq(results, aggActuals, JSON.stringify(plan, null, 4));
 
     // Verify the equivalent find command. We again don't want to go through a plan that
     // encourages a sort order to avoid BUS and index selection, so we sort after gathering the
@@ -153,33 +160,25 @@ runTest(true, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [
     {[timeFieldName]: new Date("2040-01-01")}
 ]);
 
-// Verify ranges that straddle the lower epoch work properly
+// Verify ranges that straddle the lower and upper epoch boundaries work properly.
 runTest(
     true, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [
         {[timeFieldName]: new Date("1965-01-01")},
         {[timeFieldName]: new Date("1975-01-01")},
     ]);
-
 runTest(
     false, true, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [
         {[timeFieldName]: new Date("1995-01-01")},
         {[timeFieldName]: new Date("2040-01-01")},
     ]);
-
-// TODO: SERVER-69952 Literals outside the epoch are currently compared to _id, generally,
-// so we cannot match against them. This will have to be fixed in a similar manner by determining
-// whether the compared dates can be outside the epoch range and not relying on _id in that case.
-//
-// The following scenarios fail:
-// runTest(
-//    false, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [
-//         {[timeFieldName]: new Date("1971-01-01")},
-//         {[timeFieldName]: new Date("1975-01-01")},
-//     ]);
-// runTest(
-//     false, false, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}},
-//     [
-//         {[timeFieldName]: new Date("1995-01-01")},
-//         {[timeFieldName]: new Date("2030-01-01")},
-//     ]);
-})();
+runTest(
+    false, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [
+        {[timeFieldName]: new Date("1971-01-01")},
+        {[timeFieldName]: new Date("1975-01-01")},
+    ]);
+runTest(
+    false, false, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [
+        {[timeFieldName]: new Date("1995-01-01")},
+        {[timeFieldName]: new Date("2030-01-01")},
+    ]);
+})();

jstests/core/timeseries/timeseries_groupby_reorder.js

@@ -38,7 +38,18 @@ if (!isMongos(db)) {
                  res.stages[1]);
 }
 
-const res = coll.aggregate([{$group: {_id: '$meta', accmin: {$min: '$b'}, accmax: {$max: '$c'}}}])
-                .toArray();
+let res = coll.aggregate([{$group: {_id: '$meta', accmin: {$min: '$b'}, accmax: {$max: '$c'}}}])
+              .toArray();
 assert.docEq([{"_id": null, "accmin": 1, "accmax": 3}], res);
+
+// Test SERVER-73822 fix: complex $min and $max (i.e. not just straight field refs) work correctly.
+res = coll.aggregate([{
+              $group: {
+                  _id: '$meta',
+                  accmin: {$min: {$add: ["$b", "$c"]}},
+                  accmax: {$max: {$add: ["$b", "$c"]}}
+              }
+          }])
+          .toArray();
+assert.docEq([{"_id": null, "accmin": 2, "accmax": 6}], res);
 })();

jstests/core/timeseries/timeseries_index_partial.js

@@ -128,7 +128,7 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression:
         const result = coll.aggregate([{$match: predicate}], {hint: {a: 1}}).toArray();
         const unindexed =
             coll.aggregate([{$_internalInhibitOptimization: {}}, {$match: predicate}]).toArray();
-        assert.docEq(result, unindexed);
+        assert.sameMembers(result, unindexed);
     }
     function checkPlanAndResults(predicate) {
         checkPlan(predicate);

jstests/core/timeseries/timeseries_predicates.js

@@ -19,7 +19,7 @@ const tsColl = db.timeseries_predicates_timeseries;
 coll.drop();
 tsColl.drop();
 assert.commandWorked(
-    db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'meta'}}));
+    db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'mt'}}));
 const bucketsColl = db.getCollection('system.buckets.' + tsColl.getName());
 
 // Test that 'predicate' behaves correctly on the example documents,
@@ -88,18 +88,18 @@ checkAllBucketings({x: {$exists: true}}, [
 
 // Test $or...
 {
-    // ... on metric + meta.
+    // ... on metric + mt.
     checkAllBucketings({
         $or: [
             {x: {$lt: 0}},
-            {'meta.y': {$gt: 0}},
+            {'mt.y': {$gt: 0}},
         ]
     },
                        [
-                           {x: +1, meta: {y: -1}},
-                           {x: +1, meta: {y: +1}},
-                           {x: -1, meta: {y: -1}},
-                           {x: -1, meta: {y: +1}},
+                           {x: +1, mt: {y: -1}},
+                           {x: +1, mt: {y: +1}},
+                           {x: -1, mt: {y: -1}},
+                           {x: -1, mt: {y: +1}},
                        ]);
 
     // ... when one argument can't be pushed down.
@@ -133,18 +133,18 @@ checkAllBucketings({x: {$exists: true}}, [
 
 // Test $and...
 {
-    // ... on metric + meta.
+    // ... on metric + mt.
     checkAllBucketings({
         $and: [
             {x: {$lt: 0}},
-            {'meta.y': {$gt: 0}},
+            {'mt.y': {$gt: 0}},
         ]
     },
                        [
-                           {x: +1, meta: {y: -1}},
-                           {x: +1, meta: {y: +1}},
-                           {x: -1, meta: {y: -1}},
-                           {x: -1, meta: {y: +1}},
+                           {x: +1, mt: {y: -1}},
+                           {x: +1, mt: {y: +1}},
+                           {x: -1, mt: {y: -1}},
+                           {x: -1, mt: {y: +1}},
                        ]);
 
     // ... when one argument can't be pushed down.
@@ -182,28 +182,28 @@ checkAllBucketings({
     $or: [
         {
             $and: [
-                {'meta.a': {$gt: 0}},
+                {'mt.a': {$gt: 0}},
                 {'x': {$lt: 0}},
             ]
         },
         {
             $and: [
-                {'meta.b': {$gte: 0}},
+                {'mt.b': {$gte: 0}},
                 {time: {$gt: ISODate('2020-01-01')}},
             ]
         },
     ]
 },
                    [
-                       {meta: {a: -1, b: -1}, x: -1, time: ISODate('2020-02-01')},
-                       {meta: {a: -1, b: -1}, x: -1, time: ISODate('2019-12-31')},
-                       {meta: {a: -1, b: -1}, x: +1, time: ISODate('2020-02-01')},
-                       {meta: {a: -1, b: -1}, x: +1, time: ISODate('2019-12-31')},
+                       {mt: {a: -1, b: -1}, x: -1, time: ISODate('2020-02-01')},
+                       {mt: {a: -1, b: -1}, x: -1, time: ISODate('2019-12-31')},
+                       {mt: {a: -1, b: -1}, x: +1, time: ISODate('2020-02-01')},
+                       {mt: {a: -1, b: -1}, x: +1, time: ISODate('2019-12-31')},
 
-                       {meta: {a: +1, b: -1}, x: -1, time: ISODate('2020-02-01')},
-                       {meta: {a: +1, b: -1}, x: -1, time: ISODate('2019-12-31')},
-                       {meta: {a: +1, b: -1}, x: +1, time: ISODate('2020-02-01')},
-                       {meta: {a: +1, b: -1}, x: +1, time: ISODate('2019-12-31')},
+                       {mt: {a: +1, b: -1}, x: -1, time: ISODate('2020-02-01')},
+                       {mt: {a: +1, b: -1}, x: -1, time: ISODate('2019-12-31')},
+                       {mt: {a: +1, b: -1}, x: +1, time: ISODate('2020-02-01')},
+                       {mt: {a: +1, b: -1}, x: +1, time: ISODate('2019-12-31')},
                    ]);
 
 // Test nested $and / $or where some leaf predicates cannot be pushed down.
@@ -211,72 +211,72 @@ checkAllBucketings({
     $or: [
         {
             $and: [
-                {'meta.a': {$gt: 0}},
+                {'mt.a': {$gt: 0}},
                 {'x': {$exists: false}},
             ]
         },
         {
             $and: [
-                {'meta.b': {$gte: 0}},
+                {'mt.b': {$gte: 0}},
                 {time: {$gt: ISODate('2020-01-01')}},
             ]
         },
     ]
 },
                    [
-                       {meta: {a: -1, b: -1}, time: ISODate('2020-02-01')},
-                       {meta: {a: -1, b: -1}, time: ISODate('2019-12-31')},
-                       {meta: {a: -1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')},
-                       {meta: {a: -1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')},
+                       {mt: {a: -1, b: -1}, time: ISODate('2020-02-01')},
+                       {mt: {a: -1, b: -1}, time: ISODate('2019-12-31')},
+                       {mt: {a: -1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')},
+                       {mt: {a: -1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')},
 
-                       {meta: {a: +1, b: -1}, time: ISODate('2020-02-01')},
-                       {meta: {a: +1, b: -1}, time: ISODate('2019-12-31')},
-                       {meta: {a: +1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')},
-                       {meta: {a: +1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')},
+                       {mt: {a: +1, b: -1}, time: ISODate('2020-02-01')},
+                       {mt: {a: +1, b: -1}, time: ISODate('2019-12-31')},
+                       {mt: {a: +1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')},
+                       {mt: {a: +1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')},
                    ]);
 
-// Test $exists on meta, inside $or.
+// Test $exists on mt, inside $or.
 checkAllBucketings({
     $or: [
-        {"meta.a": {$exists: true}},
+        {"mt.a": {$exists: true}},
         {"x": {$gt: 2}},
     ]
 },
                    [
-                       {meta: {a: 1}, x: 1},
-                       {meta: {a: 2}, x: 2},
-                       {meta: {a: 3}, x: 3},
-                       {meta: {a: 4}, x: 4},
-                       {meta: {}, x: 1},
-                       {meta: {}, x: 2},
-                       {meta: {}, x: 3},
-                       {meta: {}, x: 4},
+                       {mt: {a: 1}, x: 1},
+                       {mt: {a: 2}, x: 2},
+                       {mt: {a: 3}, x: 3},
+                       {mt: {a: 4}, x: 4},
+                       {mt: {}, x: 1},
+                       {mt: {}, x: 2},
+                       {mt: {}, x: 3},
+                       {mt: {}, x: 4},
                    ]);
 
-// Test $in on meta, inside $or.
+// Test $in on mt, inside $or.
 checkAllBucketings({
     $or: [
-        {"meta.a": {$in: [1, 3]}},
+        {"mt.a": {$in: [1, 3]}},
         {"x": {$gt: 2}},
     ]
 },
                    [
-                       {meta: {a: 1}, x: 1},
-                       {meta: {a: 2}, x: 2},
-                       {meta: {a: 3}, x: 3},
-                       {meta: {a: 4}, x: 4},
-                       {meta: {}, x: 1},
-                       {meta: {}, x: 2},
-                       {meta: {}, x: 3},
-                       {meta: {}, x: 4},
+                       {mt: {a: 1}, x: 1},
+                       {mt: {a: 2}, x: 2},
+                       {mt: {a: 3}, x: 3},
+                       {mt: {a: 4}, x: 4},
+                       {mt: {}, x: 1},
+                       {mt: {}, x: 2},
+                       {mt: {}, x: 3},
+                       {mt: {}, x: 4},
                    ]);
 
-// Test geo predicates on meta, inside $or.
+// Test geo predicates on mt, inside $or.
 for (const pred of ['$geoWithin', '$geoIntersects']) {
     checkAllBucketings({
         $or: [
             {
-                "meta.location": {
+                "mt.location": {
                     [pred]: {
                         $geometry: {
                             type: "Polygon",
@@ -295,66 +295,90 @@ for (const pred of ['$geoWithin', '$geoIntersects']) {
         ]
     },
                        [
-                           {meta: {location: [1, 1]}, x: 1},
-                           {meta: {location: [1, 1]}, x: 2},
-                           {meta: {location: [1, 1]}, x: 3},
-                           {meta: {location: [1, 1]}, x: 4},
-                           {meta: {location: [5, 5]}, x: 1},
-                           {meta: {location: [5, 5]}, x: 2},
-                           {meta: {location: [5, 5]}, x: 3},
-                           {meta: {location: [5, 5]}, x: 4},
+                           {mt: {location: [1, 1]}, x: 1},
+                           {mt: {location: [1, 1]}, x: 2},
+                           {mt: {location: [1, 1]}, x: 3},
+                           {mt: {location: [1, 1]}, x: 4},
+                           {mt: {location: [5, 5]}, x: 1},
+                           {mt: {location: [5, 5]}, x: 2},
+                           {mt: {location: [5, 5]}, x: 3},
+                           {mt: {location: [5, 5]}, x: 4},
                        ]);
 }
 
-// Test $mod on meta, inside $or.
+// Test $mod on mt, inside $or.
 // $mod is an example of a predicate that we don't handle specially in time-series optimizations:
 // it can be pushed down if and only if it's on a metadata field.
 checkAllBucketings({
     $or: [
-        {"meta.a": {$mod: [2, 0]}},
+        {"mt.a": {$mod: [2, 0]}},
         {"x": {$gt: 4}},
     ]
 },
                    [
-                       {meta: {a: 1}, x: 1},
-                       {meta: {a: 2}, x: 2},
-                       {meta: {a: 3}, x: 3},
-                       {meta: {a: 4}, x: 4},
-                       {meta: {a: 5}, x: 5},
-                       {meta: {a: 6}, x: 6},
-                       {meta: {a: 7}, x: 7},
-                       {meta: {a: 8}, x: 8},
+                       {mt: {a: 1}, x: 1},
+                       {mt: {a: 2}, x: 2},
+                       {mt: {a: 3}, x: 3},
+                       {mt: {a: 4}, x: 4},
+                       {mt: {a: 5}, x: 5},
+                       {mt: {a: 6}, x: 6},
+                       {mt: {a: 7}, x: 7},
+                       {mt: {a: 8}, x: 8},
                    ]);
 
-// Test $elemMatch on meta, inside $or.
+// Test $elemMatch on mt, inside $or.
 checkAllBucketings({
     $or: [
-        {"meta.a": {$elemMatch: {b: 3}}},
+        {"mt.a": {$elemMatch: {b: 3}}},
         {"x": {$gt: 4}},
     ]
 },
                    [
-                       {x: 1, meta: {a: []}},
-                       {x: 2, meta: {a: [{b: 2}]}},
-                       {x: 3, meta: {a: [{b: 3}]}},
-                       {x: 4, meta: {a: [{b: 2}, {b: 3}]}},
-                       {x: 5, meta: {a: []}},
-                       {x: 6, meta: {a: [{b: 2}]}},
-                       {x: 7, meta: {a: [{b: 3}]}},
-                       {x: 8, meta: {a: [{b: 2}, {b: 3}]}},
+                       {x: 1, mt: {a: []}},
+                       {x: 2, mt: {a: [{b: 2}]}},
+                       {x: 3, mt: {a: [{b: 3}]}},
+                       {x: 4, mt: {a: [{b: 2}, {b: 3}]}},
+                       {x: 5, mt: {a: []}},
+                       {x: 6, mt: {a: [{b: 2}]}},
+                       {x: 7, mt: {a: [{b: 3}]}},
+                       {x: 8, mt: {a: [{b: 2}, {b: 3}]}},
                    ]);
 checkAllBucketings({
     $or: [
-        {"meta.a": {$elemMatch: {b: 2, c: 3}}},
+        {"mt.a": {$elemMatch: {b: 2, c: 3}}},
         {"x": {$gt: 3}},
     ]
 },
                    [
-                       {x: 1, meta: {a: []}},
-                       {x: 2, meta: {a: [{b: 2, c: 3}]}},
-                       {x: 3, meta: {a: [{b: 2}, {c: 3}]}},
-                       {x: 4, meta: {a: []}},
-                       {x: 5, meta: {a: [{b: 2, c: 3}]}},
-                       {x: 6, meta: {a: [{b: 2}, {c: 3}]}},
+                       {x: 1, mt: {a: []}},
+                       {x: 2, mt: {a: [{b: 2, c: 3}]}},
+                       {x: 3, mt: {a: [{b: 2}, {c: 3}]}},
+                       {x: 4, mt: {a: []}},
+                       {x: 5, mt: {a: [{b: 2, c: 3}]}},
+                       {x: 6, mt: {a: [{b: 2}, {c: 3}]}},
                    ]);
+
+// Test a standalone $elemMatch on mt.
+checkAllBucketings({"mt.a": {$elemMatch: {b: 3}}}, [
+    {mt: {a: []}},
+    {mt: {a: [{b: 2}]}},
+    {mt: {a: [{b: 3}]}},
+    {mt: {a: [{b: 2}, {b: 3}]}},
+    {mt: {a: []}},
+    {mt: {a: [{b: 2}]}},
+    {mt: {a: [{b: 3}]}},
+    {mt: {a: [{b: 2}, {b: 3}]}},
+]);
+
+// Test a standalone $size on mt.
+checkAllBucketings({"mt.a": {$size: 1}}, [
+    {mt: {a: []}},
+    {mt: {a: [{b: 2}]}},
+    {mt: {a: [{b: 3}]}},
+    {mt: {a: [{b: 2}, {b: 3}]}},
+    {mt: {a: []}},
+    {mt: {a: [{b: 2}]}},
+    {mt: {a: [{b: 3}]}},
+    {mt: {a: [{b: 2}, {b: 3}]}},
+]);
 })();

jstests/core/write/update/update_arrayFilters.js

@@ -6,6 +6,8 @@
 //   requires_non_retryable_writes,
 //   # TODO SERVER-30466
 //   does_not_support_causal_consistency,
+//   # Uses $where operator
+//   requires_scripting,
 // ]
 
 // Tests for the arrayFilters option to update and findAndModify.

jstests/disk/wt_validate_table_logging.js

@@ -8,15 +8,21 @@
 (function() {
 'use strict';
 
+load("jstests/libs/columnstore_util.js");  // For setUpServerForColumnStoreIndexTest.
+
 let conn = MongoRunner.runMongod();
 
 const dbpath = conn.dbpath;
 const dbName = jsTestName();
 const collName = 'coll';
 
+const csiEnabled = setUpServerForColumnStoreIndexTest(conn.getDB(dbName));
+
 const create = function(conn) {
     assert.commandWorked(conn.getDB(dbName).createCollection(collName));
-    assert.commandWorked(conn.getDB(dbName)[collName].createIndex({'$**': "columnstore"}));
+    if (csiEnabled) {
+        assert.commandWorked(conn.getDB(dbName)[collName].createIndex({'$**': "columnstore"}));
+    }
 };
 
 const collUri = function(conn) {
@@ -55,14 +61,16 @@ const primary = replTest.getPrimary();
 // Run validate as a replica set, which will expect the tables to not be logged.
 let res = assert.commandWorked(primary.getDB(dbName).runCommand({validate: collName}));
 assert(!res.valid);
-assert.eq(res.errors.length, 3);
+assert.eq(res.errors.length, csiEnabled ? 3 : 2);
 checkLog.containsJson(primary, 6898101, {uri: collUri(primary), expected: false});
 checkLog.containsJson(
     primary, 6898101, {index: '_id_', uri: indexUri(primary, '_id_'), expected: false});
-checkLog.containsJson(
-    primary,
-    6898101,
-    {index: '$**_columnstore', uri: indexUri(primary, '$**_columnstore'), expected: false});
+if (csiEnabled) {
+    checkLog.containsJson(
+        primary,
+        6898101,
+        {index: '$**_columnstore', uri: indexUri(primary, '$**_columnstore'), expected: false});
+}
 
 // Create the collection and indexes as a replica set, which will cause the tables to not be logged.
 assert.commandWorked(primary.getDB(dbName).runCommand({drop: collName}));
@@ -74,13 +82,15 @@ conn = MongoRunner.runMongod(nodeOptions);
 // Run validate as a standalone, which will expect the tables to be logged.
 res = assert.commandWorked(conn.getDB(dbName).runCommand({validate: collName}));
 assert(!res.valid);
-assert.eq(res.errors.length, 3);
+assert.eq(res.errors.length, csiEnabled ? 3 : 2);
 checkLog.containsJson(conn, 6898101, {uri: collUri(conn), expected: true});
 checkLog.containsJson(conn, 6898101, {index: '_id_', uri: indexUri(conn, '_id_'), expected: true});
-checkLog.containsJson(
-    conn,
-    6898101,
-    {index: '$**_columnstore', uri: indexUri(conn, '$**_columnstore'), expected: true});
+if (csiEnabled) {
+    checkLog.containsJson(
+        conn,
+        6898101,
+        {index: '$**_columnstore', uri: indexUri(conn, '$**_columnstore'), expected: true});
+}
 
 MongoRunner.stopMongod(conn, null, {skipValidation: true});
 }());

jstests/libs/analyze_plan.js

@@ -524,17 +524,36 @@ function assertStagesForExplainOfCommand({coll, cmdObj, expectedStages, stagesNo
 }
 
 /**
- * Get the "planCacheKey" from the explain result.
+ * Utility to obtain a value from 'explainRes' using 'getValueCallback'.
+ */
+function getFieldValueFromExplain(explainRes, getValueCallback) {
+    assert(explainRes.hasOwnProperty("queryPlanner"), explainRes);
+    const plannerOutput = explainRes.queryPlanner;
+    const fieldValue = getValueCallback(plannerOutput);
+    assert.eq(typeof fieldValue, "string");
+    return fieldValue;
+}
+
+/**
+ * Get the 'planCacheKey' from 'explainRes'.
  */
 function getPlanCacheKeyFromExplain(explainRes, db) {
-    const hash = FixtureHelpers.isMongos(db) &&
-            explainRes.queryPlanner.hasOwnProperty("winningPlan") &&
-            explainRes.queryPlanner.winningPlan.hasOwnProperty("shards")
-        ? explainRes.queryPlanner.winningPlan.shards[0].planCacheKey
-        : explainRes.queryPlanner.planCacheKey;
-    assert.eq(typeof hash, "string");
+    return getFieldValueFromExplain(explainRes, function(plannerOutput) {
+        return FixtureHelpers.isMongos(db) && plannerOutput.hasOwnProperty("winningPlan") &&
+                plannerOutput.winningPlan.hasOwnProperty("shards")
+            ? plannerOutput.winningPlan.shards[0].planCacheKey
+            : plannerOutput.planCacheKey;
+    });
+}
 
-    return hash;
+/**
+ * Get the 'queryHash' from 'explainRes'.
+ */
+function getQueryHashFromExplain(explainRes, db) {
+    return getFieldValueFromExplain(explainRes, function(plannerOutput) {
+        return FixtureHelpers.isMongos(db) ? plannerOutput.winningPlan.shards[0].queryHash
+                                           : plannerOutput.queryHash;
+    });
 }
 
 /**

jstests/libs/check_shard_filtering_metadata_helpers.js

@@ -87,12 +87,10 @@ var CheckShardFilteringMetadataHelpers = (function() {
         const configDB = mongosConn.getDB('config');
 
         // Check shards know correct database versions.
-        // Note: We can only check the dbVersion on the primary shards because non-primary shards
-        // can have stale dbVersions cached (for versions on which they were not primary.). TODO:
-        // Once shards became authoritative for their dbVersions, we can check all shards.
-        configDB.databases.find({primary: shardId}).forEach(configDatabasesEntry => {
-            checkDatabase(configDatabasesEntry);
-        });
+        // TODO: SERVER-73991 Reenable this check.
+        // configDB.databases.find({primary: shardId}).forEach(configDatabasesEntry => {
+        //     checkDatabase(configDatabasesEntry);
+        // });
 
         // Check that shards have correct filtering metadata for sharded collections.
         if (!skipCheckShardedCollections) {

jstests/libs/clustered_collections/clustered_collection_bounded_scan_common.js

@@ -94,6 +94,23 @@ const testClusteredCollectionBoundedScan = function(coll, clusterKey) {
         assert.eq(expectedNReturned, expl.executionStats.executionStages.nReturned);
         assert.eq(expectedDocsExamined, expl.executionStats.executionStages.docsExamined);
     }
+    function testIn() {
+        initAndPopulate(coll, clusterKey);
+
+        const expl = assert.commandWorked(coll.getDB().runCommand({
+            explain: {find: coll.getName(), filter: {[clusterKeyFieldName]: {$in: [10, 20, 30]}}},
+            verbosity: "executionStats"
+        }));
+
+        assert(getPlanStage(expl, "CLUSTERED_IXSCAN"));
+        assert.eq(10, getPlanStage(expl, "CLUSTERED_IXSCAN").minRecord);
+        assert.eq(30, getPlanStage(expl, "CLUSTERED_IXSCAN").maxRecord);
+
+        assert.eq(3, expl.executionStats.executionStages.nReturned);
+        // The range scanned is 21 documents + 1 extra document by design - additional cursor
+        // 'next' beyond the range.
+        assert.eq(22, expl.executionStats.executionStages.docsExamined);
+    }
     function testNonClusterKeyScan() {
         initAndPopulate(coll, clusterKey);
 
@@ -128,6 +145,8 @@ const testClusteredCollectionBoundedScan = function(coll, clusterKey) {
         testRange("$gte", 20, "$lt", 40, 20, 22);
         testRange("$gt", 20, "$lte", 40, 20, 22);
         testRange("$gte", 20, "$lte", 40, 21, 22);
+        testIn();
+
         testNonClusterKeyScan();
     }
 

jstests/libs/clustered_collections/clustered_collection_hint_common.js

@@ -196,6 +196,91 @@ function testClusteredCollectionHint(coll, clusterKey, clusterKeyName) {
             }
         });
 
+        // Find with $natural hints and sorts: we should scan the collection in the hinted
+        // direction regardless of sort direction, and provide a blocking sort if needed.
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: 1}, sort: {[clusterKeyFieldName]: 1}},
+            expectedWinningPlanStats: {
+                stage: "COLLSCAN",
+                direction: "forward",
+            },
+            unexpectedWinningPlanStats: ["SORT"]  // We shouldn't need a blocking sort here.
+        });
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: -1}, sort: {[clusterKeyFieldName]: 1}},
+            expectedWinningPlanStats: [
+                {stage: "SORT", sortPattern: {[clusterKeyFieldName]: 1}},
+                {
+                    stage: "COLLSCAN",
+                    direction: "backward",
+                }
+            ]
+        });
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: 1}, sort: {[clusterKeyFieldName]: -1}},
+            expectedWinningPlanStats: [
+                {stage: "SORT", sortPattern: {[clusterKeyFieldName]: -1}},
+                {
+                    stage: "COLLSCAN",
+                    direction: "forward",
+                }
+            ]
+        });
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: -1}, sort: {[clusterKeyFieldName]: -1}},
+            expectedWinningPlanStats: {
+                stage: "COLLSCAN",
+                direction: "backward",
+            },
+            unexpectedWinningPlanStats: ["SORT"]  // We shouldn't need a blocking sort here.
+        });
+
+        // We always need a blocking sort when the sort pattern does not match the provided sort for
+        // the clustered collection.
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: 1}, sort: {a: 1}},
+            expectedWinningPlanStats: [
+                {stage: "SORT", sortPattern: {a: 1}},
+                {
+                    stage: "COLLSCAN",
+                    direction: "forward",
+                }
+            ]
+        });
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: -1}, sort: {a: 1}},
+            expectedWinningPlanStats: [
+                {stage: "SORT", sortPattern: {a: 1}},
+                {
+                    stage: "COLLSCAN",
+                    direction: "backward",
+                }
+            ]
+        });
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: 1}, sort: {a: -1}},
+            expectedWinningPlanStats: [
+                {stage: "SORT", sortPattern: {a: -1}},
+                {
+                    stage: "COLLSCAN",
+                    direction: "forward",
+                }
+            ]
+        });
+        validateClusteredCollectionHint(coll, {
+            expectedNReturned: batchSize,
+            cmd: {find: collName, hint: {$natural: -1}, sort: {a: -1}},
+            expectedWinningPlanStats:
+                [{stage: "SORT", sortPattern: {a: -1}}, {stage: "COLLSCAN", direction: "backward"}],
+        });
+
         // Find on a standard index.
         validateClusteredCollectionHint(coll, {
             expectedNReturned: batchSize,
@@ -280,25 +365,39 @@ function testClusteredCollectionHint(coll, clusterKey, clusterKeyName) {
     return testHint(coll, clusterKey, clusterKeyName);
 }
 
-function validateClusteredCollectionHint(coll,
-                                         {expectedNReturned, cmd, expectedWinningPlanStats = {}}) {
+function validateClusteredCollectionHint(
+    coll,
+    {expectedNReturned, cmd, expectedWinningPlanStats = {}, unexpectedWinningPlanStats = []}) {
     const explain = assert.commandWorked(coll.runCommand({explain: cmd}));
     assert.eq(explain.executionStats.nReturned, expectedNReturned, tojson(explain));
 
     const actualWinningPlan = getWinningPlan(explain.queryPlanner);
-    const stageOfInterest = getPlanStage(actualWinningPlan, expectedWinningPlanStats.stage);
-    assert.neq(null, stageOfInterest);
 
-    for (const [key, value] of Object.entries(expectedWinningPlanStats)) {
-        assert(stageOfInterest[key] !== undefined, tojson(explain));
-        assert.eq(stageOfInterest[key], value, tojson(explain));
+    if (!Array.isArray(expectedWinningPlanStats)) {
+        expectedWinningPlanStats = [expectedWinningPlanStats];
     }
 
-    // Explicitly check that the plan is not bounded by default.
-    if (!expectedWinningPlanStats.hasOwnProperty("minRecord")) {
-        assert(!actualWinningPlan.hasOwnProperty("minRecord"), tojson(explain));
+    for (const excludedStage of unexpectedWinningPlanStats) {
+        const stageOfInterest = getPlanStage(actualWinningPlan, excludedStage);
+        assert.eq(null, stageOfInterest);
     }
-    if (!expectedWinningPlanStats.hasOwnProperty("maxRecord")) {
-        assert(!actualWinningPlan.hasOwnProperty("maxRecord"), tojson(explain));
+
+    for (const expectedWinningPlanStageStats of expectedWinningPlanStats) {
+        const stageOfInterest =
+            getPlanStage(actualWinningPlan, expectedWinningPlanStageStats.stage);
+        assert.neq(null, stageOfInterest);
+
+        for (const [key, value] of Object.entries(expectedWinningPlanStageStats)) {
+            assert(stageOfInterest[key] !== undefined, tojson(explain));
+            assert.eq(stageOfInterest[key], value, tojson(explain));
+        }
+
+        // Explicitly check that the plan is not bounded by default.
+        if (!expectedWinningPlanStageStats.hasOwnProperty("minRecord")) {
+            assert(!actualWinningPlan.hasOwnProperty("minRecord"), tojson(explain));
+        }
+        if (!expectedWinningPlanStageStats.hasOwnProperty("maxRecord")) {
+            assert(!actualWinningPlan.hasOwnProperty("maxRecord"), tojson(explain));
+        }
     }
 }

jstests/libs/columnstore_util.js

@@ -58,7 +58,14 @@ function safeToCreateColumnStoreIndexInCluster(nodes) {
             return false;
         }
 
-        break;
+        const getParamFeatureFlagRes = assert.commandWorked(
+            conn.adminCommand({getParameter: 1, featureFlagColumnstoreIndexes: 1}));
+        if (!getParamFeatureFlagRes.featureFlagColumnstoreIndexes ||
+            !getParamFeatureFlagRes.featureFlagColumnstoreIndexes["value"]) {
+            jsTestLog("Note: declining to create column store index, because " +
+                      "featureFlagColumnstoreIndexes is disabled");
+            return false;
+        }
     }
 
     return true;
@@ -81,9 +88,13 @@ function setUpServerForColumnStoreIndexTest(db) {
         return false;
     }
 
-    // Parallel tests cannot set these server parameters during execution due to the side effect of
-    // clearing the SBE plan cache, so the parallel test suite overrides this for us up front.
-    if (!(TestData || {}).isParallelTest) {
+    // TODO SERVER-75026: Re-enable CSI in parallel tests.
+    // Note that we should not fully enable columnscans during the parallel tests due to the side
+    // effect of clearing the SBE plan cache. Fully enabling column scans should only be done
+    // in non-parallel environments.
+    if ((TestData || {}).isParallelTest) {
+        return false;
+    } else {
         fullyEnableColumnScan(nodes);
     }
 

jstests/libs/feature_flag_util.js

@@ -32,7 +32,13 @@ var FeatureFlagUtil = class {
         } catch (err) {
             // Some db-like objects (e.g. ShardingTest.shard0) aren't supported by FixtureHelpers,
             // but we can replace it with an object that should work and try again.
-            setConn(db.getDB(db.defaultDB));
+            if (typeof db.getDB === typeof Function) {
+                setConn(db.getDB(db.defaultDB));
+            } else {
+                // Some db-like objects (e.g ShardedClusterFixture) have a getSiblingDB method
+                // instead of getDB, use that here to avoid an undefined error.
+                setConn(db.getSiblingDB(db.defaultDB));
+            }
         }
 
         if (user) {

jstests/multiVersion/genericSetFCVUsage/tenant_migration_donor_recipient_fcv_mismatch_after_failover.js

@@ -1,71 +0,0 @@
-/**
- * Tests that restarting a migration attempt after a failover fails if the donor and recipient no
- * longer share the same FCV.
- * @tags: [
- *   requires_majority_read_concern,
- *   incompatible_with_windows_tls,
- *   serverless,
- * ]
- */
-
-import {TenantMigrationTest} from "jstests/replsets/libs/tenant_migration_test.js";
-load("jstests/libs/fail_point_util.js");
-load("jstests/libs/uuid_util.js");       // for 'extractUUIDFromObject'
-load("jstests/libs/parallelTester.js");  // for 'Thread'
-
-function runTest(downgradeFCV) {
-    const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
-
-    const tenantId = ObjectId().str;
-    const dbName = tenantMigrationTest.tenantDB(tenantId, "testDB");
-    const collName = "testColl";
-
-    const donorPrimary = tenantMigrationTest.getDonorPrimary();
-    const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
-
-    tenantMigrationTest.insertDonorDB(dbName, collName);
-
-    const migrationId = UUID();
-    const migrationOpts = {
-        migrationIdString: extractUUIDFromObject(migrationId),
-        recipientConnString: tenantMigrationTest.getRecipientConnString(),
-        tenantId: tenantId,
-    };
-
-    // Configure a failpoint to have the recipient primary hang after a successful initial
-    // comparison.
-    const recipientDB = recipientPrimary.getDB(dbName);
-    const hangAfterFirstFCVcheck =
-        configureFailPoint(recipientDB, "fpAfterComparingRecipientAndDonorFCV", {action: "hang"});
-
-    // Start a migration and wait for recipient to hang at the failpoint.
-    assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
-    hangAfterFirstFCVcheck.wait();
-
-    // Downgrade the FCV for the donor set.
-    assert.commandWorked(donorPrimary.adminCommand({setFeatureCompatibilityVersion: downgradeFCV}));
-
-    // Step up a new node in the recipient set and trigger a failover. The new primary should
-    // attempt to resume cloning, but fail upon re-checking the FCVs.
-    const recipientRst = tenantMigrationTest.getRecipientRst();
-    const newRecipientPrimary = recipientRst.getSecondaries()[0];
-    recipientRst.awaitLastOpCommitted();
-    assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1}));
-    hangAfterFirstFCVcheck.off();
-    recipientRst.getPrimary();
-
-    // Make sure we see the FCV mismatch detection message on the recipient regardless.
-    checkLog.containsJson(newRecipientPrimary, 5382300);
-
-    // Upgrade again to check on the status of the migration from the donor's point of view.
-    assert.commandWorked(donorPrimary.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
-    TenantMigrationTest.assertAborted(
-        tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
-
-    tenantMigrationTest.stop();
-}
-
-runTest(lastContinuousFCV);
-if (lastContinuousFCV != lastLTSFCV) {
-    runTest(lastLTSFCV);
-}

jstests/multiVersion/genericSetFCVUsage/tenant_migration_save_fcv.js

@@ -1,93 +0,0 @@
-/**
- * Tests that we note down the recipient FCV at the beginning of a migration and that a change
- * in that FCV will abort the migration.
- * @tags: [
- *  requires_majority_read_concern,
- *  incompatible_with_windows_tls,
- *  serverless,
- *  ]
- */
-
-import {TenantMigrationTest} from "jstests/replsets/libs/tenant_migration_test.js";
-import {
-    isShardMergeEnabled,
-    makeX509OptionsForTest,
-    runMigrationAsync
-} from "jstests/replsets/libs/tenant_migration_util.js";
-
-load("jstests/libs/fail_point_util.js");
-load("jstests/libs/uuid_util.js");       // for 'extractUUIDFromObject'
-load("jstests/libs/parallelTester.js");  // for 'Thread'
-load("jstests/replsets/rslib.js");       // 'createRstArgs'
-
-function runTest(downgradeFCV) {
-    const recipientRst = new ReplSetTest({
-        nodes: 2,
-        name: jsTestName() + "_recipient",
-        nodeOptions: makeX509OptionsForTest().recipient
-    });
-
-    recipientRst.startSet();
-    recipientRst.initiate();
-
-    const tenantMigrationTest =
-        new TenantMigrationTest({name: jsTestName(), recipientRst: recipientRst});
-    const tenantId = ObjectId().str;
-    const dbName = tenantMigrationTest.tenantDB(tenantId, "testDB");
-    const collName = "testColl";
-
-    const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
-
-    tenantMigrationTest.insertDonorDB(dbName, collName);
-
-    const migrationId = UUID();
-    const migrationOpts = {
-        migrationIdString: extractUUIDFromObject(migrationId),
-        recipientConnString: tenantMigrationTest.getRecipientConnString(),
-        tenantId: tenantId,
-    };
-
-    // Configure a failpoint to have the recipient primary hang after taking note of its FCV.
-    const recipientDb = recipientPrimary.getDB(dbName);
-    const hangAfterSavingFCV = configureFailPoint(
-        recipientDb, "fpAfterRecordingRecipientPrimaryStartingFCV", {action: "hang"});
-
-    // Start a migration and wait for recipient to hang at the failpoint.
-    const donorRstArgs = createRstArgs(tenantMigrationTest.getDonorRst());
-    const migrationThread = new Thread(runMigrationAsync, migrationOpts, donorRstArgs);
-    migrationThread.start();
-    hangAfterSavingFCV.wait();
-
-    const isRunningMergeProtocol = isShardMergeEnabled(recipientDb);
-
-    // Downgrade the FCV for the recipient set.
-    assert.commandWorked(
-        recipientPrimary.adminCommand({setFeatureCompatibilityVersion: downgradeFCV}));
-
-    // Step up a new node in the recipient set and trigger a failover. The new primary should
-    // attempt to resume cloning, but fail upon re-checking the FCV.
-    const newRecipientPrimary = recipientRst.getSecondaries()[0];
-    recipientRst.awaitLastOpCommitted();
-    assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1}));
-    hangAfterSavingFCV.off();
-    recipientRst.getPrimary();
-
-    // The migration will not be able to continue in the downgraded version.
-    TenantMigrationTest.assertAborted(migrationThread.returnData());
-    // Change-of-FCV detection message.
-    if (isRunningMergeProtocol && MongoRunner.compareBinVersions(downgradeFCV, "5.2") < 0) {
-        // FCV is too old for shard merge.
-        checkLog.containsJson(newRecipientPrimary, 5949504);
-    } else {
-        // Can't change FCVs during a migration.
-        checkLog.containsJson(newRecipientPrimary, 5356200);
-    }
-
-    tenantMigrationTest.stop();
-    recipientRst.stopSet();
-}
-
-runTest(lastContinuousFCV);
-if (lastContinuousFCV != lastLTSFCV) {
-    runTest(lastLTSFCV);
-}

jstests/multiVersion/targetedTestsLastLtsFeatures/recordPreImages_option_upgrade.js

@@ -0,0 +1,69 @@
+/**
+ * Verifies that the server ignores collection option "recordPreImages" on binary upgrade from the
+ * last LTS version to the current, as well as removes the option from collection attributes on
+ * FCV upgrade.
+ */
+(function() {
+"use strict";
+load('jstests/multiVersion/libs/multi_rs.js');
+
+const lastLTSVersion = "last-lts";
+const latestVersion = "latest";
+
+// Setup a two-node replica set with last LTS binaries, so it is possible to create a collection
+// with "recordPreImages" option.
+const rst = new ReplSetTest(
+    {name: jsTestName(), nodes: [{binVersion: lastLTSVersion}, {binVersion: lastLTSVersion}]});
+rst.startSet();
+rst.initiate();
+const testDB = rst.getPrimary().getDB("test");
+const primaryNode = rst.getPrimary();
+const secondaryNode = rst.getSecondary();
+
+// Create the collection.
+const collectionName = "coll";
+assert.commandWorked(testDB.createCollection(collectionName, {recordPreImages: true}));
+let coll = testDB[collectionName];
+
+// Insert a test document which will be updated to trigger recording of change stream pre-images.
+assert.commandWorked(coll.insert({_id: 1, a: 1}));
+assert.commandWorked(coll.updateOne({_id: 1}, {$inc: {a: 1}}));
+rst.awaitReplication();
+
+// Upgrade the binary of the secondary node to the current version to setup a mixed binary cluster.
+rst.upgradeMembers([secondaryNode], {binVersion: latestVersion});
+
+// Make sure the primary node did not change.
+rst.stepUp(primaryNode);
+
+// Verify that recording of change stream pre-images succeeds.
+assert.commandWorked(coll.updateOne({_id: 1}, {$inc: {a: 1}}));
+rst.awaitReplication();
+
+// Finally upgrade the binary of the primary node to the current version.
+rst.upgradePrimary(rst.getPrimary(), {binVersion: latestVersion});
+
+// Update a document on the collection with inactive "recordPreImages" collection option.
+coll = rst.getPrimary().getDB("test")[collectionName];
+assert.commandWorked(coll.updateOne({_id: 1}, {$inc: {a: 1}}));
+rst.awaitReplication();
+
+// Upgrade the FCV to the latest to trigger removal of "recordPreImages" collection option from
+// persistent catalog entries.
+assert.commandWorked(rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+// To check the collection options, downgrade FCV to later replace the binary of the server with
+// the last LTS binary version.
+assert.commandWorked(rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+rst.upgradeSet({binVersion: lastLTSVersion});
+
+// Verify that collection option "recordPreImages" was removed.
+const result =
+    assert.commandWorked(rst.getPrimary().getDB("test").runCommand({listCollections: 1}));
+assert.eq(result.cursor.firstBatch[0].name, collectionName);
+assert.docEq(
+    {},
+    result.cursor.firstBatch[0].options,
+    `Collection option "recordPreImages" was not removed. Got response: ${tojson(result)}`);
+rst.stopSet();
+})();

jstests/noPassthrough/clustered_collection_sorted_scan.js

@@ -15,7 +15,7 @@ const testConnection =
 const testDb = testConnection.getDB('local');
 const collectionSize = 10;
 const clusteredCollName = "clustered_index_sorted_scan_coll";
-const clusterField = "clusterKey";
+const clusterField = "_id";
 
 let nonClusteredCollName = clusteredCollName + "_nc";
 
@@ -26,7 +26,6 @@ let clusteredColl = testDb[clusteredCollName];
 
 // Generate a non-clustered collection for comparison
 assert.commandWorked(testDb.createCollection(nonClusteredCollName));
-assert.commandWorked(testDb[nonClusteredCollName].createIndex({[clusterField]: 1}, {unique: true}));
 let nonClusteredColl = testDb[nonClusteredCollName];
 
 // Put something in the collections so the planner has something to chew on.
@@ -72,12 +71,16 @@ function runTest(isClustered, hasFilter, hasHint, direction) {
     assert(!planHasStage(testDb, plan, "SORT"), "Unexpected sort in " + formatParamsAndPlan(plan));
 }
 
-function testCollations(direction) {
+function testCollations(collectionCollation, queryCollation, direction) {
+    const collationsMatch = collectionCollation == queryCollation;
+
     let strCollName = clusteredCollName + "_str";
 
     // Generate a clustered collection for the remainder of the testing
-    assert.commandWorked(testDb.createCollection(
-        strCollName, {clusteredIndex: {key: {[clusterField]: 1}, unique: true}}));
+    assert.commandWorked(testDb.createCollection(strCollName, {
+        clusteredIndex: {key: {[clusterField]: 1}, unique: true},
+        collation: collectionCollation
+    }));
 
     let tsColl = testDb[strCollName];
 
@@ -86,22 +89,120 @@ function testCollations(direction) {
         assert.commandWorked(tsColl.insert({[clusterField]: i.toString(), a: Math.random()}));
     }
 
-    // Run query with Faroese collation, just to choose something unlikely.
-    // Because the collations don't match, we can't use the clustered index
-    // to provide a sort
-    let plan = tsColl.find()
-                   .sort({[clusterField]: direction})
-                   .collation({locale: "fo", caseLevel: true})
-                   .explain();
-    assert(planHasStage(testDb, plan, "SORT"), "Expected sort in " + tojson(plan));
+    function runExplain(filter) {
+        return tsColl.find(filter)
+            .sort({[clusterField]: direction})
+            .collation(queryCollation)
+            .explain();
+    }
 
-    // However, if we can exclude strings, we don't need an explicit sort even
-    // if the collations don't match
-    plan = tsColl.find({[clusterField]: {$gt: -1}})
-               .sort({[clusterField]: direction})
-               .collation({locale: "fo", caseLevel: true})
-               .explain();
+    //
+    // Some queries need an explicit sort only when the query/collection collations do not match.
+    //
+    function assertPlanOnlyHasSortIfCollationsDontMatch(plan) {
+        if (collationsMatch) {
+            assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+        } else {
+            assert(planHasStage(testDb, plan, "SORT"), "Expected sort in " + tojson(plan));
+        }
+    }
+
+    // Empty match.
+    let plan = runExplain({});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Comparison against a field other than the cluster field.
+    plan = runExplain({a: {$lt: 2}});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Query which contains an unsupported match expression.
+    plan = runExplain({$or: [{[clusterField]: {$lt: 2}}, {[clusterField]: {$gt: 5}}]});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Conjunction with one child which is an unsupported match expression and another which is a
+    // comparison against a field other than the cluster field.
+    plan = runExplain(
+        {$and: [{$or: [{[clusterField]: {$lt: 2}}, {[clusterField]: {$gt: 5}}]}, {a: {$gt: -1}}]});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Match which compares the cluster field to a string.
+    plan = runExplain({[clusterField]: {$gt: "1"}});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Match which compares the cluster field to an object containing a string.
+    plan = runExplain({[clusterField]: {$eq: {a: "str"}}});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Match which compares the cluster field to an array containing a string.
+    plan = runExplain({[clusterField]: {$eq: [1, 2, "str"]}});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // $in query where one of the elements is a string.
+    plan = runExplain({[clusterField]: {$in: [1, "2", 3]}});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Conjunction with one child which compares the cluster field to a string and another which
+    // is a comparison against a field other than the cluster field.
+    plan = runExplain({$and: [{[clusterField]: "str"}, {a: 5}]});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    // Conjunction with one $in child which compares the cluster field to a string and another
+    // which is a comparison against a field other than the cluster field.
+    plan = runExplain({$and: [{[clusterField]: {$in: [1, "2", 3]}}, {a: 5}]});
+    assertPlanOnlyHasSortIfCollationsDontMatch(plan);
+
+    //
+    // Some queries can omit the explicit sort regardless of collations. This is the case when
+    // we can exclude string values of the cluster key in the output.
+    //
+
+    // Simple comparison on cluster key which omits strings.
+    plan = runExplain({[clusterField]: {$gt: -1}});
     assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+    plan = runExplain({[clusterField]: {$eq: {a: 5}}});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+    plan = runExplain({[clusterField]: {$eq: [1, 2, 3]}});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // Conjunction with multiple comparisons on cluster key which omits strings.
+    plan = runExplain({$and: [{[clusterField]: {$gt: -1}}, {[clusterField]: {$lt: 10}}]});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // $in query against cluster key which omits strings.
+    plan = runExplain({[clusterField]: {$in: [1, 2, 3]}});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // Conjunction of $in query against cluster key and another comparison on a field other than
+    // the cluster key. The first conjunct omits strings.
+    plan = runExplain({$and: [{[clusterField]: {$in: [1, 2, 3]}}, {a: 5}]});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // Conjunction with one comparison against the cluster key and one against another field. The
+    // second conjunct omits strings.
+    plan = runExplain({$and: [{a: {$lt: 2}}, {[clusterField]: {$gt: -1}}]});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // Conjunction with one child which is an unsupported match expression and another which is
+    // a comparison against the cluster field. The second conjunct omits strings.
+    plan = runExplain({
+        $and: [
+            {$or: [{[clusterField]: {$lt: 2}}, {[clusterField]: {$gt: 5}}]},
+            {[clusterField]: {$gt: -1}}
+        ]
+    });
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // Conjunction which contains a comparison of the cluster field to a string and a comparison
+    // of the cluster field to a number. The second conjunct omits strings.
+    plan = runExplain({$and: [{[clusterField]: {$lt: "1"}}, {[clusterField]: {$gt: 2}}]});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
+    // Conjunction which contains a $in comparison of the cluster field to a string and a $in
+    // comparison of the cluster field to a number. The second conjunct omits strings.
+    plan = runExplain(
+        {$and: [{[clusterField]: {$in: [1, "2", 3]}}, {[clusterField]: {$in: [1, 3, 4]}}]});
+    assert(!planHasStage(testDb, plan, "SORT"), "Unxpected sort in " + tojson(plan));
+
     tsColl.drop();
 }
 
@@ -146,7 +247,7 @@ function testPlanCache(direction) {
     assert.commandWorked(clusteredColl.createIndex({a: 1}, {name: indexName}));
 
     const filter = {a: {$gt: -1}};
-    const projection = {_id: 0, [clusterField]: 1};
+    const projection = {[clusterField]: 1};
     const sort = {[clusterField]: direction};
 
     // Because of the _a index above, we should have two alternatves -- filter via the
@@ -190,8 +291,31 @@ for (let isClustered = 0; isClustered <= 1; isClustered++) {
     }
 }
 
-testCollations(/* direction = */ 1);
-testCollations(/* direction = */ -1);
+//
+// Show that the direction of the sort does not affect the plans we are able to provide. Also show
+// the collation conditions under which we can avoid explicit sorts in the final plan.
+//
+
+const defaultCollation = {
+    locale: "simple",
+};
+const faroeseCollation = {
+    locale: "fo",
+    caseLevel: true
+};
+
+testCollations(
+    defaultCollation /* for collection */, faroeseCollation /* for query */, /* direction = */ 1);
+testCollations(
+    defaultCollation /* for collection */, faroeseCollation /* for query */, /* direction = */ -1);
+testCollations(
+    faroeseCollation /* for collection */, faroeseCollation /* for query */, /* direction = */ 1);
+testCollations(
+    faroeseCollation /* for collection */, faroeseCollation /* for query */, /* direction = */ -1);
+testCollations(
+    defaultCollation /* for collection */, defaultCollation /* for query */, /* direction = */ 1);
+testCollations(
+    defaultCollation /* for collection */, defaultCollation /* for query */, /* direction = */ -1);
 
 testMinMax();
 

jstests/noPassthrough/column_scan_slow_logs.js

@@ -2,7 +2,8 @@
  * Tests to ensure that COLUMN_SCAN plan and scanned columns appear in slow query log lines when
  * the columstore index is the winning plan.
  * @tags: [
- *   requires_fcv_63,
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  * ]
  */
 (function() {

jstests/noPassthrough/column_store_index_load.js

@@ -4,10 +4,11 @@
  * Indexes are validated by comparing query results that use the index with results from a control
  * query that uses a collection scan.
  * @tags: [
- *   requires_fcv_63,
  *   # We could potentially need to resume an index build in the event of a stepdown, which is not
  *   # yet implemented.
  *   does_not_support_stepdowns,
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  * ]
  */
 (function() {

jstests/noPassthrough/columnstore_index_persistence.js

@@ -4,9 +4,11 @@
  * as well as the createIndex response when a columnstore index is created.
  *
  * @tags: [
- *   requires_fcv_63,
  *   requires_persistence,
  *   requires_replication,
+ *   # column store indexes are still under a feature flag and require full sbe
+ *   featureFlagColumnstoreIndexes,
+ *   featureFlagSbeFull,
  * ]
  */