SERVER-67405: Fix is_patch lookup for build_variant_gen

(cherry picked from commit 1260ae9d64)

README.md

@@ -90,3 +90,5 @@ Welcome to MongoDB!
   October 16, 2018, including patch fixes for prior versions, are published
   under the [Server Side Public License (SSPL) v1](LICENSE-Community.txt).
   See individual files for details.
+
+

SConstruct

@@ -2,7 +2,6 @@
 
 import atexit
 import copy
-import datetime
 import errno
 import json
 import os
@@ -953,6 +952,11 @@ env_vars.Add('MSVC_VERSION',
     help='Sets the version of Visual C++ to use (e.g. 14.2 for VS2019, 14.3 for VS2022)',
     default="14.3")
 
+env_vars.Add('NINJA_BUILDDIR',
+    help="Location for shared Ninja state",
+    default="$BUILD_DIR/ninja",
+)
+
 env_vars.Add('NINJA_PREFIX',
     default="build",
     help="""A prefix to add to the beginning of generated ninja
@@ -1198,7 +1202,6 @@ envDict = dict(BUILD_ROOT=buildDir,
                LIBDEPS_TAG_EXPANSIONS=[],
                )
 
-
 # By default, we will get the normal SCons tool search. But if the
 # user has opted into the next gen tools, add our experimental tool
 # directory into the default toolpath, ahead of whatever is already in
@@ -4541,6 +4544,17 @@ if 'CCACHE' in env and env['CCACHE']:
 if 'ICECC' in env and env['ICECC']:
     env['ICECREAM_VERBOSE'] = env.Verbose()
     env['ICECREAM_TARGET_DIR'] = '$BUILD_ROOT/scons/icecream'
+
+    # Posssibly multiple ninja files are in play, and there are cases where ninja will
+    # use the wrong icecc run script, so we must create a unique script per ninja variant
+    # for ninja to track separately. We will use the variant dir which contains the each
+    # separate ninja builds meta files. This has to be under an additional flag then just
+    # ninja disabled, because the run icecc script is generated under a context where ninja
+    # is always disabled via the scons callback mechanism. The __NINJA_NO flag is intended
+    # to differentiate this particular context.
+    if env.get('__NINJA_NO') or get_option('ninja') != 'disabled':
+        env['ICECREAM_RUN_SCRIPT_SUBPATH'] = '$VARIANT_DIR'
+
     icecream = Tool('icecream')
     if not icecream.exists(env):
         env.FatalError(f"Failed to load icecream tool with ICECC={env['ICECC']}")
@@ -4615,7 +4629,7 @@ if get_option('ninja') != 'disabled':
             env.FatalError("Use of ccache is mandatory with --ninja and icecream older than 1.2. You are running {}.".format(env['ICECREAM_VERSION']))
 
     ninja_builder = Tool("ninja")
-    env["NINJA_BUILDDIR"] = env.Dir("$BUILD_DIR/ninja")
+    env["NINJA_BUILDDIR"] = env.Dir("$NINJA_BUILDDIR")
     ninja_builder.generate(env)
 
     ninjaConf = Configure(env, help=False, custom_tests = {
@@ -4779,6 +4793,8 @@ if get_option('ninja') != 'disabled':
     )
     env.NinjaRegisterFunctionHandler("test_list_builder_action", ninja_test_list_builder)
 
+    env['NINJA_GENERATED_SOURCE_ALIAS_NAME'] = 'generated-sources'
+
 
 if get_option('separate-debug') == "on" or env.TargetOSIs("windows"):
 

buildscripts/benchmarks/threshold_config.yml

@@ -27,4 +27,3 @@ overrides:
   benchmarks:
   benchmarks_sharding:
   benchmarks_bsoncolumn:
-  benchmarks_cst:

buildscripts/evergreen_gen_build_variant.py

@@ -59,7 +59,7 @@ class EvgExpansions(BaseModel):
     build_id: str
     build_variant: str
     exec_timeout_secs: Optional[int] = None
-    is_patch: Optional[bool]
+    is_patch: Optional[str]
     project: str
     max_tests_per_suite: Optional[int] = 100
     max_sub_suites: Optional[int] = 5
@@ -83,10 +83,21 @@ class EvgExpansions(BaseModel):
 
     def get_max_sub_suites(self) -> int:
         """Get the max_sub_suites to use."""
-        if not self.is_patch:
+        if not self.determine_is_patch():
             return self.mainline_max_sub_suites
         return self.max_sub_suites
 
+    def determine_is_patch(self) -> bool:
+        """
+        Determine if expansions indicate whether the script is being run in a patch build.
+
+        In a patch build, the `is_patch` expansion will be the string value of "true". In a
+        non-patch setting, it will not exist, or be an empty string.
+
+        :return: True if task is being run in a patch build.
+        """
+        return self.is_patch is not None and self.is_patch.lower() == "true"
+
     def build_suite_split_config(self, start_date: datetime,
                                  end_date: datetime) -> SuiteSplitConfig:
         """
@@ -113,7 +124,7 @@ class EvgExpansions(BaseModel):
         """
         return GenTaskOptions(
             create_misc_suite=True,
-            is_patch=self.is_patch,
+            is_patch=self.determine_is_patch(),
             generated_config_dir=GENERATED_CONFIG_DIR,
             use_default_timeouts=False,
             timeout_secs=self.timeout_secs,

buildscripts/evergreen_task_timeout.py

@@ -41,6 +41,9 @@ SPECIFIC_TASK_OVERRIDES = {
         "replica_sets_jscore_passthrough": timedelta(hours=2, minutes=30),
     },
     "ubuntu1804-debug-suggested": {"replica_sets_jscore_passthrough": timedelta(hours=3), },
+    "ubuntu1804-sbe-yielding-debug": {
+        "concurrency_replication_ubsan": timedelta(hours=2, minutes=30),
+    },
     "enterprise-rhel-80-64-bit-coverage": {
         "replica_sets_jscore_passthrough": timedelta(hours=2, minutes=30),
     },

buildscripts/generate_compile_expansions.py

@@ -61,8 +61,8 @@ def generate_version_expansions():
             raise ValueError("Unable to parse version from stdin and no version.json provided")
 
     if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
+        expansions["suffix"] = "v5.3-latest"
+        expansions["src_suffix"] = "v5.3-latest"
         expansions["is_release"] = "false"
     else:
         expansions["suffix"] = version_line

buildscripts/generate_compile_expansions_shared_cache.py

@@ -61,8 +61,8 @@ def generate_version_expansions():
             raise ValueError("Unable to parse version from stdin and no version.json provided")
 
     if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
+        expansions["suffix"] = "v5.3-latest"
+        expansions["src_suffix"] = "v5.3-latest"
         expansions["is_release"] = "false"
     else:
         expansions["suffix"] = version_line

buildscripts/idl/idl_check_compatibility.py

@@ -118,6 +118,9 @@ ALLOW_ANY_TYPE_LIST: List[str] = [
     'aggregate-param-fromMongos',
     'aggregate-param-$_requestReshardingResumeToken',
     'aggregate-param-isMapReduceCommand',
+    'count-param-hint',
+    'count-param-limit',
+    'count-param-maxTimeMS',
     'find-param-filter',
     'find-param-projection',
     'find-param-sort',

buildscripts/resmokeconfig/suites/benchmarks.yml

@@ -11,8 +11,6 @@ selector:
   - build/install/bin/chunk_manager_refresh_bm*
   - build/install/bin/migration_chunk_cloner_source_legacy_bm*
   - build/install/bin/sharding_write_router_bm*
-  # These benchmarks are being run as part of the benchmarks_cst.yml test suite.
-  - build/install/bin/cst_bm*
   # These benchmarks are being run as part of the benchmarks_bsoncolumn.yml test suite.
   - build/install/bin/bsoncolumn_bm*
   - build/install/bin/simple8b_bm*

buildscripts/resmokeconfig/suites/benchmarks_cst.yml

@@ -1,15 +0,0 @@
-# This benchmark computes the performance of the grammar-based bison parser, which involves
-# building a concrete syntax tree (CST) and translating to the relevant execution tree.
-test_kind: benchmark_test
-
-selector:
-  root: build/benchmarks.txt
-  include_files:
-  # The trailing asterisk is for handling the .exe extension on Windows.
-  - build/**/system_resource_canary_bm*
-  - build/install/bin/cst_bm*
-
-executor:
-  config: {}
-  hooks:
-  - class: CombineBenchmarkResults

buildscripts/resmokelib/suitesconfig.py

@@ -1,15 +1,15 @@
 """Module for retrieving the configuration of resmoke.py test suites."""
 import collections
 import os
-
-from typing import List, Dict
+from threading import Lock
+from typing import Dict, List
 
 import buildscripts.resmokelib.utils.filesystem as fs
 from buildscripts.resmokelib import config as _config
-from buildscripts.resmokelib import errors
-from buildscripts.resmokelib import utils
+from buildscripts.resmokelib import errors, utils
 from buildscripts.resmokelib.testing import suite as _suite
 from buildscripts.resmokelib.utils import load_yaml_file
+from buildscripts.resmokelib.utils.dictionary import merge_dicts
 
 SuiteName = str
 
@@ -39,8 +39,7 @@ def get_named_suites() -> List[SuiteName]:
 
 def get_suite_files() -> Dict[str, str]:
     """Get the physical files defining these suites for parsing comments."""
-    return MatrixSuiteConfig.merge_dicts(ExplicitSuiteConfig.get_suite_files(),
-                                         MatrixSuiteConfig.get_suite_files())
+    return merge_dicts(ExplicitSuiteConfig.get_suite_files(), MatrixSuiteConfig.get_suite_files())
 
 
 def create_test_membership_map(fail_on_missing_selector=False, test_kind=None):
@@ -141,6 +140,7 @@ class SuiteConfigInterface:
 class ExplicitSuiteConfig(SuiteConfigInterface):
     """Class for storing the resmoke.py suite YAML configuration."""
 
+    _name_suites_lock = Lock()
     _named_suites = {}
 
     @classmethod
@@ -164,17 +164,18 @@ class ExplicitSuiteConfig(SuiteConfigInterface):
     @classmethod
     def get_named_suites(cls) -> Dict[str, str]:
         """Populate the named suites by scanning config_dir/suites."""
-        if not cls._named_suites:
-            suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
-            root = os.path.abspath(suites_dir)
-            files = os.listdir(root)
-            for filename in files:
-                (short_name, ext) = os.path.splitext(filename)
-                if ext in (".yml", ".yaml"):
-                    pathname = os.path.join(root, filename)
-                    cls._named_suites[short_name] = pathname
+        with cls._name_suites_lock:
+            if not cls._named_suites:
+                suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
+                root = os.path.abspath(suites_dir)
+                files = os.listdir(root)
+                for filename in files:
+                    (short_name, ext) = os.path.splitext(filename)
+                    if ext in (".yml", ".yaml"):
+                        pathname = os.path.join(root, filename)
+                        cls._named_suites[short_name] = pathname
 
-        return cls._named_suites
+            return cls._named_suites
 
     @classmethod
     def get_suite_files(cls):
@@ -237,7 +238,7 @@ class MatrixSuiteConfig(SuiteConfigInterface):
 
         if override_names:
             for override_name in override_names:
-                cls.merge_dicts(res, overrides[override_name])
+                merge_dicts(res, overrides[override_name])
 
         return res
 
@@ -290,20 +291,6 @@ class MatrixSuiteConfig(SuiteConfigInterface):
                                          suite_config)
         return cls._all_mappings
 
-    @classmethod
-    def merge_dicts(cls, dict1, dict2):
-        """Recursively merges dict2 into dict1."""
-        if not isinstance(dict1, dict) or not isinstance(dict2, dict):
-            return dict2
-        for k in dict2:
-            if dict2[k] is None:
-                dict1.pop(k)
-            elif k in dict1:
-                dict1[k] = cls.merge_dicts(dict1[k], dict2[k])
-            else:
-                dict1[k] = dict2[k]
-        return dict1
-
     @classmethod
     def __get_suite_files_in_dir(cls, target_dir):
         """Get the physical files defining these suites for parsing comments."""

buildscripts/resmokelib/testing/fixtures/fixturelib.py

@@ -1,4 +1,5 @@
 """Facade wrapping the resmokelib dependencies used by fixtures."""
+from typing import Dict
 
 from buildscripts.resmokelib import config
 from buildscripts.resmokelib import core
@@ -6,6 +7,7 @@ from buildscripts.resmokelib import errors
 from buildscripts.resmokelib import utils
 from buildscripts.resmokelib import logging
 from buildscripts.resmokelib.core import network
+from buildscripts.resmokelib.utils.dictionary import merge_dicts
 from buildscripts.resmokelib.utils.history import make_historic as _make_historic
 from buildscripts.resmokelib.testing.fixtures import _builder
 
@@ -94,6 +96,23 @@ class FixtureLib:
         """Return the next available port that fixture can use."""
         return network.PortAllocator.next_fixture_port(job_num)
 
+    SET_PARAMETERS_KEY = "set_parameters"
+
+    def merge_mongo_option_dicts(self, original: Dict, override: Dict):
+        """
+        Merge mongod/s options such that --setParameter is merged recursively.
+
+        Values from `original` are replaced in-place with those of `override` where they exist.
+        """
+        original_set_parameters = original.get(self.SET_PARAMETERS_KEY, {})
+        override_set_parameters = override.get(self.SET_PARAMETERS_KEY, {})
+
+        merged_set_parameters = merge_dicts(original_set_parameters, override_set_parameters)
+        original.update(override)
+        original[self.SET_PARAMETERS_KEY] = merged_set_parameters
+
+        return original
+
 
 class _FixtureConfig(object):  # pylint: disable=too-many-instance-attributes
     """Class that stores fixture configuration info."""

buildscripts/resmokelib/testing/fixtures/shardedcluster.py

@@ -8,7 +8,6 @@ import pymongo
 import pymongo.errors
 
 import buildscripts.resmokelib.testing.fixtures.interface as interface
-import buildscripts.resmokelib.utils.registry as registry
 
 
 class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-instance-attributes
@@ -281,7 +280,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         replset_config_options["configsvr"] = True
 
         mongod_options = self.mongod_options.copy()
-        mongod_options.update(
+        mongod_options = self.fixturelib.merge_mongo_option_dicts(
+            mongod_options,
             self.fixturelib.make_historic(configsvr_options.pop("mongod_options", {})))
         mongod_options["configsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "config")
@@ -321,8 +321,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         replset_config_options["configsvr"] = False
 
         mongod_options = self.mongod_options.copy()
-        mongod_options.update(
-            self.fixturelib.make_historic(shard_options.pop("mongod_options", {})))
+        mongod_options = self.fixturelib.merge_mongo_option_dicts(
+            mongod_options, self.fixturelib.make_historic(shard_options.pop("mongod_options", {})))
         mongod_options["shardsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "shard{}".format(index))
         mongod_options["replSet"] = self._SHARD_REPLSET_NAME_PREFIX + str(index)

buildscripts/resmokelib/utils/dictionary.py

@@ -0,0 +1,17 @@
+"""Utility functions for working with Dict-type structures."""
+from typing import MutableMapping
+
+
+def merge_dicts(dict1, dict2):
+    """Recursively merges dict2 into dict1."""
+    if not (isinstance(dict1, MutableMapping) and isinstance(dict2, MutableMapping)):
+        return dict2
+
+    for k in dict2.keys():
+        if dict2[k] is None:
+            dict1.pop(k)
+        elif k in dict1:
+            dict1[k] = merge_dicts(dict1[k], dict2[k])
+        else:
+            dict1[k] = dict2[k]
+    return dict1

buildscripts/setup_spawnhost_coredump

@@ -2,15 +2,69 @@
 
 cd $HOME # workaround EVG-12829
 
-# Communicate to users that logged in before the script started that nothing is ready.
-wall "The setup_spawnhost_coredump script has just started setting up the debugging environment."
+unameOut=$(uname -s)
+case "${unameOut}" in
+    Linux*)     machine=Linux;;
+    Darwin*)    machine=Mac;;
+    CYGWIN*)    machine=Cygwin;;
+    *)          machine="UNKNOWN:${unameOut}"
+esac
 
-# Write this file that gets cat'ed on login to communicate to users logging in if this setup script is still running.
-echo '+-----------------------------------------------------------------+' > ~/.setup_spawnhost_coredump_progress
-echo '| The setup script is still setting up data files for inspection. |' >> ~/.setup_spawnhost_coredump_progress
-echo '+-----------------------------------------------------------------+' >> ~/.setup_spawnhost_coredump_progress
+if [[ "${machine}" = "Cygwin" ]]; then
+    out_dir="/cygdrive/c/setup_script_output.txt"
+    desktop_dir="/cygdrive/c/Users/Administrator/Desktop"
 
-cat >> ~/.profile <<EOF
+    {
+        date
+        env
+
+        echo "----------------------"
+        echo -e "\n=> Setting _NT_SOURCE_PATH environment variable for debuggers to pick up source files."
+        src_dir_hash=$(readlink -f /cygdrive/z/data/mci/source-*)
+        full_src_dir="${src_dir_hash}/src"
+        echo "Source Path: [${full_src_dir}]"
+        set -x;
+        setx _NT_SOURCE_PATH "${full_src_dir}"
+        { set +x; } 2>/dev/null
+
+        echo -e "\n=> Setting _NT_SYMBOL_PATH environment variable for debuggers to pick up the symbols."
+        sym_parent_dir=$(readlink -f /cygdrive/z/data/mci/artifacts-*dist_test_debug)
+        sym_dir=$(readlink -f ${sym_parent_dir}/debugsymbols-mongodb*zip)
+        sym_extracted_dir="${sym_parent_dir}/extracted_symbols"
+        full_sym_dir="${sym_extracted_dir}/dist-test/bin"
+        echo "Symbols Dir: [${full_sym_dir}]"
+
+        echo -e "\n=> Extracting Symbol files."
+        set -x;
+        mkdir ${sym_extracted_dir}
+        unzip -n ${sym_dir} -d ${sym_extracted_dir}
+        setx _NT_SYMBOL_PATH "${full_sym_dir};srv*;"
+        { set +x; } 2>/dev/null
+
+        echo -e "\n=> Extracting Core Dump to Desktop."
+        full_dump_dir=$(readlink -f /cygdrive/z/data/mci/artifacts-* | grep -v dist_test)
+        full_dump_parent_dir=$(readlink -f ${full_dump_dir}/mongo-coredumps*tgz)
+        extracted_dump_dir="${full_dump_dir}/extracted_dump"
+        set -x;
+        mkdir ${extracted_dump_dir}
+        tar -xzvf ${full_dump_parent_dir} -C ${extracted_dump_dir}
+        cp ${extracted_dump_dir}/* ${desktop_dir}
+        { set +x; } 2>/dev/null
+        echo "Copied to Desktop."
+
+    } &> ${out_dir}
+
+    cp ${out_dir} ${desktop_dir}
+else
+    # Communicate to users that logged in before the script started that nothing is ready.
+    wall "The setup_spawnhost_coredump script has just started setting up the debugging environment."
+
+    # Write this file that gets cat'ed on login to communicate to users logging in if this setup script is still running.
+    echo '+-----------------------------------------------------------------------------------+' > ~/.setup_spawnhost_coredump_progress
+    echo '| The setup script is still setting up data files for inspection on a [${machine}] host. |' >> ~/.setup_spawnhost_coredump_progress
+    echo '+-----------------------------------------------------------------------------------+' >> ~/.setup_spawnhost_coredump_progress
+
+    cat >> ~/.profile <<EOF
 cat ~/.setup_spawnhost_coredump_progress
 # Coredumps generated by a toolchain built mongodb can be problematic when examined with the system
 # gdb.
@@ -33,54 +87,54 @@ echo "To examine a core dump, type 'gdb ./<binary> ./<core file>'"
 cat ~/.setup_spawnhost_coredump_progress
 EOF
 
-export PATH=/opt/mongodbtoolchain/gdb/bin:$PATH
-echo 'if [ -f ~/.profile ]; then
+    export PATH=/opt/mongodbtoolchain/gdb/bin:$PATH
+    echo 'if [ -f ~/.profile ]; then
     . ~/.profile
 fi' >> .bash_profile
 
-# Make a directory on the larger EBS volume. Soft-link it under the home directory. The smaller home
-# volume can have trouble particularly with coredumps from sharded timeouts.
-mkdir /data/debug
-ln -s /data/debug .
-cd debug
+    # Make a directory on the larger EBS volume. Soft-link it under the home directory. The smaller home
+    # volume can have trouble particularly with coredumps from sharded timeouts.
+    mkdir /data/debug
+    ln -s /data/debug .
+    cd debug
 
-# As the name suggests, pretty printers. Primarily for boost::optional<T>
-git clone git@github.com:ruediger/Boost-Pretty-Printer.git &
+    # As the name suggests, pretty printers. Primarily for boost::optional<T>
+    git clone git@github.com:ruediger/Boost-Pretty-Printer.git &
 
-# Discover and unarchive necessary files and source code. This will put mongo binaries and their
-# partner .debug files in the same `debug/bin` directory. The `bin` directory will later be symbolic
-# linked into the top-level (`debug`) directory. Shared library files and their debug symbols will
-# be dumped into a `debug/lib` directory for tidiness. The mongo `<reporoot>/src/` directory is soft
-# linked as `debug/src`. The .gdbinit file assumes gdb is being run from the `debug` directory.
-BIN_ARCHIVE=`ls /data/mci/artifacts-*archive_dist_test*/mongo-*.tgz`
-tar --wildcards --strip-components=1 -xzf $BIN_ARCHIVE '*/bin/mongod' '*/bin/mongos' '*/bin/mongo' '*/bin/mongobridge' &
-tar --wildcards --strip-components=1 -xzf $BIN_ARCHIVE '*/lib/*' &
-DBG_ARCHIVE=`ls /data/mci/artifacts-*archive_dist_test_debug/debugsymbols-*.tgz`
-tar --wildcards --strip-components=1 -xzf $DBG_ARCHIVE '*/bin/mongod.debug' '*/bin/mongos.debug' '*/bin/mongo.debug' '*/bin/mongobridge.debug' &
-tar --wildcards --strip-components=1 -xzf $DBG_ARCHIVE '*/lib/*' &
-UNITTEST_ARCHIVE=`ls /data/mci/artifacts-*run_unittests/mongo-unittests-*.tgz`
-tar --wildcards --strip-components=0 -xzf $UNITTEST_ARCHIVE 'bin/*' &
-tar --wildcards -xzf $UNITTEST_ARCHIVE 'lib/*' &
+    # Discover and unarchive necessary files and source code. This will put mongo binaries and their
+    # partner .debug files in the same `debug/bin` directory. The `bin` directory will later be symbolic
+    # linked into the top-level (`debug`) directory. Shared library files and their debug symbols will
+    # be dumped into a `debug/lib` directory for tidiness. The mongo `<reporoot>/src/` directory is soft
+    # linked as `debug/src`. The .gdbinit file assumes gdb is being run from the `debug` directory.
+    BIN_ARCHIVE=`ls /data/mci/artifacts-*archive_dist_test*/mongo-*.tgz`
+    tar --wildcards --strip-components=1 -xzf $BIN_ARCHIVE '*/bin/mongod' '*/bin/mongos' '*/bin/mongo' '*/bin/mongobridge' &
+    tar --wildcards --strip-components=1 -xzf $BIN_ARCHIVE '*/lib/*' &
+    DBG_ARCHIVE=`ls /data/mci/artifacts-*archive_dist_test_debug/debugsymbols-*.tgz`
+    tar --wildcards --strip-components=1 -xzf $DBG_ARCHIVE '*/bin/mongod.debug' '*/bin/mongos.debug' '*/bin/mongo.debug' '*/bin/mongobridge.debug' &
+    tar --wildcards --strip-components=1 -xzf $DBG_ARCHIVE '*/lib/*' &
+    UNITTEST_ARCHIVE=`ls /data/mci/artifacts-*run_unittests/mongo-unittests-*.tgz`
+    tar --wildcards --strip-components=0 -xzf $UNITTEST_ARCHIVE 'bin/*' &
+    tar --wildcards -xzf $UNITTEST_ARCHIVE 'lib/*' &
 
-SRC_DIR=`find /data/mci/ -maxdepth 1 | grep source`
-ln -s $SRC_DIR/.gdbinit .
-ln -s $SRC_DIR/src src
-ln -s $SRC_DIR/buildscripts buildscripts
+    SRC_DIR=`find /data/mci/ -maxdepth 1 | grep source`
+    ln -s $SRC_DIR/.gdbinit .
+    ln -s $SRC_DIR/src src
+    ln -s $SRC_DIR/buildscripts buildscripts
 
-# Install pymongo to get the bson library for pretty-printers.
-/opt/mongodbtoolchain/v3/bin/pip3 install -r $SRC_DIR/etc/pip/dev-requirements.txt &
+    # Install pymongo to get the bson library for pretty-printers.
+    /opt/mongodbtoolchain/v3/bin/pip3 install -r $SRC_DIR/etc/pip/dev-requirements.txt &
 
-COREDUMP_ARCHIVE=`ls /data/mci/artifacts-*/mongo-coredumps-*.tgz`
-tar -xzf $COREDUMP_ARCHIVE &
-echo "Waiting for background processes to complete."
-wait
+    COREDUMP_ARCHIVE=`ls /data/mci/artifacts-*/mongo-coredumps-*.tgz`
+    tar -xzf $COREDUMP_ARCHIVE &
+    echo "Waiting for background processes to complete."
+    wait
 
-# Symbolic linking all of the executable files is sufficient for `gdb ./mongod ./dump_mongod.core`
-# to succeed. This inadvertantly also links in the ".debug" files which is unnecessary, but
-# harmless. gdb expects the .debug files to live adjacent to the physical binary.
-ln -s bin/* ./
+    # Symbolic linking all of the executable files is sufficient for `gdb ./mongod ./dump_mongod.core`
+    # to succeed. This inadvertantly also links in the ".debug" files which is unnecessary, but
+    # harmless. gdb expects the .debug files to live adjacent to the physical binary.
+    ln -s bin/* ./
 
-cat >> ~/.gdbinit <<EOF
+    cat >> ~/.gdbinit <<EOF
 set auto-load safe-path /
 set solib-search-path ./lib/
 set pagination off
@@ -96,10 +150,11 @@ boost.register_printers()
 end
 EOF
 
-echo "dir $HOME/debug" >> ~/.gdbinit
+    echo "dir $HOME/debug" >> ~/.gdbinit
 
-# Empty out the progress script that warns users about the set script still running when users log in.
-echo "" > ~/.setup_spawnhost_coredump_progress
-# Alert currently logged in users that this setup script has completed. Logging back in will ensure any
-# paths/environment variables will be set as intended.
-wall "The setup_spawnhost_coredump script has completed, please relogin to ensure the right environment variables are set."
+    # Empty out the progress script that warns users about the set script still running when users log in.
+    echo "" > ~/.setup_spawnhost_coredump_progress
+    # Alert currently logged in users that this setup script has completed. Logging back in will ensure any
+    # paths/environment variables will be set as intended.
+    wall "The setup_spawnhost_coredump script has completed, please relogin to ensure the right environment variables are set."
+fi

buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py

@@ -0,0 +1,65 @@
+"""Unittest for the resmokelib.testing.fixturelib.utils module"""
+
+import copy
+import unittest
+
+# pylint: disable=missing-docstring,protected-access
+from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
+
+
+class TestMergeMongoOptionDicts(unittest.TestCase):
+    def setUp(self) -> None:
+        self.under_test = FixtureLib()
+
+    def test_merge_empty(self):  # pylint: disable=no-self-use
+        original = {
+            "dbpath": "value0", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value1",
+                "param2": "value2",
+            }
+        }
+
+        override = {}
+        merged = self.under_test.merge_mongo_option_dicts(copy.deepcopy(original), override)
+
+        self.assertDictEqual(merged, original)
+
+    def test_merge_non_params(self):  # pylint: disable=no-self-use
+        non_param1_key = "non_param1"
+        non_param2_key = "non_param2"
+        original = {
+            non_param1_key: "value0", non_param2_key: {"nested_param1": "value0", },
+            self.under_test.SET_PARAMETERS_KEY: {"param1": "value1", }
+        }
+
+        override = {
+            non_param1_key: "value1",
+            non_param2_key: "value1",
+        }
+
+        self.under_test.merge_mongo_option_dicts(original, override)
+
+        expected = {
+            non_param1_key: "value1", non_param2_key: "value1",
+            self.under_test.SET_PARAMETERS_KEY: {"param1": "value1", }
+        }
+        self.assertEqual(original, expected)
+
+    def test_merge_params(self):  # pylint: disable=no-self-use
+        original = {
+            "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value",
+                "param2": {"param3": "value", },
+            }
+        }
+
+        override = {self.under_test.SET_PARAMETERS_KEY: {"param2": {"param3": {"param4": "value"}}}}
+        self.under_test.merge_mongo_option_dicts(original, override)
+
+        expected = {
+            "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value", "param2": {"param3": {"param4": "value"}}
+            }
+        }
+
+        self.assertDictEqual(original, expected)

buildscripts/tests/test_evergreen_gen_build_variant.py

@@ -87,7 +87,7 @@ def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_tas
 class TestEvgExpansions(unittest.TestCase):
     def test_get_max_sub_suites_should_use_patch_value_in_patches(self):
         evg_expansions = under_test.EvgExpansions(
-            is_patch=True,
+            is_patch="true",
             max_sub_suites=5,
             mainline_max_sub_suites=1,
             build_id="build_id",
@@ -102,7 +102,7 @@ class TestEvgExpansions(unittest.TestCase):
 
     def test_get_max_sub_suites_should_use_mainline_value_in_non_patches(self):
         evg_expansions = under_test.EvgExpansions(
-            is_patch=False,
+            is_patch="false",
             max_sub_suites=5,
             mainline_max_sub_suites=1,
             build_id="build_id",
@@ -133,6 +133,60 @@ class TestEvgExpansions(unittest.TestCase):
                          evg_expansions.mainline_max_sub_suites)
 
 
+class TestDetermineIsPatch(unittest.TestCase):
+    def test_is_patch_is_none_should_return_false(self):
+        evg_expansions = under_test.EvgExpansions(
+            is_patch=None,
+            build_id="build_id",
+            build_variant="build variant",
+            project="project",
+            revision="abc123",
+            task_name="task name",
+            task_id="task_314",
+        )
+
+        self.assertFalse(evg_expansions.determine_is_patch())
+
+    def test_is_patch_is_false_should_return_false(self):
+        evg_expansions = under_test.EvgExpansions(
+            is_patch="false",
+            build_id="build_id",
+            build_variant="build variant",
+            project="project",
+            revision="abc123",
+            task_name="task name",
+            task_id="task_314",
+        )
+
+        self.assertFalse(evg_expansions.determine_is_patch())
+
+    def test_is_patch_is_empty_string_should_return_false(self):
+        evg_expansions = under_test.EvgExpansions(
+            is_patch="",
+            build_id="build_id",
+            build_variant="build variant",
+            project="project",
+            revision="abc123",
+            task_name="task name",
+            task_id="task_314",
+        )
+
+        self.assertFalse(evg_expansions.determine_is_patch())
+
+    def test_is_patch_is_true_should_return_true(self):
+        evg_expansions = under_test.EvgExpansions(
+            is_patch="true",
+            build_id="build_id",
+            build_variant="build variant",
+            project="project",
+            revision="abc123",
+            task_name="task name",
+            task_id="task_314",
+        )
+
+        self.assertTrue(evg_expansions.determine_is_patch())
+
+
 class TestTranslateRunVar(unittest.TestCase):
     def test_normal_value_should_be_returned(self):
         run_var = "some value"

etc/backports_required_for_multiversion_tests.yml

@@ -150,6 +150,16 @@ last-continuous:
       test_file: jstests/core/exhaust.js
     - ticket: SERVER-63141
       test_file: jstests/aggregation/lookup_let_optimization.js
+    - ticket: SERVER-63129
+      test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js
+    - ticket: SERVER-62759
+      test_file: jstests/replsets/apply_ops_dropDatabase.js
+    - ticket: SERVER-64485
+      test_file: jstests/sharding/update_with_dollar_fields.js
+    - ticket: SERVER-66719
+      test_file: jstests/replsets/dbhash_lock_acquisition.js
+    - ticket: SERVER-64780
+      test_file: jstest/sharding/resharding_change_stream_namespace_filtering.js
 
   # Tests that should only be excluded from particular suites should be listed under that suite.
   suites:
@@ -439,6 +449,16 @@ last-lts:
       test_file: jstests/core/exhaust.js
     - ticket: SERVER-63141
       test_file: jstests/aggregation/lookup_let_optimization.js
+    - ticket: SERVER-63129
+      test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js
+    - ticket: SERVER-62759
+      test_file: jstests/replsets/apply_ops_dropDatabase.js
+    - ticket: SERVER-64485
+      test_file: jstests/sharding/update_with_dollar_fields.js
+    - ticket: SERVER-63531
+      test_file: jstests/replsets/buildindexes_false_commit_quorum.js
+    - ticket: SERVER-66719
+      test_file: jstests/replsets/dbhash_lock_acquisition.js
 
   # Tests that should only be excluded from particular suites should be listed under that suite.
   suites:

etc/generate_subtasks_config.yml

@@ -9,9 +9,6 @@ build_variant_large_distro_exceptions:
   - enterprise-debian92-64
   - enterprise-linux-64-amazon-ami
   - enterprise-macos
-  - enterprise-macos-rosetta-2
-  - enterprise-macos-cxx20
-  - enterprise-macos-arm64
   - enterprise-rhel-67-s390x
   - enterprise-rhel-70-64-bit
   - enterprise-rhel-70-64-bit-no-libunwind

etc/perf.yml

@@ -15,7 +15,7 @@ variables:
     - variant: linux-wt-standalone
       name: compile
   _real_expansions: &_expansion_updates
-    []
+      []
   ###
 
   ###
@@ -104,7 +104,7 @@ modules:
   - name: enterprise
     repo: git@github.com:10gen/mongo-enterprise-modules.git
     prefix: src/mongo/db/modules
-    branch: master
+    branch: v5.3
   - name: mongo-tools
     repo: git@github.com:mongodb/mongo-tools.git
     prefix: mongo-tools/src/github.com/mongodb
@@ -166,10 +166,6 @@ functions:
       params:
         script: ./src/dsi/run-dsi determine_failure -m TEST
   f_dsi_post_run:
-    - command: json.send
-      params:
-        name: perf
-        file: ./build/LegacyPerfJson/perf.json
     - command: shell.exec
       params:
         script: ./src/dsi/run-dsi post_run

etc/pip/components/lint.req

@@ -9,3 +9,4 @@ typing <= 3.7.4.3
 yamllint == 1.15.0
 yapf == 0.26.0
 evergreen-lint == 0.1.3
+types-setuptools == 57.4.12

etc/pip/components/testing.req

@@ -7,7 +7,9 @@ mock <= 4.0.3
 shrub.py == 1.1.4
 ocspresponder == 0.5.0
 flask == 1.1.1
+itsdangerous == 2.0.0
 ocspbuilder == 0.10.2
+Werkzeug == 2.0.3
 grpcio == 1.37.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or sys_platform == "win32"
 grpcio-tools == 1.37.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or sys_platform == "win32"
 googleapis-common-protos == 1.53.0

etc/scons/experimental_unified_ninja.vars

@@ -0,0 +1,6 @@
+# Configures the build for building with a unified ninja
+# Each configuration will share a ninja log
+# This allows the output binaries of each configuration to share a common directory
+
+NINJA_BUILDDIR="$BUILD_ROOT/ninja"
+DESTDIR="$BUILD_ROOT/install"

etc/system_perf.yml

@@ -22,9 +22,7 @@ variables:
       - name: schedule_global_auto_tasks
         variant: task_generation
   _real_expansions: &_expansion_updates
-    # TODO: Disable in SERVER-57226.
-    - key: enable_detect_changes
-      value: "true"
+      []
   ###
 
 ###
@@ -121,7 +119,7 @@ modules:
   - name: enterprise
     repo: git@github.com:10gen/mongo-enterprise-modules.git
     prefix: src/mongo/db/modules
-    branch: master
+    branch: v5.3
   - name: mongo-tools
     repo: git@github.com:mongodb/mongo-tools.git
     prefix: mongo-tools/src/github.com/mongodb
@@ -190,12 +188,6 @@ functions:
       params:
         script: ./src/dsi/run-dsi determine_failure -m TEST
   f_dsi_post_run:
-    # TODO: SERVER-57226 will let us move this json.send to after dsi's post_run.
-    # This is preferred to keep DSI execution contiguous.
-    - command: json.send
-      params:
-        name: perf
-        file: ./build/LegacyPerfJson/perf.json
     - command: shell.exec
       params:
         script: ./src/dsi/run-dsi post_run
@@ -882,8 +874,8 @@ tasks:
         vars:
           test_control: "initialsync-logkeeper"
           mongodb_setup: "initialsync-logkeeper-short"
-          # Logkeeper dataset with FCV set to 5.0
-          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-5.0.tgz"
+          # Logkeeper dataset with FCV set to 5.3
+          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-5.3.tgz"
 
   - name: initialsync-logkeeper-short-fcbis
     priority: 5
@@ -892,8 +884,8 @@ tasks:
         vars:
           test_control: "initialsync-logkeeper"
           mongodb_setup: "initialsync-logkeeper-short-fcbis"
-          # Logkeeper dataset with FCV set to 5.0
-          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-5.0.tgz"
+          # Logkeeper dataset with FCV set to 5.3
+          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-5.3.tgz"
 
   - name: initialsync-logkeeper
     priority: 5
@@ -925,7 +917,7 @@ tasks:
           test_control: "initialsync-logkeeper-short-s3-update"
           mongodb_setup: "initialsync-logkeeper-short-s3-update"
           # Update this to Logkeeper dataset with FCV set to latest after each LTS release.
-          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-5.0.tgz"
+          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-5.3.tgz"
 
   - name: initialsync-logkeeper-snapshot-update
     priority: 5
@@ -1081,24 +1073,6 @@ buildvariants:
       - name: tpch_1_normalized
       - name: tpch_1_denormalized
 
-  - name: linux-standalone-all-feature-flags
-    display_name: Linux Standalone (all feature flags)
-    cron: "0 0 * * *" # Everyday at 00:00
-    modules: *modules
-    expansions:
-      mongodb_setup: standalone-all-feature-flags
-      infrastructure_provisioning: single
-      platform: linux
-      project_dir: *project_dir
-      authentication: enabled
-      storageEngine: wiredTiger
-    run_on:
-      - "rhel70-perf-single"
-    depends_on: *_compile_amazon2
-    tasks:
-      - name: schedule_variant_auto_tasks
-      - name: schedule_patch_auto_tasks
-
   - name: compile-rhel70
     display_name: Compile for Atlas-like
     modules: *modules
@@ -1436,26 +1410,6 @@ buildvariants:
       - name: industry_benchmarks
       - name: linkbench
 
-  - name: linux-shard-lite-all-feature-flags
-    display_name: Linux Shard Lite (all feature flags)
-    cron: "0 0 * * 4" # 00:00 on Thursday
-    modules: *modules
-    expansions:
-      mongodb_setup: shard-lite-all-feature-flags
-      infrastructure_provisioning: shard-lite
-      platform: linux
-      project_dir: *project_dir
-      authentication: enabled
-      storageEngine: wiredTiger
-    run_on:
-      - "rhel70-perf-shard-lite"
-    depends_on: *_compile_amazon2
-    tasks:
-      - name: schedule_patch_auto_tasks
-      - name: schedule_variant_auto_tasks
-      - name: change_streams_preimage_throughput
-      - name: change_streams_preimage_latency
-
   - name: linux-shard-single
     display_name: Linux Shard Single
     cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday
@@ -1633,63 +1587,6 @@ buildvariants:
       - name: sb_large_scale
       - name: sb_timeseries
 
-  # Note that the "disabled-feature-flags" part of the name is kept to avoid breaking
-  # history even though the display name is "all feature flags"
-  - name: linux-3-node-replSet-disabled-feature-flags
-    display_name: Linux 3-Node ReplSet (all feature flags)
-    cron: "0 0 * * 4" # 00:00 on Thursday
-    modules: *modules
-    expansions:
-      mongodb_setup: replica-all-feature-flags
-      infrastructure_provisioning: replica
-      platform: linux
-      project_dir: *project_dir
-      authentication: enabled
-      storageEngine: wiredTiger
-    run_on:
-      - "rhel70-perf-replset"
-    depends_on: *_compile_amazon2
-    tasks:
-      - name: schedule_patch_auto_tasks
-      - name: schedule_variant_auto_tasks
-      - name: industry_benchmarks
-      - name: ycsb_60GB
-      - name: industry_benchmarks_secondary_reads
-      - name: crud_workloads
-      - name: crud_workloads_majority
-      - name: mixed_workloads
-      - name: misc_workloads
-      - name: map_reduce_workloads
-      - name: refine_shard_key_transaction_stress
-      - name: smoke_test
-      - name: retryable_writes_workloads
-      - name: industry_benchmarks_wmajority
-      - name: secondary_performance # Uses a special 2 node mongodb setup
-      - name: non_sharded_workloads
-      - name: bestbuy_agg
-      - name: bestbuy_agg_merge_different_db
-      - name: bestbuy_agg_merge_same_db
-      - name: bestbuy_agg_merge_wordcount
-      - name: bestbuy_query
-      - name: change_streams_throughput
-      - name: change_streams_latency
-      - name: change_streams_listen_throughput
-      - name: change_streams_preimage_throughput
-      - name: change_streams_preimage_latency
-      - name: snapshot_reads
-      - name: secondary_reads
-      - name: tpcc
-      - name: tpch_1_normalized
-      - name: tpch_1_denormalized
-      - name: linkbench
-      - name: linkbench2
-      # Time-series collections are available since v5.0.
-      # - name: tsbs_load
-      # - name: tsbs_query
-      # - name: tsbs_query_manual_bucketing
-      - name: sb_large_scale
-      - name: sb_timeseries
-
   - name: linux-3-node-replSet-noflowcontrol
     display_name: Linux 3-Node ReplSet (Flow Control off)
     cron: "0 0 * * 4" # 00:00 on Thursday
@@ -1800,8 +1697,8 @@ buildvariants:
     expansions:
       mongodb_setup: initialsync-logkeeper
       infrastructure_provisioning: initialsync-logkeeper
-      # EBS logkeeper snapshot with FCV set to 5.0
-      snapshotId: snap-0cc5e61399e2e79f6
+      # EBS logkeeper snapshot with FCV set to 5.3
+      snapshotId: snap-08b5f9109b4bcc789
       platform: linux
       authentication: disabled
       storageEngine: wiredTiger
@@ -1825,7 +1722,7 @@ buildvariants:
   #    mongodb_setup: initialsync-logkeeper-snapshot-update
   #    infrastructure_provisioning: initialsync-logkeeper-snapshot-update
   #    # Update this to latest snapshot after each LTS release.
-  #    snapshotId: snap-0cc5e61399e2e79f6
+  #    snapshotId: snap-08b5f9109b4bcc789
   #    platform: linux
   #    authentication: disabled
   #    storageEngine: wiredTiger

evergreen/commit_message_validate.sh

@@ -7,7 +7,7 @@ set -o verbose
 set -o errexit
 if [ "${is_commit_queue}" = "true" ]; then
   activate_venv
-  $python -m pip --disable-pip-version-check install --upgrade cryptography || exit 1
+  $python -m pip --disable-pip-version-check install --upgrade cryptography==36.0.2 || exit 1
   $python buildscripts/validate_commit_message.py \
     --evg-config-file ./.evergreen.yml \
     ${version_id}

evergreen/csfle_run_tests.sh

@@ -37,6 +37,8 @@ echo "Running CSFLE exported symbols test"
 expect='A MONGO_CSFLE_1.0
 T mongo_csfle_v1_analyze_query
 T mongo_csfle_v1_bson_free
+T mongo_csfle_v1_get_version
+T mongo_csfle_v1_get_version_str
 T mongo_csfle_v1_lib_create
 T mongo_csfle_v1_lib_destroy
 T mongo_csfle_v1_query_analyzer_create
@@ -56,7 +58,7 @@ if [ "$actual" != "$expect" ]; then
   exit 1
 fi
 
-echo "CSFLE exported symbols test succeeded!"
+echo "CSFLE exported symbols test succeeded"
 
 #
 # If the shared library version of the unit tests exists, run it,
@@ -78,4 +80,4 @@ fi
 
 echo "Running CSFLE shared library debuggability test"
 $GDB_PATH "$UNITTEST_PATH" --batch -ex "source ${EXTRACT_DIR}/csfle_debuggability_test.py"
-echo "CSFLE shared library debuggability test succeeded!"
+echo "CSFLE shared library debuggability test succeeded"

evergreen/jepsen_docker/list-append.sh

@@ -1,7 +1,10 @@
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$DIR/../prelude.sh"
 
-set -euo pipefail
+# this file does not use set -euo pipefail because we determine test success or
+# failure parsing the log file, instead of the return value.
+# This whole script must run to ensure the report is generated and test
+# artifacts are placed in the right location.
 
 cd jepsen/docker
 

jstests/concurrency/fsm_workload_helpers/auto_retry_transaction.js

@@ -35,7 +35,7 @@ var {withTxnAndAutoRetry, isKilledSessionCode} = (function() {
         // Don't retry the entire transaction on commit errors that aren't labeled as transient
         // transaction errors because it's unknown if the commit succeeded. commitTransaction is
         // individually retryable and should be retried at a lower level (e.g.
-        // network_error_and_txn_override.js or commitTransactionWithKilledSessionRetries()), so any
+        // network_error_and_txn_override.js or commitTransactionWithRetries()), so any
         // error that reached here must not be transient.
         if (hasCommitTxnError) {
             print("-=-=-=- Cannot retry entire transaction on commit transaction error without" +
@@ -60,8 +60,9 @@ var {withTxnAndAutoRetry, isKilledSessionCode} = (function() {
     }
 
     // Commits the transaction active on the given session, retrying on killed session errors if
-    // configured to do so. Throws if the commit fails and cannot be retried.
-    function commitTransactionWithKilledSessionRetries(session, retryOnKilledSession) {
+    // configured to do so. Also retries commitTransaction on FailedToSatisfyReadPreference error.
+    // Throws if the commit fails and cannot be retried.
+    function commitTransactionWithRetries(session, retryOnKilledSession) {
         while (true) {
             const commitRes = session.commitTransaction_forTesting();
 
@@ -76,6 +77,14 @@ var {withTxnAndAutoRetry, isKilledSessionCode} = (function() {
                 continue;
             }
 
+            if (commitRes.code === ErrorCodes.FailedToSatisfyReadPreference) {
+                print("-=-=-=- Retrying commit due to FailedToSatisfyReadPreference, sessionId: " +
+                      tojsononeline(session.getSessionId()) +
+                      ", txnNumber: " + tojsononeline(session.getTxnNumber_forTesting()) +
+                      ", res: " + tojsononeline(commitRes));
+                continue;
+            }
+
             // Use assert.commandWorked() because it throws an exception in the format expected by
             // the caller of this function if the commit failed. Committing may fail with a
             // transient error that can be retried on at a higher level, so suppress unnecessary
@@ -139,7 +148,7 @@ var {withTxnAndAutoRetry, isKilledSessionCode} = (function() {
                         const prepareTimestamp = PrepareHelpers.prepareTransaction(session);
                         PrepareHelpers.commitTransaction(session, prepareTimestamp);
                     } else {
-                        commitTransactionWithKilledSessionRetries(session, retryOnKilledSession);
+                        commitTransactionWithRetries(session, retryOnKilledSession);
                     }
                 } catch (e) {
                     hasCommitTxnError = true;
@@ -165,15 +174,6 @@ var {withTxnAndAutoRetry, isKilledSessionCode} = (function() {
                     continue;
                 }
 
-                // FailedToSatisfyReadPreference errors are not retryable.
-                // However, they should be because if there is no primary, there should be one soon.
-                // TODO SERVER-60706: Make FailedToSatisfyReadPreference a transient error
-                if (e.code == ErrorCodes.FailedToSatisfyReadPreference) {
-                    print("Retrying transaction due to a FailedToSatisfyReadPreference error.");
-                    hasTransientError = true;
-                    continue;
-                }
-
                 throw e;
             }
         } while (hasTransientError);

jstests/concurrency/fsm_workloads/CRUD_clustered_collection.js

@@ -38,27 +38,19 @@ var $config = extendWorkload($config, function($config, $super) {
         // As the default collection created by runner.js won't be clustered we need to recreate it.
         db[coll].drop();
 
-        cluster.executeOnMongodNodes(function(nodeAdminDB) {
-            assert.commandWorked(nodeAdminDB.runCommand(
-                {configureFailPoint: 'clusterAllCollectionsByDefault', mode: 'alwaysOn'}));
-        });
-
-        $super.setup.apply(this, [db, coll, cluster]);
+        assertAlways.commandWorked(
+            db.runCommand({create: coll, clusteredIndex: {key: {_id: 1}, unique: true}}));
+        for (let i = 0; i < this.numIds; i++) {
+            const res = db[coll].insert({_id: i, value: this.docValue, num: 1});
+            assertAlways.commandWorked(res);
+            assert.eq(1, res.nInserted);
+        }
 
         if (cluster.isSharded()) {
             cluster.shardCollection(db[coll], {_id: 1}, true);
         }
     };
 
-    $config.teardown = function(db, collName, cluster) {
-        $super.teardown.apply(this, [db, collName, cluster]);
-
-        cluster.executeOnMongodNodes(function(nodeAdminDB) {
-            assert.commandWorked(nodeAdminDB.runCommand(
-                {configureFailPoint: 'clusterAllCollectionsByDefault', mode: 'off'}));
-        });
-    };
-
     // Exclude dropCollection to prevent implicit collection creation of a non-clustered
     // collection.
     const newTransitions = Object.extend({}, $super.transitions);

jstests/concurrency/fsm_workloads/drop_collection_sharded.js

@@ -9,7 +9,7 @@
  */
 'use strict';
 
-const dbPrefix = 'fsmDB_';
+const dbPrefix = jsTestName() + '_DB_';
 const dbCount = 2;
 const collPrefix = 'sharded_coll_';
 const collCount = 2;

jstests/concurrency/fsm_workloads/drop_database_sharded.js

@@ -7,7 +7,7 @@
  */
 'use strict';
 
-const dbPrefix = 'fsmDB_';
+const dbPrefix = jsTestName() + '_DB_';
 const dbCount = 2;
 const collPrefix = 'sharded_coll_';
 const collCount = 2;

jstests/concurrency/fsm_workloads/random_DDL_operations.js

@@ -10,7 +10,7 @@
  *  ]
  */
 
-const dbPrefix = 'fsmDB_';
+const dbPrefix = jsTestName() + '_DB_';
 const dbCount = 2;
 const collPrefix = 'sharded_coll_';
 const collCount = 2;

jstests/core/apply_ops1.js

@@ -457,7 +457,7 @@ res = assert.commandFailed(db.adminCommand({
         }
     ]
 }));
-assert.eq(res.code, 4772604);
+assert.eq(res.code, 4772600);
 
 // When we explicitly specify {$v: 1}, we should get 'UpdateNode' update semantics, and $set
 // operations get performed in lexicographic order.
@@ -503,7 +503,7 @@ res = assert.commandFailed(db.adminCommand({
         }
     ]
 }));
-assert.eq(res.code, 4772604);
+assert.eq(res.code, 4772600);
 
 var insert_op1 = {_id: 13, x: 'inserted apply ops1'};
 var insert_op2 = {_id: 14, x: 'inserted apply ops2'};

jstests/core/builtin_roles_external.js

@@ -0,0 +1,47 @@
+/**
+ * Attempting to enumerate roles on the $external database should return an empty set.
+ * @tags: [multiversion_incompatible,tenant_migration_incompatible]
+ */
+(function() {
+"use strict";
+
+function assertBuiltinRoles(dbname, shouldHaveRoles) {
+    const allRoles = assert
+                         .commandWorked(db.getSiblingDB(dbname).runCommand(
+                             {rolesInfo: 1, showBuiltinRoles: 1, showPrivileges: 1}))
+                         .roles;
+    jsTest.log(dbname + ' roles: ' + tojson(allRoles));
+
+    const builtinRoles = allRoles.filter((r) => r.isBuiltin);
+    if (shouldHaveRoles) {
+        assert.gt(builtinRoles.length, 0, dbname + ' should have builtin roles, but none returned');
+
+        function assertRole(role, expect = true) {
+            const filtered = builtinRoles.filter((r) => r.role === role);
+            if (expect) {
+                assert.gt(
+                    filtered.length, 0, dbname + ' should have role ' + role + ' but does not');
+            } else {
+                assert.eq(
+                    filtered.length,
+                    0,
+                    dbname + ' should have not role ' + role + ' but does: ' + tojson(filtered));
+            }
+        }
+
+        assertRole('read');
+        assertRole('readWrite');
+        assertRole('readWriteAnyDatabase', dbname === 'admin');
+        assertRole('hostManager', dbname === 'admin');
+    } else {
+        assert.eq(builtinRoles.length,
+                  0,
+                  dbname + ' should not have builtin roles, found: ' + tojson(builtinRoles));
+    }
+}
+
+assertBuiltinRoles('admin', true);
+assertBuiltinRoles('test', true);
+assertBuiltinRoles('$external', false);
+assertBuiltinRoles('$test', true);
+}());

jstests/core/connection_status.js

@@ -2,6 +2,7 @@
 //   assumes_superuser_permissions,
 //   assumes_write_concern_unchanged,
 //   creates_and_authenticates_user,
+//   no_selinux,
 //   requires_auth,
 //   requires_non_retryable_commands,
 // ]

jstests/core/merge_sort_collation.js

@@ -263,33 +263,33 @@ const testCases = [
         // collation of the indexes. Also, the second/third operands to $or queries on fields
         // 'e'/'g' that are not covered by the indexes therefore triggers addition of a FETCH stage
         // between MERGE_SORT and IXSCAN. This tests comparison of index versus fetched document
-        // provided sort keys. In addition to that, some documents have objects as sort attribute
-        // values.
+        // provided sort keys. In addition to that, some documents have missing values and objects
+        // as sort attribute values.
         indexes: [{a: 1, b: 1, c: 1}, {d: 1, c: 1}, {f: 1, c: 1}],
         indexOptions: {collation: {locale: "en", strength: 2}},
         filter: {$or: [{a: {$in: ["1", "2"]}, b: "1"}, {d: "3", e: "3"}, {f: "4", g: "3"}]},
         sort: {c: 1},
         findCollation: {locale: "en", strength: 2},
         inputDocuments: [
-            {_id: 0, a: "1", b: "1", c: "a"},
-            {_id: 1, a: "1", b: "1", c: "d"},
-            {_id: 2, a: "2", b: "1", c: "b"},
-            {_id: 3, a: "2", b: "1", c: "e"},
-            {_id: 6, a: "2", b: "1", c: {a: "B"}},
+            {_id: 0, d: "3", e: "3", c: "a"},
+            {_id: 1, d: "3", e: "3", c: "b"},
+            {_id: 2, d: "3", e: "3"},
+            {_id: 3, d: "3", e: "3", c: {}},
             {_id: 4, d: "3", e: "3", c: "c"},
-            {_id: 5, d: "3", e: "3", c: "f"},
+            {_id: 5, d: "3", e: "3", c: 1},
+            {_id: 6, a: "2", b: "1", c: {a: "B"}},
             {_id: 7, d: "3", e: "3", c: {a: "A"}},
             {_id: 8, d: "3", e: "3", c: {a: "C"}},
-            {_id: 9, f: "4", g: "3", c: "g"},
+            {_id: 9, f: "4", g: "3", c: "d"},
         ],
         expectedResults: [
-            {_id: 0, a: "1", b: "1", c: "a"},
-            {_id: 2, a: "2", b: "1", c: "b"},
+            {_id: 2, d: "3", e: "3"},
+            {_id: 5, d: "3", e: "3", c: 1},
+            {_id: 0, d: "3", e: "3", c: "a"},
+            {_id: 1, d: "3", e: "3", c: "b"},
             {_id: 4, d: "3", e: "3", c: "c"},
-            {_id: 1, a: "1", b: "1", c: "d"},
-            {_id: 3, a: "2", b: "1", c: "e"},
-            {_id: 5, d: "3", e: "3", c: "f"},
-            {_id: 9, f: "4", g: "3", c: "g"},
+            {_id: 9, f: "4", g: "3", c: "d"},
+            {_id: 3, d: "3", e: "3", c: {}},
             {_id: 7, d: "3", e: "3", c: {a: "A"}},
             {_id: 6, a: "2", b: "1", c: {a: "B"}},
             {_id: 8, d: "3", e: "3", c: {a: "C"}},

jstests/core/profile1.js

@@ -2,6 +2,7 @@
 //   assumes_superuser_permissions,
 //   creates_and_authenticates_user,
 //   does_not_support_stepdowns,
+//   no_selinux,
 //   requires_capped,
 //   requires_collstats,
 //   requires_non_retryable_commands,

jstests/core/profile3.js

@@ -1,6 +1,7 @@
 // @tags: [
 //   assumes_superuser_permissions,
 //   creates_and_authenticates_user,
+//   no_selinux,
 //   requires_profiling,
 // ]
 // special db so that it can be run in parallel tests

jstests/core/timeseries/timeseries_project.js

@@ -81,4 +81,36 @@ assert.docEq(result, [{_id: 0, time: docDate, a: {b: 1}, b: 4, c: [{}, {}]}]);
 // Test that an exclude does not overwrite meta field pushdown.
 result = coll.aggregate([{$unset: "b"}, {$set: {b: "$meta"}}]).toArray();
 assert.docEq(result, [{_id: 0, time: docDate, meta: 4, a: {b: 1}, b: 4, c: [{}, {}]}]);
+
+// Test that a field reference in a projection refers to the stage's input document
+// rather than another field with the same name in the projection.
+(function() {
+const regColl = db.timeseries_project_reg;
+regColl.drop();
+
+const tsColl = db.timeseries_project_ts;
+tsColl.drop();
+assert.commandWorked(
+    db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'x'}}));
+
+const doc = {
+    time: new Date("2019-10-11T14:39:18.670Z"),
+    x: 5,
+    a: 3,
+};
+assert.commandWorked(tsColl.insert(doc));
+assert.commandWorked(regColl.insert(doc));
+
+// Test $project.
+let pipeline = [{$project: {_id: 0, a: "$x", b: "$a"}}];
+let tsDoc = tsColl.aggregate(pipeline).toArray();
+let regDoc = regColl.aggregate(pipeline).toArray();
+assert.docEq(tsDoc, regDoc);
+
+// Test $addFields.
+pipeline = [{$addFields: {a: "$x", b: "$a"}}, {$project: {_id: 0}}];
+tsDoc = tsColl.aggregate(pipeline).toArray();
+regDoc = regColl.aggregate(pipeline).toArray();
+assert.docEq(tsDoc, regDoc);
+})();
 })();

jstests/core/update_with_pipeline.js

@@ -180,7 +180,6 @@ assert.commandFailedWithCode(coll.update({x: 1}, [{$match: {x: 1}}]), ErrorCodes
 assert.commandFailedWithCode(coll.update({x: 1}, [{$sort: {x: 1}}]), ErrorCodes.InvalidOptions);
 assert.commandFailedWithCode(coll.update({x: 1}, [{$facet: {a: [{$match: {x: 1}}]}}]),
                              ErrorCodes.InvalidOptions);
-assert.commandFailedWithCode(coll.update({x: 1}, [{$indexStats: {}}]), ErrorCodes.InvalidOptions);
 assert.commandFailedWithCode(
     coll.update(
         {x: 1}, [{
@@ -198,6 +197,11 @@ assert.commandFailedWithCode(
         }]),
     ErrorCodes.InvalidOptions);
 
+// $indexStats is not supported in a transaction passthrough and will fail with a different error.
+assert.commandFailedWithCode(
+    coll.update({x: 1}, [{$indexStats: {}}]),
+    [ErrorCodes.InvalidOptions, ErrorCodes.OperationNotSupportedInTransaction]);
+
 // Update fails when supported agg stage is specified outside of pipeline.
 assert.commandFailedWithCode(coll.update({_id: 1}, {$addFields: {x: 1}}), ErrorCodes.FailedToParse);
 

jstests/core/user_management_helpers.js

@@ -2,6 +2,7 @@
 //   assumes_superuser_permissions,
 //   assumes_write_concern_unchanged,
 //   creates_and_authenticates_user,
+//   no_selinux,
 //   requires_auth,
 //   requires_non_retryable_commands,
 // ]

jstests/core/version_api_v1_command_coverage.js

@@ -5,6 +5,7 @@
  * @tags: [
  *   requires_non_retryable_commands,
  *   uses_api_parameters,
+ *   requires_fcv_53,
  * ]
  */
 
@@ -45,7 +46,7 @@ const commands = [
     {cmd: () => ({serverStatus: 1}), apiVersion1: false},
     {cmd: () => ({usersInfo: 1}), apiVersion1: false},
     {cmd: () => ({aggregate: testColl.getName(), pipeline: [], cursor: {}}), apiVersion1: true},
-    {cmd: () => ({count: "system.js"}), apiVersion1: false},
+    {cmd: () => ({count: "system.js"}), apiVersion1: true},
     {cmd: () => ({create: counter_fun()}), apiVersion1: true},
     {cmd: () => ({find: counter_fun()}), apiVersion1: true},
     {

jstests/core/views/views_all_commands.js

@@ -586,6 +586,7 @@ let viewsCommandTests = {
     setFreeMonitoring: {skip: isUnrelated},
     setParameter: {skip: isUnrelated},
     setShardVersion: {skip: isUnrelated},
+    setUserWriteBlockMode: {skip: isUnrelated},
     shardCollection: {
         command: {shardCollection: "test.view", key: {_id: 1}},
         setup: function(conn) {

jstests/core/views/views_validation.js

@@ -166,6 +166,12 @@ assert.commandFailedWithCode(
     "BSON field 'collMod.pipeline' is the wrong type 'object', expected type 'array'");
 clear();
 
+// Check that collMod disallows the 'expireAfterSeconds' option over a view.
+makeView("a", "b");
+assert.commandFailedWithCode(viewsDb.runCommand({collMod: "a", expireAfterSeconds: 1}),
+                             ErrorCodes.InvalidOptions);
+clear();
+
 // Check that invalid pipelines are disallowed. The following $lookup is missing the 'as' field.
 makeView("a",
          "b",

jstests/hooks/run_dbcheck_background.js

@@ -17,115 +17,116 @@ TestData = TestData || {};
 const conn = db.getMongo();
 const topology = DiscoverTopology.findConnectedNodes(conn);
 
-function runBackgroundDbCheck(hosts) {
-    const quietly = (func) => {
-        const printOriginal = print;
-        try {
-            print = Function.prototype;
-            func();
-        } finally {
-            print = printOriginal;
-        }
-    };
-
-    let rst;
-    // We construct the ReplSetTest instance with the print() function overridden to be a no-op
-    // in order to suppress the log messages about the replica set configuration. The
-    // run_dbcheck_background.js hook is executed frequently by resmoke.py and would
-    // otherwise lead to generating an overwhelming amount of log messages.
-    quietly(() => {
-        rst = new ReplSetTest(hosts[0]);
-    });
-
-    const dbNames = new Set();
-    const primary = rst.getPrimary();
-
-    const version =
-        assert
-            .commandWorked(primary.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}))
-            .featureCompatibilityVersion.version;
-    if (version != latestFCV) {
-        print("Not running dbCheck in FCV " + version);
-        return {ok: 1};
-    }
-
-    print("Running dbCheck for: " + rst.getURL());
-
-    const adminDb = primary.getDB('admin');
-    let res = assert.commandWorked(adminDb.runCommand({listDatabases: 1, nameOnly: true}));
-    for (let dbInfo of res.databases) {
-        dbNames.add(dbInfo.name);
-    }
-
-    // Transactions cannot be run on the following databases so we don't attempt to read at a
-    // clusterTime on them either. (The "local" database is also not replicated.)
-    // The config.transactions collection is different between primaries and secondaries.
-    dbNames.delete('config');
-    dbNames.delete('local');
-
-    dbNames.forEach((dbName) => {
-        try {
-            assert.commandWorked(primary.getDB(dbName).runCommand({dbCheck: 1}));
-            jsTestLog("dbCheck done on database " + dbName);
-        } catch (e) {
-            if (e.code === ErrorCodes.NamespaceNotFound || e.code === ErrorCodes.LockTimeout ||
-                e.code == ErrorCodes.Interrupted) {
-                jsTestLog("Skipping dbCheck on database " + dbName +
-                          " due to transient error: " + tojson(e));
-                return;
-            } else {
-                throw e;
+const exceptionFilteredBackgroundDbCheck = function(hosts) {
+    const runBackgroundDbCheck = function(hosts) {
+        const quietly = (func) => {
+            const printOriginal = print;
+            try {
+                print = Function.prototype;
+                func();
+            } finally {
+                print = printOriginal;
             }
-        }
-
-        const dbCheckCompleted = (db) => {
-            return db.currentOp().inprog.filter(x => x["desc"] == "dbCheck")[0] === undefined;
         };
 
-        assert.soon(() => dbCheckCompleted(adminDb),
-                    "timed out waiting for dbCheck to finish on database: " + dbName);
-    });
+        let rst;
+        // We construct the ReplSetTest instance with the print() function overridden to be a no-op
+        // in order to suppress the log messages about the replica set configuration. The
+        // run_dbcheck_background.js hook is executed frequently by resmoke.py and would
+        // otherwise lead to generating an overwhelming amount of log messages.
+        quietly(() => {
+            rst = new ReplSetTest(hosts[0]);
+        });
 
-    // Wait for all secondaries to finish applying all dbcheck batches.
-    rst.awaitReplication();
+        const dbNames = new Set();
+        const primary = rst.getPrimary();
 
-    const nodes = [
-        rst.getPrimary(),
-        ...rst.getSecondaries().filter(conn => {
-            return !conn.adminCommand({isMaster: 1}).arbiterOnly;
-        })
-    ];
-    nodes.forEach((node) => {
-        // Assert no errors (i.e., found inconsistencies). Allow warnings. Tolerate SnapshotTooOld
-        // errors, as they can occur if the primary is slow enough processing a batch that the
-        // secondary is unable to obtain the timestamp the primary used.
-        const healthlog = node.getDB('local').system.healthlog;
-        // Regex matching strings that start without "SnapshotTooOld"
-        const regexStringWithoutSnapTooOld = /^((?!^SnapshotTooOld).)*$/;
-        let errs =
-            healthlog.find({"severity": "error", "data.error": regexStringWithoutSnapTooOld});
-        if (errs.hasNext()) {
-            const err = "dbCheck found inconsistency on " + node.host;
-            jsTestLog(err + ". Errors: ");
-            for (let count = 0; errs.hasNext() && count < 20; count++) {
-                jsTestLog(tojson(errs.next()));
-            }
-            assert(false, err);
+        const version = assert
+                            .commandWorked(primary.adminCommand(
+                                {getParameter: 1, featureCompatibilityVersion: 1}))
+                            .featureCompatibilityVersion.version;
+        if (version != latestFCV) {
+            print("Not running dbCheck in FCV " + version);
+            return {ok: 1};
         }
-        jsTestLog("Checked health log on " + node.host);
-    });
 
-    return {ok: 1};
-}
+        print("Running dbCheck for: " + rst.getURL());
+
+        const adminDb = primary.getDB('admin');
+        let res = assert.commandWorked(adminDb.runCommand({listDatabases: 1, nameOnly: true}));
+        for (let dbInfo of res.databases) {
+            dbNames.add(dbInfo.name);
+        }
+
+        // Transactions cannot be run on the following databases so we don't attempt to read at a
+        // clusterTime on them either. (The "local" database is also not replicated.)
+        // The config.transactions collection is different between primaries and secondaries.
+        dbNames.delete('config');
+        dbNames.delete('local');
+
+        dbNames.forEach((dbName) => {
+            assert.commandWorked(primary.getDB(dbName).runCommand({dbCheck: 1}));
+            jsTestLog("dbCheck done on database " + dbName);
+
+            const dbCheckCompleted = (db) => {
+                return db.currentOp({$all: true}).inprog.filter(x => x["desc"] === "dbCheck")[0] ===
+                    undefined;
+            };
+
+            assert.soon(() => dbCheckCompleted(adminDb),
+                        "timed out waiting for dbCheck to finish on database: " + dbName);
+        });
+
+        // Wait for all secondaries to finish applying all dbcheck batches.
+        rst.awaitReplication();
+
+        const nodes = [
+            rst.getPrimary(),
+            ...rst.getSecondaries().filter(conn => {
+                return !conn.adminCommand({isMaster: 1}).arbiterOnly;
+            })
+        ];
+        nodes.forEach((node) => {
+            // Assert no errors (i.e., found inconsistencies). Allow warnings. Tolerate
+            // SnapshotTooOld errors, as they can occur if the primary is slow enough processing a
+            // batch that the secondary is unable to obtain the timestamp the primary used.
+            const healthlog = node.getDB('local').system.healthlog;
+            // Regex matching strings that start without "SnapshotTooOld"
+            const regexStringWithoutSnapTooOld = /^((?!^SnapshotTooOld).)*$/;
+            let errs =
+                healthlog.find({"severity": "error", "data.error": regexStringWithoutSnapTooOld});
+            if (errs.hasNext()) {
+                const err = "dbCheck found inconsistency on " + node.host;
+                jsTestLog(err + ". Errors: ");
+                for (let count = 0; errs.hasNext() && count < 20; count++) {
+                    jsTestLog(tojson(errs.next()));
+                }
+                assert(false, err);
+            }
+            jsTestLog("Checked health log on " + node.host);
+        });
+
+        return {ok: 1};
+    };
+
+    const onDrop = function(e) {
+        jsTestLog("Skipping dbCheck due to transient error: " + tojson(e));
+        return {ok: 1};
+    };
+
+    return assert.dropExceptionsWithCode(() => {
+        return runBackgroundDbCheck(hosts);
+    }, [ErrorCodes.NamespaceNotFound, ErrorCodes.LockTimeout, ErrorCodes.Interrupted], onDrop);
+};
 
 if (topology.type === Topology.kReplicaSet) {
-    let res = runBackgroundDbCheck(topology.nodes);
+    let res = exceptionFilteredBackgroundDbCheck(topology.nodes);
     assert.commandWorked(res, () => 'dbCheck replication consistency check failed: ' + tojson(res));
 } else if (topology.type === Topology.kShardedCluster) {
     const threads = [];
     try {
         if (topology.configsvr.type === Topology.kReplicaSet) {
-            const thread = new Thread(runBackgroundDbCheck, topology.configsvr.nodes);
+            const thread = new Thread(exceptionFilteredBackgroundDbCheck, topology.configsvr.nodes);
             threads.push(thread);
             thread.start();
         }
@@ -133,7 +134,7 @@ if (topology.type === Topology.kReplicaSet) {
         for (let shardName of Object.keys(topology.shards)) {
             const shard = topology.shards[shardName];
             if (shard.type === Topology.kReplicaSet) {
-                const thread = new Thread(runBackgroundDbCheck, shard.nodes);
+                const thread = new Thread(exceptionFilteredBackgroundDbCheck, shard.nodes);
                 threads.push(thread);
                 thread.start();
             } else {

jstests/libs/clustered_collections/clustered_collection_create_index_clustered_common.js

@@ -17,6 +17,15 @@ const CreateIndexesClusteredTest = (function() {
         // Start with the collection empty.
         assert.commandFailedWithCode(
             testColl.createIndex({_id: 1}, {clustered: true, unique: true}), 6243700);
+
+        // Pass non-boolean value to safeBool 'clustered' option. Should be equivalent to next
+        // command.
+        assert.commandFailedWithCode(testDB.runCommand({
+            createIndexes: collName,
+            "indexes": [{key: {"newKey": 1}, name: "anyName", clustered: 2, unique: true}],
+        }),
+                                     6243700);
+
         assert.commandFailedWithCode(testColl.createIndex({a: 1}, {clustered: true, unique: true}),
                                      6243700);
 
@@ -30,6 +39,15 @@ const CreateIndexesClusteredTest = (function() {
         assert.commandWorked(bulk.execute());
         assert.commandFailedWithCode(
             testColl.createIndex({_id: 1}, {clustered: true, unique: true}), 6243700);
+
+        // Pass non-boolean value to safeBool 'clustered' option. Should be equivalent to next
+        // command.
+        assert.commandFailedWithCode(testDB.runCommand({
+            createIndexes: collName,
+            "indexes": [{key: {"newKey2": 1}, name: "anyName2", clustered: 2, unique: true}],
+        }),
+                                     6243700);
+
         assert.commandFailedWithCode(testColl.createIndex({a: 1}, {clustered: true, unique: true}),
                                      6243700);
     };
@@ -67,6 +85,14 @@ const CreateIndexesClusteredTest = (function() {
         // createIndex on the cluster key with the 'clustered' option is a no-op.
         assert.commandWorked(testColl.createIndex({_id: 1}, {clustered: true, unique: true}));
 
+        // Pass non-boolean value to safeBool 'clustered' option. Should be equivalent to next
+        // command.
+        assert.commandFailedWithCode(testDB.runCommand({
+            createIndexes: collName,
+            "indexes": [{key: {"newKey": 1}, name: "anyName", clustered: 2, unique: true}],
+        }),
+                                     6243700);
+
         // 'clustered' is not a valid option for an index not on the cluster key.
         assert.commandFailedWithCode(
             testColl.createIndex({notMyIndex: 1}, {clustered: true, unique: true}), 6243700);
@@ -83,6 +109,14 @@ const CreateIndexesClusteredTest = (function() {
         assert.commandWorked(testColl.createIndex({_id: 1}));
         assert.commandWorked(testColl.createIndex({_id: 1}, {clustered: true, unique: true}));
 
+        // Pass non-boolean value to safeBool 'clustered' option. Should be equivalent to next
+        // command.
+        assert.commandFailedWithCode(testDB.runCommand({
+            createIndexes: collName,
+            "indexes": [{key: {"newKey2": 1}, name: "anyName2", clustered: 2, unique: true}],
+        }),
+                                     6243700);
+
         // 'clustered' is still not a valid option for an index not on the cluster key.
         assert.commandFailedWithCode(testColl.createIndex({a: 1}, {clustered: true, unique: true}),
                                      6243700);

jstests/multiVersion/internal_sessions_setfcv_wait_for_transaction_coordinator_cleanup.js

@@ -1,7 +1,6 @@
 /*
  * Tests that setFCV waits for transaction coordinators for internal transactions to be removed.
- *
- * @tags: [requires_fcv_51, featureFlagInternalTransactions]
+ * @tags: [featureFlagInternalTransactions]
  */
 
 (function() {

jstests/multiVersion/targetedTestsLastLtsFeatures/invalid_index_options.js

@@ -0,0 +1,56 @@
+/**
+ * Tests that in 5.3 version listIndexes can parse invalid index specs created before 5.0 version.
+ *
+ * @tags: [requires_replication]
+ */
+(function() {
+"use strict";
+
+load('jstests/multiVersion/libs/multi_rs.js');
+
+var nodes = {
+    n1: {binVersion: "4.4"},
+    n2: {binVersion: "4.4"},
+};
+
+var rst = new ReplSetTest({nodes: nodes});
+rst.startSet();
+rst.initiate();
+
+const dbName = "test";
+const collName = jsTestName();
+
+let primaryDB = rst.getPrimary().getDB(dbName);
+let primaryColl = primaryDB.getCollection(collName);
+
+// In earlier versions, users were able to add invalid index options when creating an index. The
+// option could still be interpreted accordingly.
+assert.commandWorked(primaryColl.createIndex({x: 1}, {sparse: "yes"}));
+
+// Upgrades from 4.4 to 5.0.
+jsTestLog("Upgrading to version 5.0");
+rst.upgradeSet({binVersion: "last-lts"});
+assert.commandWorked(rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+// Upgrades from 5.0 to 5.3.
+jsTestLog("Upgrading to version latest");
+rst.upgradeSet({binVersion: "latest"});
+const primary = rst.getPrimary();
+assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+primaryDB = primary.getDB(dbName);
+
+// Verify listIndexes command can correctly output the repaired index specs.
+assert.commandWorked(primaryDB.runCommand({listIndexes: collName}));
+
+// Add a new node to make sure the initial sync works correctly with the invalid index specs.
+jsTestLog("Bringing up a new node");
+rst.add();
+rst.reInitiate();
+
+jsTestLog("Waiting for new node to be synced.");
+rst.awaitReplication();
+rst.awaitSecondaryNodes();
+
+rst.stopSet();
+})();

jstests/noPassthrough/api_version_parameters_shell.js

@@ -24,8 +24,8 @@ const testCases = [
     [false, false, {testDeprecation: 1}, {version: '1', deprecationErrors: true}],
     [false, false, {testDeprecation: 1}, {version: '1', strict: true, deprecationErrors: true}],
     // tests with setParameter requireApiVersion: true.
-    [true, false, {count: 'collection'}, {version: '1', strict: true}],
     [true, true, {count: 'collection'}, {version: '1'}],
+    [true, true, {count: 'collection'}, {version: '1', strict: true}],
     [true, false, {ping: 1}, {}],
     [true, true, {ping: 1}, {version: '1'}],
 ];

jstests/noPassthrough/clustered_capped_collection.js

@@ -5,6 +5,7 @@
  *   requires_fcv_53,
  *   requires_replication,
  *   does_not_support_stepdowns,
+ *   incompatible_with_eft
  * ]
  */
 (function() {

jstests/noPassthrough/clustered_capped_collection_arbitrary_key.js

@@ -5,6 +5,7 @@
  *   requires_fcv_53,
  *   requires_replication,
  *   does_not_support_stepdowns,
+ *   incompatible_with_eft
  * ]
  */
 (function() {

jstests/noPassthrough/commit_quorum_voting_nodes.js

@@ -0,0 +1,83 @@
+/**
+ * Tests that index build commitQuorum can include non-voting data-bearing nodes.
+ *
+ * @tags: [
+ *   requires_journaling,
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
+ */
+
+(function() {
+const replSet = new ReplSetTest({
+    nodes: [
+        {
+            rsConfig: {
+                tags: {es: 'dc1'},
+            }
+        },
+        {
+            rsConfig: {
+                tags: {es: 'dc2'},
+            }
+        },
+        {
+            // Analytics node with zero votes should be included in a commitQuorum.
+            rsConfig: {
+                priority: 0,
+                votes: 0,
+                tags: {es: 'analytics'},
+            },
+        },
+    ],
+});
+
+replSet.startSet();
+replSet.initiate();
+
+const primary = replSet.getPrimary();
+const config = primary.getDB('local').system.replset.findOne();
+
+// Create a custom write concern for both nodes with the 'es' tag. We will expect this to be usable
+// as an option to commitQuorum.
+config.settings = {
+    getLastErrorModes: {ESP: {"es": 3}}
+};
+config.version++;
+assert.commandWorked(primary.getDB("admin").runCommand({replSetReconfig: config}));
+
+const db = replSet.getPrimary().getDB('test');
+const coll = db['coll'];
+assert.commandWorked(coll.insert({a: 1}));
+
+// Shut the analytics node down so that it cannot contribute to the commitQuorum.
+const analyticsNodeId = 2;
+replSet.stop(analyticsNodeId, {forRestart: true});
+
+// The default commitQuorum should not include the non-voting analytics node.
+assert.commandWorked(coll.createIndex({a: 1}));
+coll.dropIndexes();
+
+// The explicit "votingMembers" should not include the non-voting analytics node.
+assert.commandWorked(coll.createIndex({a: 1}, {}, "votingMembers"));
+coll.dropIndexes();
+
+// Restart the analytics node down so that it can contribute to the commitQuorum.
+replSet.start(analyticsNodeId, {}, true /* restart */);
+
+// This should include the non-voting analytics node.
+const nNodes = replSet.nodeList().length;
+assert.commandWorked(coll.createIndex({a: 1}, {}, nNodes));
+coll.dropIndexes();
+
+// This custom tag should include the analytics node.
+assert.commandWorked(coll.createIndex({a: 1}, {}, "ESP"));
+coll.dropIndexes();
+
+// Not enough data-bearing nodes to satisfy commit quorum.
+assert.commandFailedWithCode(coll.createIndex({a: 1}, {}, nNodes + 1),
+                             ErrorCodes.UnsatisfiableCommitQuorum);
+coll.dropIndexes();
+
+replSet.stopSet();
+})();

jstests/noPassthrough/serverstatus_index_stats.js

@@ -0,0 +1,257 @@
+/**
+ * Tests that serverStatus contains an indexStats section. This section reports globally-aggregated
+ * statistics about features in use by indexes and how often they are used.
+ *
+ * @tags: [
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
+ */
+(function() {
+"use strict";
+
+const assertStats = (db, assertFn) => {
+    const stats = db.serverStatus().indexStats;
+    try {
+        assertFn(stats);
+    } catch (e) {
+        print("result: " + tojson(stats));
+        throw e;
+    }
+};
+
+// If new features are added, they must also be added to this list or the test will fail.
+const knownFeatures = [
+    "2d",
+    "2dsphere",
+    "2dsphere_bucket",
+    "collation",
+    "compound",
+    "hashed",
+    "id",
+    "normal",
+    "partial",
+    "single",
+    "sparse",
+    "text",
+    "ttl",
+    "unique",
+    "wildcard",
+];
+
+const assertZeroCounts = (db) => {
+    assertStats(db, (featureStats) => {
+        assert.eq(featureStats.count, 0);
+        for (const [feature, stats] of Object.entries(featureStats.features)) {
+            assert.contains(feature, knownFeatures, "unknown feature reported by indexStats");
+            assert.eq(0, stats.count, feature);
+        }
+    });
+};
+
+const assertZeroAccess = (db) => {
+    assertStats(db, (featureStats) => {
+        for (const [feature, stats] of Object.entries(featureStats.features)) {
+            assert.contains(feature, knownFeatures, "unknown feature reported by indexStats");
+            assert.eq(0, stats.accesses, feature);
+        }
+    });
+};
+
+const assertCountIncrease = (last, current, inc) => {
+    assert.eq(last.count + inc, current.count, "incorrect index count");
+};
+
+const assertFeatureCountIncrease = (last, current, feature, inc) => {
+    assert.eq(last.features[feature].count + inc,
+              current.features[feature].count,
+              "incorrect feature count for " + feature);
+};
+
+const assertFeatureAccessIncrease = (last, current, feature, inc) => {
+    assert.eq(last.features[feature].accesses + inc,
+              current.features[feature].accesses,
+              "incorrect feature accesses for " + feature);
+};
+
+const replSet = new ReplSetTest({nodes: 1});
+replSet.startSet();
+replSet.initiate();
+
+let primary = replSet.getPrimary();
+let db = primary.getDB('test');
+
+assertZeroCounts(db);
+assertZeroAccess(db);
+
+let lastStats = db.serverStatus().indexStats;
+
+assert.commandWorked(db.testColl.createIndex({twoD: '2d', b: 1}, {unique: true, sparse: true}));
+assert.commandWorked(db.testColl.insert({twoD: [0, 0], b: 1}));
+assert.eq(1, db.testColl.find({twoD: {$geoNear: [0, 0]}}).itcount());
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 2);
+    assertFeatureCountIncrease(lastStats, stats, '2d', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'compound', 1);
+    // The index build implicitly created the collection, which also builds an _id index.
+    assertFeatureCountIncrease(lastStats, stats, 'id', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'sparse', 1);
+    // Note that the _id index is not included in this unique counter. This is due to a quirk in the
+    // _id index spec that does not actually have a unique:true property.
+    assertFeatureCountIncrease(lastStats, stats, 'unique', 1);
+
+    assertFeatureAccessIncrease(lastStats, stats, '2d', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'compound', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'id', 0);
+    assertFeatureAccessIncrease(lastStats, stats, 'sparse', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'unique', 1);
+});
+
+lastStats = db.serverStatus().indexStats;
+
+assert.commandWorked(db.testColl.createIndex({sphere: '2dsphere'}));
+assert.commandWorked(db.testColl.insert({sphere: {type: "Point", coordinates: [0, 0]}}));
+assert.eq(1,
+          db.testColl
+              .aggregate([{
+                  $geoNear: {
+                      near: {type: "Point", coordinates: [1, 1]},
+                      key: 'sphere',
+                      distanceField: 'dist',
+                  }
+              }])
+              .itcount());
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 1);
+    assertFeatureCountIncrease(lastStats, stats, '2dsphere', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'single', 1);
+
+    assertFeatureAccessIncrease(lastStats, stats, '2dsphere', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'single', 1);
+});
+
+lastStats = db.serverStatus().indexStats;
+
+assert.commandWorked(
+    db.testColl.createIndex({hashed: 'hashed', p: 1}, {partialFilterExpression: {p: 1}}));
+assert.commandWorked(db.testColl.insert({hashed: 1, p: 1}));
+assert.eq(1, db.testColl.find({hashed: 1}).hint({hashed: 'hashed', p: 1}).itcount());
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 1);
+    assertFeatureCountIncrease(lastStats, stats, 'compound', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'hashed', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'partial', 1);
+
+    assertFeatureAccessIncrease(lastStats, stats, 'compound', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'hashed', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'partial', 1);
+});
+
+lastStats = db.serverStatus().indexStats;
+
+assert.commandWorked(
+    db.testColl.createIndex({a: 1}, {expireAfterSeconds: 3600, collation: {locale: 'en'}}));
+let now = new Date();
+assert.commandWorked(db.testColl.insert({a: now}));
+assert.eq(1, db.testColl.find({a: now}).itcount());
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 1);
+    assertFeatureCountIncrease(lastStats, stats, 'collation', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'normal', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'single', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'ttl', 1);
+
+    assertFeatureAccessIncrease(lastStats, stats, 'collation', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'normal', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'single', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'ttl', 1);
+});
+
+lastStats = db.serverStatus().indexStats;
+
+assert.commandWorked(db.testColl.createIndex({text: 'text'}));
+assert.commandWorked(db.testColl.insert({text: "a string"}));
+assert.eq(1, db.testColl.find({$text: {$search: "string"}}).itcount());
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 1);
+    // Text indexes are internally compound, but that should not be reflected in the stats.
+    assertFeatureCountIncrease(lastStats, stats, 'compound', 0);
+    assertFeatureCountIncrease(lastStats, stats, 'text', 1);
+
+    assertFeatureAccessIncrease(lastStats, stats, 'compound', 0);
+    assertFeatureAccessIncrease(lastStats, stats, 'text', 1);
+});
+
+lastStats = db.serverStatus().indexStats;
+
+assert.commandWorked(db.testColl.createIndex({'wild.$**': 1}));
+assert.commandWorked(db.testColl.insert({wild: {a: 1}}));
+assert.eq(1, db.testColl.find({'wild.a': 1}).itcount());
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 1);
+    assertFeatureCountIncrease(lastStats, stats, 'single', 1);
+    assertFeatureCountIncrease(lastStats, stats, 'wildcard', 1);
+
+    assertFeatureAccessIncrease(lastStats, stats, 'single', 1);
+    assertFeatureAccessIncrease(lastStats, stats, 'wildcard', 1);
+});
+
+lastStats = db.serverStatus().indexStats;
+
+const timeSeriesMetricIndexesEnabled = db.adminCommand({
+                                             getParameter: 1,
+                                             featureFlagTimeseriesMetricIndexes: 1
+                                         }).featureFlagTimeseriesMetricIndexes.value;
+if (timeSeriesMetricIndexesEnabled) {
+    assert.commandWorked(db.createCollection('ts', {timeseries: {timeField: 't'}}));
+    assert.commandWorked(db.ts.createIndex({loc: '2dsphere'}));
+    assert.commandWorked(db.ts.insert({t: new Date(), loc: [0, 0]}));
+    assert.eq(1,
+              db.ts
+                  .aggregate([{
+                                 $geoNear: {
+                                     near: [1, 1],
+                                     key: 'loc',
+                                     distanceField: 'dist',
+                                 }
+                             }],
+                             {hint: 'loc_2dsphere'})
+                  .itcount());
+    assertStats(db, (stats) => {
+        // Includes _id index built for system.views.
+        assertCountIncrease(lastStats, stats, 2);
+        assertFeatureCountIncrease(lastStats, stats, 'id', 1);
+        assertFeatureCountIncrease(lastStats, stats, 'single', 1);
+        assertFeatureCountIncrease(lastStats, stats, '2dsphere_bucket', 1);
+
+        assertFeatureAccessIncrease(lastStats, stats, 'id', 0);
+        assertFeatureAccessIncrease(lastStats, stats, 'single', 1);
+        assertFeatureAccessIncrease(lastStats, stats, '2dsphere_bucket', 1);
+    });
+}
+
+lastStats = db.serverStatus().indexStats;
+
+// After restarting the server, we expect all of the access counters to reset to zero, but that the
+// feature counters remain the same as before startup.
+replSet.stopSet(undefined, /* restart */ true);
+replSet.startSet({}, /* restart */ true);
+primary = replSet.getPrimary();
+db = primary.getDB('test');
+
+assertZeroAccess(db);
+assertStats(db, (stats) => {
+    assertCountIncrease(lastStats, stats, 0);
+
+    const features = stats.features;
+    for (const [feature, _] of Object.entries(features)) {
+        assert.contains(feature, knownFeatures);
+        assertFeatureCountIncrease(lastStats, stats, feature, 0);
+    }
+});
+
+assert.commandWorked(db.dropDatabase());
+assertZeroCounts(db);
+
+replSet.stopSet();
+})();

jstests/noPassthrough/set_fcv_prepared_transaction.js

@@ -0,0 +1,93 @@
+/**
+ * Tests running the setFeatureCompatibilityVersion command concurrently with a prepared
+ * transaction. Specifically, runs the setFCV right before the TransactionCoordinator writes the
+ * commit decision.
+ *
+ * @tags: [
+ *   requires_sharding,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+
+const st = new ShardingTest({shards: 2});
+const shard0Primary = st.rs0.getPrimary();
+
+// Set up a sharded collection with two chunks:
+// shard0: [MinKey, 0]
+// shard1: [0, MaxKey]
+const dbName = "testDb";
+const collName = "testColl";
+const ns = dbName + "." + collName;
+assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {x: 1}}));
+assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}}));
+assert.commandWorked(
+    st.s.adminCommand({moveChunk: ns, find: {x: MinKey}, to: st.shard0.shardName}));
+assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: 1}, to: st.shard1.shardName}));
+
+function runTxn(mongosHost, dbName, collName) {
+    const mongosConn = new Mongo(mongosHost);
+    jsTest.log("Starting a cross-shard transaction with shard0 and shard1 as the participants " +
+               "and shard0 as the coordinator shard");
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(35);
+    assert.commandWorked(mongosConn.getDB(dbName).runCommand({
+        insert: collName,
+        documents: [{x: -1}],
+        lsid,
+        txnNumber,
+        startTransaction: true,
+        autocommit: false,
+    }));
+    assert.commandWorked(mongosConn.getDB(dbName).runCommand({
+        insert: collName,
+        documents: [{x: 1}],
+        lsid,
+        txnNumber,
+        autocommit: false,
+    }));
+    assert.commandWorked(
+        mongosConn.adminCommand({commitTransaction: 1, lsid, txnNumber, autocommit: false}));
+    jsTest.log("Committed the cross-shard transaction");
+}
+
+function runSetFCV(primaryHost) {
+    const primaryConn = new Mongo(primaryHost);
+    jsTest.log("Starting a setFCV command on " + primaryHost);
+    assert.commandWorked(primaryConn.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+    jsTest.log("Completed the setFCV command");
+}
+
+// Run a cross-shard transaction that has shard0 as the coordinator. Make the TransactionCoordinator
+// thread hang right before the commit decision is written (i.e. after the transaction has entered
+// the "prepared" state).
+const writeDecisionFp = configureFailPoint(shard0Primary, "hangBeforeWritingDecision");
+const txnThread = new Thread(runTxn, st.s.host, dbName, collName);
+txnThread.start();
+writeDecisionFp.wait();
+
+// Run a setFCV command against shard0 and wait for the setFCV thread to start waiting to acquire
+// the setFCV S lock (i.e. waiting for existing prepared transactions to commit or abort).
+const setFCVThread = new Thread(runSetFCV, shard0Primary.host);
+setFCVThread.start();
+assert.soon(() => {
+    return shard0Primary.getDB(dbName).currentOp().inprog.find(
+        op => op.command && op.command.setFeatureCompatibilityVersion && op.locks &&
+            op.locks.FeatureCompatibilityVersion === "R" && op.waitingForLock === true);
+});
+
+// Unpause the TransactionCoordinator. The transaction should be able to commit despite the fact
+// that the FCV S lock is enqueued.
+writeDecisionFp.off();
+
+jsTest.log("Waiting for the cross-shard transaction to commit");
+txnThread.join();
+jsTest.log("Waiting for setFCV command to complete");
+setFCVThread.join();
+jsTest.log("Done");
+
+st.stop();
+})();

jstests/noPassthroughWithMongod/group_pushdown.js

@@ -247,6 +247,14 @@ assertResultsMatchWithAndWithoutPushdown(
     [{"_id": 1, "s": 2}, {"_id": 2, "s": 2}, {"_id": 4, "s": 1}],
     2);
 
+// Verifies that an optimized expression can be pushed down.
+assertResultsMatchWithAndWithoutPushdown(
+    coll,
+    // {"$ifNull": [1, 2]} will be optimized into just the constant 1.
+    [{$group: {_id: {"$ifNull": [1, 2]}, o: {$min: "$quantity"}}}],
+    [{"_id": 1, o: 1}],
+    1);
+
 // Run a group with a supported $stdDevSamp accumultor and check that it gets pushed down.
 assertGroupPushdown(coll,
                     [{$group: {_id: "$item", s: {$stdDevSamp: "$quantity"}}}],

jstests/noPassthroughWithMongod/reindex_duplicate_keys.js

@@ -0,0 +1,46 @@
+/**
+ * Tests that reIndex command fails with duplicate key error when there are duplicates in the
+ * collection.
+ */
+
+(function() {
+"use strict";
+
+const collNamePrefix = "reindex_duplicate_keys_";
+let count = 0;
+
+// Bypasses DuplicateKey insertion error for testing via failpoint.
+let addDuplicateDocumentsToCol = function(db, coll, doc) {
+    jsTestLog("Inserts documents without index entries.");
+    assert.commandWorked(
+        db.adminCommand({configureFailPoint: "skipIndexNewRecords", mode: "alwaysOn"}));
+
+    assert.commandWorked(coll.insert(doc));
+    assert.commandWorked(coll.insert(doc));
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "skipIndexNewRecords", mode: "off"}));
+};
+
+let runTest = function(doc) {
+    const collName = collNamePrefix + count++;
+    const coll = db.getCollection(collName);
+    coll.drop();
+
+    // Makes sure to create the _id index.
+    assert.commandWorked(db.createCollection(collName));
+    if (doc) {
+        assert.commandWorked(coll.createIndex(doc, {unique: true}));
+    } else {
+        doc = {_id: 1};
+    }
+
+    // Inserts two violating documents without indexing them.
+    addDuplicateDocumentsToCol(db, coll, doc);
+
+    // Checks reIndex command fails with duplicate key error.
+    assert.commandFailedWithCode(coll.reIndex(), ErrorCodes.DuplicateKey);
+};
+
+runTest();
+runTest({a: 1});
+})();

jstests/noPassthroughWithMongod/sort_big_documents_with_multi_planning.js

@@ -0,0 +1,46 @@
+// This test is designed to reproduce the test described in SERVER-65270, where a query
+// which uses multi-planning and large documents does not respect the sort order.
+(function() {
+"use strict";
+
+const coll = db.sort_big_documents_with_multi_planning;
+coll.drop();
+
+function makeDoc(i) {
+    return {_id: i, filterKey: 1, num: i, bytes: BinData(0, "A".repeat(13981014) + "==")};
+}
+
+for (let i = 0; i < 10; i++) {
+    assert.commandWorked(coll.insert(makeDoc(i)));
+}
+
+// Two possible indexes can answer the query.
+assert.commandWorked(coll.createIndex({filterKey: 1, num: 1, foo: 1}));
+assert.commandWorked(coll.createIndex({filterKey: 1, num: 1}));
+
+const sortSpec = {
+    num: 1
+};
+
+const kExpectedNums = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
+
+{
+    // We do a "client side projection," to avoid printing out the massive BinData string if
+    // there's an error.
+    const nums = [];
+    coll.find({filterKey: 1}).sort(sortSpec).forEach(doc => nums.push(doc.num));
+
+    // The results should be in order.
+    assert.eq(nums, kExpectedNums);
+}
+
+// Same test, but with aggregation.
+{
+    const nums = [];
+    coll.aggregate([{$match: {filterKey: 1}}, {$sort: sortSpec}])
+        .forEach(doc => nums.push(doc.num));
+
+    // The results should be in order.
+    assert.eq(nums, kExpectedNums);
+}
+})();

jstests/replsets/apply_ops_dropDatabase.js

@@ -0,0 +1,46 @@
+/**
+ * Ensures that the dropDatabase op can only be run if it's the single entry in an applyOps context.
+ * Exercises that the operation can run even if it has to await replication of the intermediate
+ * collection drops.
+ *
+ * @tags: [
+ *   requires_replication,
+ *   # dropDatabase can work in conjunction with other operations in 5.0.
+ *   requires_fcv_53
+ * ]
+ */
+(function() {
+"use strict";
+
+const rst = new ReplSetTest({nodes: 2});
+rst.startSet();
+rst.initiate();
+
+const dbName = "dbDrop";
+const collName = "coll";
+const cmdNss = dbName + ".$cmd";
+const primaryDB = rst.getPrimary().getDB(dbName);
+
+primaryDB.createCollection(collName);
+
+// Verify that dropDatabase is only supported if it's the only op entry.
+assert.commandFailedWithCode(primaryDB.adminCommand({
+    applyOps: [
+        {op: "c", ns: cmdNss, o: {create: "collection"}},
+        {op: "c", ns: cmdNss, o: {dropDatabase: 1}}
+    ]
+}),
+                             6275900);
+
+assert.contains(dbName, rst.getPrimary().getDBNames());
+assert.sameMembers(primaryDB.getCollectionNames(), [collName]);
+
+// Run the dropDatabase op on a database with an existing collection so that we can exercise
+// dropDatabase cleanly awaiting replication of the collection drop internally.
+assert.commandWorked(
+    primaryDB.adminCommand({applyOps: [{op: "c", ns: cmdNss, o: {dropDatabase: 1}}]}));
+
+assert.eq(rst.getPrimary().getDBNames().indexOf(dbName), -1);
+
+rst.stopSet();
+})();

jstests/replsets/buildindexes_false_commit_quorum.js

@@ -18,9 +18,11 @@ TestData.skipCheckDBHashes = true;
 const replTest = new ReplSetTest({
     name: jsTestName(),
     nodes: [
+        {},
         {},
         {},
         {rsConfig: {priority: 0, buildIndexes: false}},
+        {rsConfig: {priority: 0, votes: 0, buildIndexes: false}},
     ]
 });
 replTest.startSet();
@@ -44,12 +46,12 @@ assert.commandFailedWithCode(primaryDb.runCommand({
 }),
                              ErrorCodes.UnsatisfiableCommitQuorum);
 
-// With a commit quorum that includes all nodes, the quorum is unsatisfiable for the same reason as
-// 'votingMembers'.
+// With a commit quorum that includes 4 nodes, the quorum is unsatisfiable because it includes a
+// buildIndexes: false node.
 assert.commandFailedWithCode(primaryDb.runCommand({
     createIndexes: collName,
     indexes: [{key: {y: 1}, name: 'y_1_commitQuorum_3'}],
-    commitQuorum: 3,
+    commitQuorum: 4,
 }),
                              ErrorCodes.UnsatisfiableCommitQuorum);
 
@@ -71,7 +73,9 @@ replTest.getSecondaries().forEach((conn) => {
 });
 
 IndexBuildTest.assertIndexes(secondaryDbs[0][collName], 2, ['_id_', indexName]);
-IndexBuildTest.assertIndexes(secondaryDbs[1][collName], 1, ['_id_']);
+IndexBuildTest.assertIndexes(secondaryDbs[1][collName], 2, ['_id_', indexName]);
+IndexBuildTest.assertIndexes(secondaryDbs[2][collName], 1, ['_id_']);
+IndexBuildTest.assertIndexes(secondaryDbs[3][collName], 1, ['_id_']);
 
 replTest.stopSet();
 }());

jstests/replsets/dbhash_lock_acquisition.js

@@ -32,8 +32,13 @@ assert.commandWorked(sessionDB.mycoll.insert({}));
 const ops = db.currentOp({"lsid.id": session.getSessionId().id}).inprog;
 assert.eq(
     1, ops.length, () => "Failed to find session in currentOp() output: " + tojson(db.currentOp()));
-assert.eq(ops[0].locks,
-          {ReplicationStateTransition: "w", Global: "w", Database: "w", Collection: "w"});
+assert.eq(ops[0].locks, {
+    FeatureCompatibilityVersion: "w",
+    ReplicationStateTransition: "w",
+    Global: "w",
+    Database: "w",
+    Collection: "w",
+});
 
 const threadCaptruncCmd = new Thread(function(host) {
     try {
@@ -80,8 +85,13 @@ assert.soon(() => {
     if (ops.length === 0) {
         return false;
     }
-    assert.eq(ops[0].locks,
-              {ReplicationStateTransition: "w", Global: "r", Database: "r", Collection: "r"});
+    assert.eq(ops[0].locks, {
+        FeatureCompatibilityVersion: "r",
+        ReplicationStateTransition: "w",
+        Global: "r",
+        Database: "r",
+        Collection: "r",
+    });
     return true;
 }, () => "Failed to find create collection in currentOp() output: " + tojson(db.currentOp()));
 

jstests/replsets/rollback_views.js

@@ -101,6 +101,8 @@ assert.commandWorked(b3.z.remove({z: 1}));
 nodeB.disconnect(arbiter);
 replTest.awaitNoPrimary();
 nodeA.reconnect(arbiter);
+assert.soon(() => replTest.getPrimary() == nodeA, "nodeA did not become primary as expected");
+// Ensure that the arbiter recognizes nodeA as primary.
 replTest.awaitNodesAgreeOnPrimary(replTest.kDefaultTimeoutMS, [nodeA, arbiter], nodeA);
 
 // A is now primary and will perform writes that must be copied by B after rollback.

jstests/replsets/tenant_migration_fetch_committed_transactions.js

@@ -1,7 +1,7 @@
 /**
- * Tests that the migration recipient will retrieve committed transactions on the donor with a
- * 'lastWriteOpTime' before the stored 'startFetchingOpTime'. The recipient should store these
- * committed transaction entries in its own 'config.transactions' collection.
+ * Tests that the migration recipient will retrieve committed transactions on the donor
+ * with lastWriteOpTime <= the stored startApplyingOpTime. The recipient should store
+ * these committed transaction entries in its own 'config.transactions' collection.
  *
  * @tags: [
  *   incompatible_with_eft,
@@ -10,6 +10,7 @@
  *   requires_majority_read_concern,
  *   requires_persistence,
  *   serverless,
+ *   requires_fcv_53
  * ]
  */
 
@@ -22,24 +23,48 @@ load("jstests/replsets/libs/tenant_migration_util.js");
 load("jstests/replsets/rslib.js");
 load("jstests/libs/uuid_util.js");
 
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
-
 const tenantId = "testTenantId";
+const transactionsNS = "config.transactions";
+const collName = "testColl";
+
+const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
 const tenantDB = tenantMigrationTest.tenantDB(tenantId, "testDB");
 const nonTenantDB = tenantMigrationTest.nonTenantDB(tenantId, "testDB");
-const collName = "testColl";
 const tenantNS = `${tenantDB}.${collName}`;
-const transactionsNS = "config.transactions";
 
 const donorPrimary = tenantMigrationTest.getDonorPrimary();
 const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
 
+function validateTransactionEntryonRecipient(sessionId) {
+    const donorTxnEntry =
+        donorPrimary.getCollection(transactionsNS).findOne({"_id.id": sessionId.id});
+    const recipientTxnEntry =
+        recipientPrimary.getCollection(transactionsNS).findOne({"_id.id": sessionId.id});
+
+    assert.eq(donorTxnEntry.txnNum, recipientTxnEntry.txnNum);
+    assert.eq(donorTxnEntry.state, recipientTxnEntry.state);
+
+    // The recipient should have replaced the 'lastWriteOpTime' and 'lastWriteDate' fields.
+    assert.neq(donorTxnEntry.lastWriteOpTime, recipientTxnEntry.lastWriteOpTime);
+    assert.neq(donorTxnEntry.lastWriteDate, recipientTxnEntry.lastWriteDate);
+
+    // Test that the client can retry the first 'commitTransaction' on the recipient.
+    assert.commandWorked(recipientPrimary.adminCommand({
+        commitTransaction: 1,
+        lsid: recipientTxnEntry._id,
+        txnNumber: recipientTxnEntry.txnNum,
+        autocommit: false,
+    }));
+}
+
 assert.commandWorked(donorPrimary.getCollection(tenantNS).insert([{_id: 0, x: 0}, {_id: 1, x: 1}],
                                                                  {writeConcern: {w: "majority"}}));
 
+let sessionIdBeforeMigration;
 {
     jsTestLog("Run and commit a transaction prior to the migration");
     const session = donorPrimary.startSession({causalConsistency: false});
+    sessionIdBeforeMigration = session.getSessionId();
     const sessionDb = session.getDatabase(tenantDB);
     const sessionColl = sessionDb.getCollection(collName);
 
@@ -51,9 +76,7 @@ assert.commandWorked(donorPrimary.getCollection(tenantNS).insert([{_id: 0, x: 0}
     session.endSession();
 }
 
-// This should be the only transaction entry on the donor fetched by the recipient.
 assert.eq(1, donorPrimary.getCollection(transactionsNS).find().itcount());
-const donorTxnEntryBeforeMigration = donorPrimary.getCollection(transactionsNS).find().toArray()[0];
 
 {
     jsTestLog("Run and abort a transaction prior to the migration");
@@ -65,9 +88,9 @@ const donorTxnEntryBeforeMigration = donorPrimary.getCollection(transactionsNS).
     const findAndModifyRes0 = sessionColl.findAndModify({query: {x: 1}, remove: true});
     assert.eq({_id: 1, x: 1}, findAndModifyRes0);
 
-    // We prepare the transaction so that 'abortTransaction' will update the transactions table. We
-    // should later see that the recipient will not update its transactions table with this entry,
-    // since we only fetch committed transactions.
+    // We prepare the transaction so that 'abortTransaction' will update the transactions table.
+    // We should later see that the recipient will not update its transactions table with this
+    // entry, since we only fetch committed transactions.
     PrepareHelpers.prepareTransaction(session);
 
     assert.commandWorked(session.abortTransaction_forTesting());
@@ -75,6 +98,8 @@ const donorTxnEntryBeforeMigration = donorPrimary.getCollection(transactionsNS).
     session.endSession();
 }
 
+assert.eq(2, donorPrimary.getCollection(transactionsNS).find().itcount());
+
 {
     jsTestLog("Run and commit a transaction that does not belong to the tenant");
     const session = donorPrimary.startSession({causalConsistency: false});
@@ -87,9 +112,7 @@ const donorTxnEntryBeforeMigration = donorPrimary.getCollection(transactionsNS).
     session.endSession();
 }
 
-const donorTxnEntries = donorPrimary.getCollection(transactionsNS).find().toArray();
-jsTestLog(`All donor entries: ${tojson(donorTxnEntries)}`);
-assert.eq(3, donorTxnEntries.length, `donor transaction entries: ${tojson(donorTxnEntries)}`);
+assert.eq(3, donorPrimary.getCollection(transactionsNS).find().itcount());
 
 jsTestLog("Running a migration");
 const migrationId = UUID();
@@ -97,28 +120,40 @@ const migrationOpts = {
     migrationIdString: extractUUIDFromObject(migrationId),
     tenantId,
 };
-TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts));
 
-// Verify that the recipient has fetched and written only the first committed transaction entry from
-// the donor.
-assert.eq(1, recipientPrimary.getCollection(transactionsNS).find().itcount());
-const recipientTxnEntry = recipientPrimary.getCollection(transactionsNS).find().toArray()[0];
+const pauseAfterRetrievingLastTxnMigrationRecipientInstance =
+    configureFailPoint(recipientPrimary, "pauseAfterRetrievingLastTxnMigrationRecipientInstance");
 
-assert.eq(donorTxnEntryBeforeMigration._id, recipientTxnEntry._id);
-assert.eq(donorTxnEntryBeforeMigration.txnNum, recipientTxnEntry.txnNum);
-assert.eq(donorTxnEntryBeforeMigration.state, recipientTxnEntry.state);
+assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
 
-// The recipient should have replaced the 'lastWriteOpTime' and 'lastWriteDate' fields.
-assert.neq(donorTxnEntryBeforeMigration.lastWriteOpTime, recipientTxnEntry.lastWriteOpTime);
-assert.neq(donorTxnEntryBeforeMigration.lastWriteDate, recipientTxnEntry.lastWriteDate);
+pauseAfterRetrievingLastTxnMigrationRecipientInstance.wait();
 
-// Test that the client can retry 'commitTransaction' on the recipient.
-assert.commandWorked(recipientPrimary.adminCommand({
-    commitTransaction: 1,
-    lsid: donorTxnEntryBeforeMigration._id,
-    txnNumber: donorTxnEntryBeforeMigration.txnNum,
-    autocommit: false,
-}));
+let sessionIdBetweenFetchingAndApplyingOpTime;
+{
+    jsTestLog("Start and commit a transaction at startFetchingOpTime < t <= startApplyingOpTime");
+    const session = donorPrimary.startSession({causalConsistency: false});
+    sessionIdBetweenFetchingAndApplyingOpTime = session.getSessionId();
+    const sessionDb = session.getDatabase(tenantDB);
+    const sessionColl = sessionDb.getCollection(collName);
+    session.startTransaction({writeConcern: {w: "majority"}});
+    sessionColl.insert({doc: {}});
+    assert.commandWorked(session.commitTransaction_forTesting());
+    session.endSession();
+}
+
+assert.eq(4, donorPrimary.getCollection(transactionsNS).find().itcount());
+
+pauseAfterRetrievingLastTxnMigrationRecipientInstance.off();
+
+TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
+
+// Verify that the recipient has fetched and written the two committed transaction entries
+// from the donor.
+assert.eq(2, recipientPrimary.getCollection(transactionsNS).find().itcount());
+
+validateTransactionEntryonRecipient(sessionIdBeforeMigration);
+
+validateTransactionEntryonRecipient(sessionIdBetweenFetchingAndApplyingOpTime);
 
 tenantMigrationTest.stop();
 })();

jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js

@@ -0,0 +1,136 @@
+/**
+ * Tests that in tenant migration, the collection recreated on a dropped view namespace is handled
+ * correctly on resuming the logical tenant collection cloning phase due to recipient failover.
+ * @tags: [
+ *   incompatible_with_eft,
+ *   incompatible_with_macos,
+ *   incompatible_with_shard_merge,
+ *   incompatible_with_windows_tls,
+ *   requires_majority_read_concern,
+ *   requires_persistence,
+ *   serverless,
+ * ]
+ */
+
+(function() {
+"use strict";
+
+const tenantMigrationFailoverTest = function(isTimeSeries, createCollFn) {
+    load("jstests/libs/fail_point_util.js");
+    load("jstests/libs/uuid_util.js");  // for 'extractUUIDFromObject'
+    load("jstests/replsets/libs/tenant_migration_test.js");
+    load("jstests/replsets/libs/tenant_migration_util.js");
+
+    const recipientRst = new ReplSetTest({
+        nodes: 2,
+        name: jsTestName() + "_recipient",
+        nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().recipient, {
+            setParameter: {
+                // Allow reads on recipient before migration completes for testing.
+                'failpoint.tenantMigrationRecipientNotRejectReads': tojson({mode: 'alwaysOn'}),
+            }
+        })
+    });
+
+    recipientRst.startSet();
+    recipientRst.initiate();
+
+    const tenantMigrationTest =
+        new TenantMigrationTest({name: jsTestName(), recipientRst: recipientRst});
+
+    const donorRst = tenantMigrationTest.getDonorRst();
+    const donorPrimary = donorRst.getPrimary();
+
+    const tenantId = "testTenantId";
+    const dbName = tenantMigrationTest.tenantDB(tenantId, "testDB");
+    const donorDB = donorPrimary.getDB(dbName);
+    const collName = "testColl";
+    const donorColl = donorDB[collName];
+
+    let getCollectionInfo = function(conn) {
+        return conn.getDB(dbName).getCollectionInfos().filter(coll => {
+            return coll.name === collName;
+        });
+    };
+
+    // Create a timeseries collection or a regular view.
+    assert.commandWorked(createCollFn(donorDB, collName));
+    donorRst.awaitReplication();
+
+    const migrationId = UUID();
+    const migrationIdString = extractUUIDFromObject(migrationId);
+    const migrationOpts = {
+        migrationIdString: migrationIdString,
+        recipientConnString: tenantMigrationTest.getRecipientConnString(),
+        tenantId: tenantId,
+    };
+
+    const recipientPrimary = recipientRst.getPrimary();
+    const recipientDb = recipientPrimary.getDB(dbName);
+    const recipientSystemViewsColl = recipientDb.getCollection("system.views");
+
+    // Configure a fail point to have the recipient primary hang after cloning
+    // "testTenantId_testDB.system.views" collection.
+    const hangDuringCollectionClone =
+        configureFailPoint(recipientPrimary,
+                           "tenantMigrationHangCollectionClonerAfterHandlingBatchResponse",
+                           {nss: recipientSystemViewsColl.getFullName()});
+
+    // Start the migration and wait for the migration to hang after cloning
+    // "testTenantId_testDB.system.views" collection.
+    assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
+    hangDuringCollectionClone.wait();
+
+    assert.soon(() => recipientSystemViewsColl.find().itcount() >= 1);
+    recipientRst.awaitLastOpCommitted();
+    const newRecipientPrimary = recipientRst.getSecondaries()[0];
+
+    // Verify that a view has been registered for "testTenantId_testDB.testColl" on the new
+    // recipient primary.
+    let collectionInfo = getCollectionInfo(newRecipientPrimary);
+    assert.eq(1, collectionInfo.length);
+    assert(collectionInfo[0].type === (isTimeSeries ? "timeseries" : "view"),
+           "data store type mismatch: " + tojson(collectionInfo[0]));
+
+    // Drop the view and create a regular collection with the same namespace as the
+    // dropped view on donor.
+    assert(donorColl.drop());
+    assert.commandWorked(donorDB.createCollection(collName));
+
+    // We need to skip TenantDatabaseCloner::listExistingCollectionsStage() to make sure
+    // the recipient always clone the above newly created regular collection after the failover.
+    // Currently, we restart cloning after a failover, only from the collection whose UUID is
+    // greater than or equal to the last collection we have on disk.
+    const skiplistExistingCollectionsStage =
+        configureFailPoint(newRecipientPrimary, "skiplistExistingCollectionsStage");
+
+    // Step up a new node in the recipient set and trigger a failover.
+    recipientRst.stepUp(newRecipientPrimary);
+    hangDuringCollectionClone.off();
+
+    // The migration should go through after recipient failover.
+    TenantMigrationTest.assertCommitted(
+        tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
+
+    // Check that recipient has dropped the view and and re-created the regular collection as part
+    // of migration oplog catchup phase.
+    collectionInfo = getCollectionInfo(newRecipientPrimary);
+    assert.eq(1, collectionInfo.length);
+    assert(collectionInfo[0].type === "collection",
+           "data store type mismatch: " + tojson(collectionInfo[0]));
+
+    tenantMigrationTest.stop();
+    recipientRst.stopSet();
+};
+
+jsTestLog("Running tenant migration test for time-series collection");
+// Creating a timeseries collection, implicity creates a view on the 'collName' collection
+// namespace.
+tenantMigrationFailoverTest(true,
+                            (db, collName) => db.createCollection(
+                                collName, {timeseries: {timeField: "time", metaField: "bucket"}}));
+
+jsTestLog("Running tenant migration test for regular view");
+tenantMigrationFailoverTest(false,
+                            (db, collName) => db.createView(collName, "sourceCollection", []));
+})();

jstests/sharding/auth_add_shard.js

@@ -100,6 +100,20 @@ assert.soon(function() {
     return result.ok && result.state == "completed";
 }, "failed to drain shard completely", 5 * 60 * 1000);
 
+// create user directly on new shard to allow direct reads from config.migrationCoordinators
+rst.getPrimary()
+    .getDB(adminUser.db)
+    .createUser({user: adminUser.username, pwd: adminUser.password, roles: jsTest.adminUserRoles});
+rst.getPrimary().getDB(adminUser.db).auth(adminUser.username, adminUser.password);
+
+// wait until migration coordinator is finished
+assert.soon(function() {
+    let migrationCoordinatorDocs =
+        rst.getPrimary().getDB('config').migrationCoordinators.find().toArray();
+
+    return migrationCoordinatorDocs.length === 0;
+}, "failed to remove migration coordinator", 5 * 60 * 1000);
+
 assert.eq(1, st.config.shards.count(), "removed server still appears in count");
 
 rst.stopSet();

jstests/sharding/balancer_collection_status.js

@@ -5,7 +5,15 @@
 (function() {
 'use strict';
 
-var st = new ShardingTest({mongos: 1, shards: 3});
+const chunkSizeMB = 1;
+
+let st = new ShardingTest({
+    shards: 3,
+    other: {
+        // Set global max chunk size to 1MB
+        chunkSize: chunkSizeMB
+    }
+});
 
 function runBalancer(rounds) {
     st.startBalancer();
@@ -28,7 +36,7 @@ assert.commandFailedWithCode(st.s0.adminCommand({balancerCollectionStatus: 'db'}
 
 // only sharded databases are allowed
 assert.commandFailedWithCode(st.s0.adminCommand({balancerCollectionStatus: 'db.col'}),
-                             ErrorCodes.NamespaceNotFound);
+                             ErrorCodes.NamespaceNotSharded);
 
 // setup the collection for the test
 assert.commandWorked(st.s0.adminCommand({enableSharding: 'db'}));
@@ -39,13 +47,13 @@ assert.commandWorked(st.s0.getDB('db').runCommand({create: "col2"}));
 assert.commandFailedWithCode(st.s0.adminCommand({balancerCollectionStatus: 'db.col2'}),
                              ErrorCodes.NamespaceNotSharded);
 
-var result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'}));
+let result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'}));
 
 // new collections must be balanced
 assert.eq(result.balancerCompliant, true);
 
 // get shardIds
-var shards = st.s0.getDB('config').shards.find().toArray();
+const shards = st.s0.getDB('config').shards.find().toArray();
 
 // manually split and place the 3 chunks on the same shard
 assert.commandWorked(st.s0.adminCommand({split: 'db.col', middle: {key: 10}}));
@@ -98,5 +106,11 @@ result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.
 // All chunks are balanced and in the correct zone
 assert.eq(result.balancerCompliant, true);
 
+const configDB = st.configRS.getPrimary().getDB('config');
+const fcvDoc = configDB.adminCommand({getParameter: 1, featureCompatibilityVersion: 1});
+if (MongoRunner.compareBinVersions(fcvDoc.featureCompatibilityVersion.version, '5.3') >= 0) {
+    // Ensure that the expected chunk size is part of the response.
+    assert.eq(result.chunkSize, chunkSizeMB);
+}
 st.stop();
-})();
+})();

jstests/sharding/balancer_defragmentation_merge_chunks.js

@@ -61,7 +61,8 @@ function setupCollection() {
                                                    targetChunkSizeMB / 2 /* maxChunkFillMB */,
                                                    0 /* numZones */,
                                                    32 * 1024 /* docSizeBytes */,
-                                                   1000 /* chunkSpacing */);
+                                                   1000 /* chunkSpacing */,
+                                                   false /* disableCollectionBalancing */);
     jsTest.log("Collection " + coll.getFullName() + ", number of chunks before defragmentation: " +
                findChunksUtil.countChunksForNs(st.s.getDB('config'), coll.getFullName()));
     return coll;
@@ -116,8 +117,11 @@ jsTest.log("Split chunks while defragmenting");
     const chunks = findChunksUtil.findChunksByNs(st.config, nss).toArray();
     assert.eq(1, chunks.length);
     assert.commandWorked(st.s.adminCommand({split: nss, middle: {skey: 0}}));
-    assert.commandWorked(st.s.adminCommand(
-        {moveChunk: nss, find: {skey: 0}, to: st.getOther(chunks[0]['shard']).name}));
+
+    const primaryShard = st.getPrimaryShard(coll.getDB().getName());
+    assert.eq(st.normalize(primaryShard.name), st.normalize(chunks[0]['shard']));
+    assert.commandWorked(
+        st.s.adminCommand({moveChunk: nss, find: {skey: 0}, to: st.getOther(primaryShard).name}));
 
     // Pause defragmentation after initialization but before phase 1 runs
     setFailPointOnConfigNodes("afterBuildingNextDefragmentationPhase", {skip: 1});
@@ -272,10 +276,6 @@ jsTest.log("Changed uuid causes defragmentation to restart");
     st.startBalancer();
     // Reshard collection
     assert.commandWorked(db.adminCommand({reshardCollection: nss, key: {key2: 1}}));
-    assert.commandWorked(
-        db.adminCommand({moveChunk: nss, find: {key2: MinKey}, to: st.shard0.shardName}));
-    assert.commandWorked(
-        db.adminCommand({moveChunk: nss, find: {key2: 1}, to: st.shard0.shardName}));
     // Let defragementation run
     clearFailPointOnConfigNodes("afterBuildingNextDefragmentationPhase");
     defragmentationUtil.waitForEndOfDefragmentation(st.s, nss);

jstests/sharding/count1.js

@@ -146,11 +146,11 @@ assert.commandFailedWithCode(db.runCommand({count: 'foo', query: {$c: {$abc: 3}}
                              ErrorCodes.BadValue);
 
 // ii. Negative skip values should return error.
-assert.commandFailedWithCode(db.runCommand({count: 'foo', skip: -2}), ErrorCodes.FailedToParse);
-
+assert.commandFailedWithCode(db.runCommand({count: 'foo', skip: -2}),
+                             [ErrorCodes.FailedToParse, 51024]);
 // iii. Negative skip values with positive limit should return error.
 assert.commandFailedWithCode(db.runCommand({count: 'foo', skip: -2, limit: 1}),
-                             ErrorCodes.FailedToParse);
+                             [ErrorCodes.FailedToParse, 51024]);
 
 // iv. Unknown options should return error.
 assert.commandFailedWithCode(db.runCommand({count: 'foo', random: true}), 40415);

jstests/sharding/defragment_large_collection.js

@@ -47,8 +47,14 @@ for (let i = 0; i < numCollections; ++i) {
 
     const coll = db[coll_prefix + i];
 
-    defragmentationUtil.createFragmentedCollection(
-        st.s, coll.getFullName(), numChunks, maxChunkFillMB, numZones, docSizeBytes, chunkSpacing);
+    defragmentationUtil.createFragmentedCollection(st.s,
+                                                   coll.getFullName(),
+                                                   numChunks,
+                                                   maxChunkFillMB,
+                                                   numZones,
+                                                   docSizeBytes,
+                                                   chunkSpacing,
+                                                   true);
 
     collections.push(coll);
 }

jstests/sharding/libs/defragmentation_util.js

@@ -1,12 +1,23 @@
 var defragmentationUtil = (function() {
     load("jstests/sharding/libs/find_chunks_util.js");
 
-    let createFragmentedCollection = function(
-        mongos, ns, numChunks, maxChunkFillMB, numZones, docSizeBytes, chunkSpacing) {
+    let createFragmentedCollection = function(mongos,
+                                              ns,
+                                              numChunks,
+                                              maxChunkFillMB,
+                                              numZones,
+                                              docSizeBytes,
+                                              chunkSpacing,
+                                              disableCollectionBalancing) {
         jsTest.log("Creating fragmented collection " + ns + " with parameters: numChunks = " +
                    numChunks + ", numZones = " + numZones + ", docSizeBytes = " + docSizeBytes +
                    ", maxChunkFillMB = " + maxChunkFillMB + ", chunkSpacing = " + chunkSpacing);
         assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {key: 1}}));
+        // Turn off balancer for this collection
+        if (disableCollectionBalancing) {
+            assert.commandWorked(
+                mongos.getDB('config').collections.update({_id: ns}, {$set: {"noBalance": true}}));
+        }
 
         createAndDistributeChunks(mongos, ns, numChunks, chunkSpacing);
         createRandomZones(mongos, ns, numZones, chunkSpacing);
@@ -84,8 +95,9 @@ var defragmentationUtil = (function() {
 
     let checkPostDefragmentationState = function(mongos, ns, maxChunkSizeMB, shardKey) {
         const oversizedChunkThreshold = maxChunkSizeMB * 1024 * 1024 * 4 / 3;
-        const chunks =
-            findChunksUtil.findChunksByNs(mongos.getDB('config'), ns).sort({shardKey: 1}).toArray();
+        const chunks = findChunksUtil.findChunksByNs(mongos.getDB('config'), ns)
+                           .sort({[shardKey]: 1})
+                           .toArray();
         const coll = mongos.getCollection(ns);
         const pipeline = [
             {'$collStats': {'storageStats': {}}},
@@ -94,12 +106,16 @@ var defragmentationUtil = (function() {
         const storageStats = coll.aggregate(pipeline).toArray();
         let avgObjSizeByShard = {};
         storageStats.forEach((storageStat) => {
-            avgObjSizeByShard[storageStat['shard']] = storageStat['storageStats']['avgObjSize'];
+            avgObjSizeByShard[storageStat['shard']] =
+                typeof (storageStat['storageStats']['avgObjSize']) === "undefined"
+                ? 0
+                : storageStat['storageStats']['avgObjSize'];
         });
         let checkForOversizedChunk = function(
             coll, chunk, shardKey, avgObjSize, oversizedChunkThreshold) {
             let chunkSize =
-                coll.countDocuments({key: {$gte: chunk.min[shardKey], $lt: chunk.max[shardKey]}}) *
+                coll.countDocuments(
+                    {[shardKey]: {$gte: chunk.min[shardKey], $lt: chunk.max[shardKey]}}) *
                 avgObjSize;
             assert.lte(
                 chunkSize,
@@ -111,16 +127,22 @@ var defragmentationUtil = (function() {
             let chunk1 = chunks[i - 1];
             let chunk2 = chunks[i];
             // Check for mergeable chunks with combined size less than maxChunkSize
-            if (chunk1["shard"] === chunk2["shard"] && chunk1["max"] === chunk2["min"]) {
+            if (chunk1["shard"] === chunk2["shard"] &&
+                bsonWoCompare(chunk1["max"], chunk2["min"]) === 0) {
                 let chunk1Zone = getZoneForRange(mongos, ns, chunk1.min, chunk1.max);
                 let chunk2Zone = getZoneForRange(mongos, ns, chunk2.min, chunk2.max);
-                if (chunk1Zone === chunk2Zone) {
-                    let combinedDataSize = coll.countDocuments({
-                        shardKey: {$gte: chunk1.min[shardKey], $lt: chunk2.max[shardKey]}
-                    }) * avgObjSizeByShard[chunk1['shard']];
-                    assert.lte(
+                if (bsonWoCompare(chunk1Zone, chunk2Zone) === 0) {
+                    let combinedDataSize =
+                        coll.countDocuments(
+                            {[shardKey]: {$gte: chunk1.min[shardKey], $lt: chunk2.max[shardKey]}}) *
+                        avgObjSizeByShard[chunk1['shard']];
+                    // The autosplitter should not split chunks whose combined size is < 133% of
+                    // maxChunkSize but this threshold may be off by a few documents depending on
+                    // rounding of avgObjSize.
+                    const autosplitRoundingTolerance = 3 * avgObjSizeByShard[chunk1['shard']];
+                    assert.gte(
                         combinedDataSize,
-                        oversizedChunkThreshold,
+                        oversizedChunkThreshold - autosplitRoundingTolerance,
                         `Chunks ${tojson(chunk1)} and ${
                             tojson(chunk2)} are mergeable with combined size ${combinedDataSize}`);
                 }
@@ -144,7 +166,7 @@ var defragmentationUtil = (function() {
         const tags = mongos.getDB('config')
                          .tags.find({ns: ns, min: {$lte: minKey}, max: {$gte: maxKey}})
                          .toArray();
-        assert.leq(tags.length, 1);
+        assert.lte(tags.length, 1);
         if (tags.length === 1) {
             return tags[0].tag;
         }

jstests/sharding/libs/mongos_api_params_util.js

@@ -297,13 +297,13 @@ let MongosAPIParametersUtil = (function() {
         {
             commandName: "count",
             run: {
-                inAPIVersion1: false,
+                inAPIVersion1: true,
                 shardCommandName: "count",
                 permittedInTxn: false,
                 command: () => ({count: "collection"})
             },
             explain: {
-                inAPIVersion1: false,
+                inAPIVersion1: true,
                 shardCommandName: "explain",
                 permittedInTxn: false,
                 command: () => ({explain: {count: "collection"}})
@@ -312,13 +312,13 @@ let MongosAPIParametersUtil = (function() {
         {
             commandName: "count",
             run: {
-                inAPIVersion1: false,
+                inAPIVersion1: true,
                 shardCommandName: "count",
                 permittedInTxn: false,
                 command: () => ({count: "collection", query: {x: 1}})
             },
             explain: {
-                inAPIVersion1: false,
+                inAPIVersion1: true,
                 shardCommandName: "explain",
                 permittedInTxn: false,
                 command: () => ({explain: {count: "collection", query: {x: 1}}})
@@ -1263,7 +1263,7 @@ let MongosAPIParametersUtil = (function() {
                     keyPattern: {_id: 1},
                     min: {_id: 0},
                     max: {_id: MaxKey},
-                    maxChunkSizeBytes: 1024
+                    maxChunkSizeBytes: 1024 * 1024
                 })
             }
         },

jstests/sharding/load_balancer_support/disconnect_aborts_txns.js

@@ -1,6 +1,6 @@
 /**
- * @tags: [featureFlagLoadBalancer, uses_transactions, uses_multi_shard_transaction,
- * requires_sharding, requires_fcv_51]
+ * @tags: [uses_transactions, uses_multi_shard_transaction,
+ * requires_sharding, requires_fcv_53]
  *
  * Tests that when a load-balanced client disconnects, its in-progress transactions are aborted
  */

jstests/sharding/load_balancer_support/disconnect_kills_cursors.js

@@ -1,5 +1,5 @@
 /**
- * @tags: [requires_fcv_51, featureFlagLoadBalancer]
+ * @tags: [requires_fcv_53]
  *
  * Tests that when a load-balanced client disconnects, its cursors are killed.
  */

jstests/sharding/load_balancer_support/load_balanced_server_status_metrics.js

@@ -1,5 +1,5 @@
 /**
- * @tags: [requires_fcv_51, featureFlagLoadBalancer]
+ * @tags: [requires_fcv_53]
  *
  * Tests that load-balanced connections are reported correctly in server status metrics.
  */

jstests/sharding/load_balancer_support/load_balancer_hello.js

@@ -1,5 +1,5 @@
 /**
- * @tags: [requires_fcv_51, featureFlagLoadBalancer]
+ * @tags: [requires_fcv_53]
  *
  * Test the extension to the mongos `hello` command by which clients
  * that have arrived through a load balancer affirm that they are

jstests/sharding/proxy_protocol_connect.js

@@ -18,12 +18,8 @@ function testProxyProtocolConnect(ingressPort, egressPort, version) {
     let proxy_server = new ProxyProtocolServer(ingressPort, egressPort, version);
     proxy_server.start();
 
-    let st = new ShardingTest({
-        shards: 1,
-        mongos: 1,
-        mongosOptions:
-            {setParameter: {"featureFlagLoadBalancer": true, "loadBalancerPort": egressPort}}
-    });
+    let st = new ShardingTest(
+        {shards: 1, mongos: 1, mongosOptions: {setParameter: {"loadBalancerPort": egressPort}}});
 
     const uri = `mongodb://127.0.0.1:${ingressPort}/?loadBalanced=true`;
     const conn = new Mongo(uri);
@@ -37,11 +33,8 @@ function testProxyProtocolConnect(ingressPort, egressPort, version) {
 function testProxyProtocolConnectFailure(lbPort, sendLoadBalanced) {
     'use strict';
 
-    let st = new ShardingTest({
-        shards: 1,
-        mongos: 1,
-        mongosOptions: {setParameter: {"featureFlagLoadBalancer": true, "loadBalancerPort": lbPort}}
-    });
+    let st = new ShardingTest(
+        {shards: 1, mongos: 1, mongosOptions: {setParameter: {"loadBalancerPort": lbPort}}});
 
     const hostName = st.s.host.substring(0, st.s.host.indexOf(":"));
     const uri = `mongodb://${hostName}:${lbPort}/?loadBalanced=${sendLoadBalanced}`;

jstests/sharding/query/pipeline_length_limit.js

@@ -164,18 +164,22 @@ function runTest(lengthLimit, mongosConfig = {}, mongodConfig = {}) {
     st.stop();
 }
 
+// This is a sanity check to make sure that the default value is correct. If the limit is changed,
+// it will break for users and this check catches that.
 const st = new ShardingTest({shards: 1, rs: {nodes: 1}});
-const debugBuild = st.s0.getDB("TestDB").adminCommand("buildInfo").debug;
-st.stop();
+let buildInfo = assert.commandWorked(st.s0.getDB("test").adminCommand("buildInfo"));
+let pipelineLimit =
+    assert.commandWorked(st.s0.adminCommand({"getParameter": 1, "internalPipelineLengthLimit": 1}));
+let expectedPipelineLimit = buildInfo.debug ? 200 : 1000;
+assert.eq(expectedPipelineLimit, pipelineLimit["internalPipelineLengthLimit"]);
 
-if (!debugBuild) {
-    // Test default pipeline length limit.
-    runTest(1000);
-} else {
-    // In debug builds we need to run with a lower limit because the available stack space is half
-    // what is available in normal builds.
-    runTest(200);
-}
+const shardPrimary = st.rs0.getPrimary().getDB("test");
+buildInfo = assert.commandWorked(shardPrimary.adminCommand("buildInfo"));
+expectedPipelineLimit = buildInfo.debug ? 200 : 1000;
+pipelineLimit = assert.commandWorked(
+    shardPrimary.adminCommand({"getParameter": 1, "internalPipelineLengthLimit": 1}));
+assert.eq(expectedPipelineLimit, pipelineLimit["internalPipelineLengthLimit"]);
+st.stop();
 
 // Test with modified pipeline length limit.
 runTest(50,

jstests/sharding/refine_collection_shard_key_basic.js

@@ -274,6 +274,11 @@ assert.commandFailedWithCode(
     mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
     ErrorCodes.NamespaceNotSharded);
 
+// Should fail because operation can't run on config server
+assert.commandFailedWithCode(
+    mongos.adminCommand({refineCollectionShardKey: "config.collections", key: {_id: 1, aKey: 1}}),
+    ErrorCodes.NoShardingEnabled);
+
 enableShardingAndShardColl({_id: 1});
 
 // Should fail because shard key is invalid (i.e. bad values).

jstests/sharding/resharding_change_stream_namespace_filtering.js

@@ -0,0 +1,65 @@
+/*
+ * Tests that a change stream on collection X doesn't erroneously see resharding events which occur
+ * on collection Y. Exercises the fix for SERVER-64780.
+ * @tags: [
+ *     uses_change_streams,
+ *     requires_fcv_50,
+ *     # TODO SERVER-66034: consider removing this tag
+ *     multiversion_incompatible
+ * ]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/libs/collection_drop_recreate.js");  // For assertDropAndRecreateCollection.
+load("jstests/libs/chunk_manipulation_util.js");   // For pauseMigrateAtStep, waitForMigrateStep and
+                                                   // unpauseMigrateAtStep.
+
+const st = new ShardingTest({
+    shards: 2,
+    rs: {nodes: 1, setParameter: {writePeriodicNoops: true, periodicNoopIntervalSecs: 1}},
+    other: {
+        configOptions: {setParameter: {reshardingCriticalSectionTimeoutMillis: 24 * 60 * 60 * 1000}}
+    }
+});
+
+const dbName = jsTestName();
+const reshardCollName = "coll_reshard";
+const otherCollName = "coll_other";
+
+const mongosDB = st.s.getDB(dbName);
+const mongosReshardColl = mongosDB[reshardCollName];
+
+const mongosOtherColl = mongosDB[otherCollName];
+const shardOtherColl = st.rs0.getPrimary().getDB(dbName)[otherCollName];
+
+// Open a {showMigrationEvents:true} change stream directly on the shard, monitoring events on
+// 'coll_other'.
+const shardOtherCollCsCursor =
+    shardOtherColl.aggregate([{$changeStream: {showMigrationEvents: true}}]);
+
+// Drop, recreate, and shard the 'coll_reshard' collection.
+assertDropAndRecreateCollection(mongosDB, reshardCollName);
+
+st.ensurePrimaryShard(dbName, st.rs0.name);
+st.shardColl(mongosReshardColl, {a: 1}, {a: 50});
+
+for (let i = 0; i < 100; ++i) {
+    assert.commandWorked(mongosReshardColl.insert({a: i, b: -i}));
+}
+
+// Reshard the 'coll_reshard' collection on {b: 1}.
+assert.commandWorked(
+    mongosDB.adminCommand({reshardCollection: mongosReshardColl.getFullName(), key: {b: 1}}));
+
+// Confirm that the change stream we opened on 'coll_other' only sees the sentinel 'insert' but does
+// not see the earlier 'reshardBegin' or 'reshardDoneCatchUp' events on the 'coll_reshard'
+// collection.
+assert.commandWorked(mongosOtherColl.insert({_id: "sentinel_write"}));
+
+assert.soon(() => shardOtherCollCsCursor.hasNext());
+assert.eq(shardOtherCollCsCursor.next().operationType, "insert");
+
+st.stop();
+})();

jstests/sharding/resharding_coordinator_recovers_abort_decision.js

@@ -48,9 +48,19 @@ reshardingTest.withReshardingInBackground(
         // Wait until participants are aware of the resharding operation.
         reshardingTest.awaitCloneTimestampChosen();
 
+        const ns = sourceCollection.getFullName();
         awaitAbort = startParallelShell(funWithArgs(function(ns) {
                                             db.adminCommand({abortReshardCollection: ns});
-                                        }, sourceCollection.getFullName()), mongos.port);
+                                        }, ns), mongos.port);
+
+        // Wait for the coordinator to have persisted its decision to abort the resharding operation
+        // as a result of the abortReshardCollection command being processed.
+        assert.soon(() => {
+            const coordinatorDoc =
+                mongos.getCollection("config.reshardingOperations").findOne({ns: ns});
+
+            return coordinatorDoc !== null && coordinatorDoc.state === "aborting";
+        });
     },
     {
         expectedErrorCode: ErrorCodes.ReshardCollectionAborted,

jstests/sharding/resharding_histogram_metrics.js

@@ -1,10 +1,10 @@
-// Test to verify that latency metrics are collected in both currentOp and cumulativeOp
-// during resharding.
-//
-// @tags: [
-//   uses_atclustertime,
-// ]
-//
+/**
+ * Test to verify that latency metrics are collected in both currentOp and cumulativeOp during
+ * resharding.
+ * @tags: [
+ *  uses_atclustertime,
+ * ]
+ */
 
 (function() {
 'use strict';
@@ -65,6 +65,14 @@ function getReshardingMetricsReport(mongo, role) {
     }
 }
 
+function readHistogramTotal(histogram) {
+    let total = histogram["totalCount"];
+    if (total === undefined) {
+        total = histogram["ops"];
+    }
+    return total;
+}
+
 const mongos = testColl.getMongo();
 const topology = DiscoverTopology.findConnectedNodes(mongos);
 const recipientShardNames = reshardingTest.recipientShardNames;
@@ -122,12 +130,12 @@ reshardingTest.withReshardingInBackground(
                 // We expect 1 batch insert per document on each shard, plus 1 empty batch
                 // to discover no documents are left.
                 const expectedBatchInserts = reshardingMetrics[kDocumentsCopied] + 1;
-                const receivedBatchInserts = collClonerFillBatchForInsertHist["ops"];
+                const receivedBatchInserts = readHistogramTotal(collClonerFillBatchForInsertHist);
                 assert(expectedBatchInserts == receivedBatchInserts,
                        `expected ${expectedBatchInserts} batch inserts,
                        received ${receivedBatchInserts}`);
 
-                firstReshardBatchApplies += oplogApplierApplyBatchHist["ops"];
+                firstReshardBatchApplies += readHistogramTotal(oplogApplierApplyBatchHist);
             });
 
             assert(firstReshardBatchApplies > 0,
@@ -174,8 +182,8 @@ recipientShardNames.forEach(function(shardName) {
     const collClonerFillBatchForInsertHist =
         reshardingMetrics[kCollClonerFillBatchForInsertLatencyMillis];
 
-    cumulativeBatchApplies += oplogApplierApplyBatchHist["ops"];
-    cumulativeBatchInserts += collClonerFillBatchForInsertHist["ops"];
+    cumulativeBatchApplies += readHistogramTotal(oplogApplierApplyBatchHist);
+    cumulativeBatchInserts += readHistogramTotal(collClonerFillBatchForInsertHist);
     totalDocumentsCopied += reshardingMetrics[kDocumentsCopied];
 });
 

jstests/sharding/sample_timeseries.js

@@ -156,7 +156,8 @@ function containsDocs(actualDocs, expectedDocs) {
 const randomCursor = "COLLSCAN";
 const topK = "UNPACK_BUCKET";
 const arhash = "QUEUED_DATA";
-function assertPlanForSampleOnShard({root, planName}) {
+
+function checkShardPlanHasStage({root, planName}) {
     // The plan should only contain a TRIAL stage if we had to evaluate whether an ARHASH or Top-K
     // plan was best.
     const hasTrialStage = planHasStage(testDB, root, "TRIAL");
@@ -166,23 +167,27 @@ function assertPlanForSampleOnShard({root, planName}) {
         assert(hasTrialStage, root);
     }
 
-    // Ensure the plan contains the stage we expect to see for that plan.
-    assert(planHasStage(testDB, root, planName), root);
     if (planName !== arhash) {
         // The plan should always filter out orphans, but we only see this stage in the top-K case.
         assert(planHasStage(testDB, root, "SHARDING_FILTER"), root);
     }
+
+    return planHasStage(testDB, root, planName);
 }
 
-function assertPlanForSample({explainRes, planForShards}) {
-    const shardsExplain = explainRes.shards;
+function assertPlanForSample({explainResults, expectedPlan}) {
     for (const shardName of [primary.shardName, otherShard.shardName]) {
-        const root = shardsExplain[shardName].stages[0].$cursor;
-        assertPlanForSampleOnShard({root, planName: planForShards[shardName]});
+        let shardHasPlan = false;
+        for (const explainRes of explainResults) {
+            const shardsExplain = explainRes.shards;
+            const root = shardsExplain[shardName].stages[0].$cursor;
+            shardHasPlan = shardHasPlan || checkShardPlanHasStage({root, planName: expectedPlan});
+        }
+        assert(shardHasPlan, {shardName: shardName, explain: explainResults});
     }
 }
 
-function testPipeline({pipeline, expectedDocs, expectedCount, shardsTargetedCount, planForShards}) {
+function testPipeline({pipeline, expectedDocs, expectedCount, shardsTargetedCount, expectedPlan}) {
     // Restart profiling.
     for (const db of [primaryDB, otherShardDB]) {
         db.setProfilingLevel(0);
@@ -194,9 +199,15 @@ function testPipeline({pipeline, expectedDocs, expectedCount, shardsTargetedCoun
     const result = testColl.aggregate(pipeline).toArray();
 
     // Verify plan used.
-    if (planForShards) {
-        const explainRes = testColl.explain().aggregate(pipeline);
-        assertPlanForSample({explainRes, planForShards});
+    if (expectedPlan) {
+        // The ARHash plan is probabilistic. We may not always pick the plan. So we run the explain
+        // command three times to increase the chance of the plan getting picked.
+        const numInteration = (expectedPlan == arhash) ? 3 : 1;
+        const explainResults = [];
+        for (let i = 0; i < numInteration; ++i) {
+            explainResults.push(testColl.explain().aggregate(pipeline));
+        }
+        assertPlanForSample({explainResults, expectedPlan});
     }
 
     if (expectedCount) {
@@ -245,15 +256,15 @@ const projection = {
  *  4. Sample the given 'proportion' of non-Dublin (Galway, Cork) documents, which can be found on
  * both shards, and ensure we target both shards.
  */
-function runTest({proportion, planForShards, generateAdditionalData}) {
+function runTest({proportion, expectedPlan, generateAdditionalData}) {
     const expectedDocs = setUpTestColl(generateAdditionalData);
 
     let expectedCount = Math.floor(proportion * Object.keys(expectedDocs).length);
     jsTestLog("Running test with proportion: " + proportion + ", expected count: " + expectedCount +
-              ", expected plan: " + tojson(planForShards));
+              ", expected plan: " + tojson(expectedPlan));
 
     let pipeline = [{$sample: {size: expectedCount}}, projection];
-    testPipeline({pipeline, expectedDocs, expectedCount, shardsTargetedCount: 2, planForShards});
+    testPipeline({pipeline, expectedDocs, expectedCount, shardsTargetedCount: 2, expectedPlan});
 
     expectedCount = 1;
     pipeline = [{$sample: {size: expectedCount}}, projection];
@@ -379,14 +390,14 @@ runTest({
     generateAdditionalData: () => {
         return insertAdditionalData(false);
     },
-    planForShards: {[primary.shardName]: arhash, [otherShard.shardName]: arhash},
+    expectedPlan: arhash
 });
 runTest({
     proportion: 0.005,
     generateAdditionalData: () => {
         return insertAdditionalData(true);
     },
-    planForShards: {[primary.shardName]: topK, [otherShard.shardName]: topK},
+    expectedPlan: topK
 });
 
 // Top-K plan without the trail stage.
@@ -395,7 +406,7 @@ runTest({
     generateAdditionalData: () => {
         return insertAdditionalData(false);
     },
-    planForShards: {[primary.shardName]: randomCursor, [otherShard.shardName]: randomCursor},
+    expectedPlan: randomCursor,
 });
 
 // Verify that for a sample size > 1000, we pick the Top-K sort plan without any trial.
@@ -407,10 +418,7 @@ testPipeline({
     expectedCount: 1001,
     expectedDocs: expectedDocs,
     shardsTargetedCount: 2,
-    planForShards: {
-        [primary.shardName]: randomCursor,
-        [otherShard.shardName]: randomCursor,
-    }
+    expectedPlan: randomCursor
 });
 
 st.stop();

jstests/sharding/update_sharded.js

@@ -40,6 +40,14 @@ for (let i = 0; i < 2; i++) {
     assert.commandWorked(coll.update({_id: 2, key: 2}, {key: 2, foo: 'bar'}, {upsert: true}));
     assert.commandWorked(coll.update({_id: 3, key: 3}, {$set: {foo: 'bar'}}, {upsert: true}));
 
+    // Mixing operator & non-operator fields in updates is not allowed.
+    assert.commandFailedWithCode(
+        coll.update({_id: 4, key: 4}, {key: 4, $baz: {foo: 'bar'}}, {upsert: true}),
+        ErrorCodes.UnsupportedFormat);
+    assert.commandFailedWithCode(
+        coll.update({_id: 5, key: 5}, {$baz: {foo: 'bar'}, key: 5}, {upsert: true}),
+        ErrorCodes.UnsupportedFormat);
+
     assert.eq(coll.count(), 3, "count A");
     assert.eq(coll.findOne({_id: 3}).key, 3, "findOne 3 key A");
     assert.eq(coll.findOne({_id: 3}).foo, 'bar', "findOne 3 foo A");

jstests/sharding/update_with_dollar_fields.js

@@ -0,0 +1,48 @@
+/**
+ * Tests that replacement style update with $v field in the document is correctly applied.
+ * @tags: [
+ *  requires_fcv_50,
+ * ]
+ */
+
+(function() {
+"use strict";
+
+const st = new ShardingTest({nodes: 2});
+
+const dbName = 'testDb';
+const collName = 'testColl';
+const coll = st.s.getDB(dbName).getCollection(collName);
+
+st.adminCommand({enablesharding: dbName});
+const oplog = st.getPrimaryShard(dbName).getDB('local').getCollection('oplog.rs');
+
+function assertLastUpdateOplogEntryIsReplacement() {
+    const lastUpdate = oplog.find({op: 'u'}).sort({$natural: -1}).limit(1).next();
+    assert(lastUpdate.o._id);
+}
+
+[true].forEach($v => {
+    const _id = assert.commandWorked(coll.insertOne({$v})).insertedId;
+    assert.commandWorked(coll.update({_id}, [{$set: {p: 1, q: 1}}]));
+    assertLastUpdateOplogEntryIsReplacement();
+});
+
+[true, "hello", 0, 1, 2, 3].forEach($v => {
+    const _id = assert.commandWorked(coll.insertOne({})).insertedId;
+    assert.commandWorked(coll.update(
+        {_id},
+        [{$replaceWith: {"$setField": {field: {$literal: "$v"}, input: "$$ROOT", value: $v}}}]));
+    assertLastUpdateOplogEntryIsReplacement();
+});
+
+(function() {
+const _id = assert.commandWorked(coll.insertOne({})).insertedId;
+assert.commandWorked(coll.update(
+    {_id},
+    [{$replaceWith: {"$setField": {field: {$literal: "$set"}, input: "$$ROOT", value: {a: 1}}}}]));
+assertLastUpdateOplogEntryIsReplacement();
+})();
+
+st.stop();
+}());

jstests/slow1/geo_array1.js

@@ -1,6 +1,7 @@
 // Make sure many locations in one doc works, in the form of an array
 // @tags: [
 //   does_not_support_stepdowns,
+//   requires_replication,
 //   resource_intensive,
 // ]
 

site_scons/site_tools/icecream.py

@@ -58,7 +58,7 @@ def icecc_create_env(env, target, source, for_signature):
     # store it in a known location. Add any files requested from the user environment.
     create_env = "ICECC_VERSION_TMP=$$(${SOURCES[0]} --$ICECC_COMPILER_TYPE ${SOURCES[1]} ${SOURCES[2]}"
 
-    # TODO: It would be a little more elegant if things in
+    # TODO: SERVER-57393 It would be a little more elegant if things in
     # ICECC_CREATE_ENV_ADDFILES were handled as sources, because we
     # would get automatic dependency tracking. However, there are some
     # wrinkles around the mapped case so we have opted to leave it as
@@ -328,7 +328,7 @@ def generate(env):
     # of such a node easily. Creating a Substfile means that SCons
     # will take care of generating a file that Ninja can use.
     run_icecc = setupEnv.Textfile(
-        target="$ICECREAM_TARGET_DIR/run-icecc.sh",
+        target="$ICECREAM_TARGET_DIR/$ICECREAM_RUN_SCRIPT_SUBPATH/run-icecc.sh",
         source=[
             '#!/bin/sh',
             'ICECC_VERSION=@icecc_version_arch@@icecc_version@ exec @icecc@ "$@"',

site_scons/site_tools/ninja.py

@@ -660,26 +660,56 @@ class NinjaState:
                 kwargs['pool'] = 'local_pool'
             ninja.rule(rule, **kwargs)
 
-        generated_source_files = sorted({
-            output
-            # First find builds which have header files in their outputs.
-            for build in self.builds.values()
-            if self.has_generated_sources(build["outputs"])
-            for output in build["outputs"]
-            # Collect only the header files from the builds with them
-            # in their output. We do this because is_generated_source
-            # returns True if it finds a header in any of the outputs,
-            # here we need to filter so we only have the headers and
-            # not the other outputs.
-            if self.is_generated_source(output)
-        })
+        # If the user supplied an alias to determine generated sources, use that, otherwise
+        # determine what the generated sources are dynamically.
+        generated_sources_alias = self.env.get('NINJA_GENERATED_SOURCE_ALIAS_NAME')
+        generated_sources_build = None
 
-        if generated_source_files:
-            ninja.build(
-                outputs="_generated_sources",
-                rule="phony",
-                implicit=generated_source_files
+        if generated_sources_alias:
+            generated_sources_build = self.builds.get(generated_sources_alias)
+            if generated_sources_build is None or generated_sources_build["rule"] != 'phony':
+                raise Exception(
+                    "ERROR: 'NINJA_GENERATED_SOURCE_ALIAS_NAME' set, but no matching Alias object found."
+                )
+
+        if generated_sources_alias and generated_sources_build:
+            generated_source_files = sorted(
+                [] if not generated_sources_build else generated_sources_build['implicit']
             )
+            def check_generated_source_deps(build):
+                return (
+                    build != generated_sources_build
+                    and set(build["outputs"]).isdisjoint(generated_source_files)
+                )
+        else:
+            generated_sources_build = None
+            generated_source_files = sorted({
+                output
+                # First find builds which have header files in their outputs.
+                for build in self.builds.values()
+                if self.has_generated_sources(build["outputs"])
+                for output in build["outputs"]
+                # Collect only the header files from the builds with them
+                # in their output. We do this because is_generated_source
+                # returns True if it finds a header in any of the outputs,
+                # here we need to filter so we only have the headers and
+                # not the other outputs.
+                if self.is_generated_source(output)
+            })
+
+            if generated_source_files:
+                generated_sources_alias = "_ninja_generated_sources"
+                ninja.build(
+                    outputs=generated_sources_alias,
+                    rule="phony",
+                    implicit=generated_source_files
+                )
+                def check_generated_source_deps(build):
+                    return (
+                        not build["rule"] == "INSTALL"
+                        and set(build["outputs"]).isdisjoint(generated_source_files)
+                        and set(build.get("implicit", [])).isdisjoint(generated_source_files)
+                    )
 
         template_builders = []
 
@@ -698,9 +728,7 @@ class NinjaState:
             # cycle.
             if (
                 generated_source_files
-                and not build["rule"] == "INSTALL"
-                and set(build["outputs"]).isdisjoint(generated_source_files)
-                and set(build.get("implicit", [])).isdisjoint(generated_source_files)
+                and check_generated_source_deps(build)
             ):
 
                 # Make all non-generated source targets depend on
@@ -710,7 +738,7 @@ class NinjaState:
                 # sure that all of these sources are generated before
                 # other builds.
                 order_only = build.get("order_only", [])
-                order_only.append("_generated_sources")
+                order_only.append(generated_sources_alias)
                 build["order_only"] = order_only
             if "order_only" in build:
                 build["order_only"].sort()
@@ -1154,7 +1182,10 @@ def get_command(env, node, action):  # pylint: disable=too-many-branches
         # Possibly these could be ignore and the build would still work, however it may not always
         # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided
         # ninja rule.
-        raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'")
+        err_msg = f"Could not resolve path for '{provider_dep}' dependency on node '{node}', you may need to setup your shell environment for ninja builds."
+        if os.name == "nt":
+            err_msg += " On Windows, please ensure that you have run the necessary Visual Studio environment setup scripts (e.g. vcvarsall.bat ...,  or launching a Visual Studio Command Prompt) before invoking SCons."
+        raise Exception(err_msg)
 
     ninja_build = {
         "order_only": get_order_only(node),

src/mongo/bson/util/bsoncolumn.cpp

@@ -43,8 +43,9 @@ namespace {
 // Start capacity for memory blocks allocated by ElementStorage
 constexpr int kStartCapacity = 128;
 
-// Max capacity for memory blocks allocated by ElementStorage
-constexpr int kMaxCapacity = 1024 * 32;
+// Max capacity for memory blocks allocated by ElementStorage. We need to allow blocks to grow to at
+// least BSONObjMaxUserSize so we can construct user objects efficiently.
+constexpr int kMaxCapacity = BSONObjMaxUserSize;
 
 // Memory offset to get to BSONElement value when field name is an empty string.
 constexpr int kElementValueOffset = 2;

src/mongo/db/SConscript

@@ -270,12 +270,16 @@ env.Library(
 env.Library(
     target='collection_index_usage_tracker',
     source=[
-        'collection_index_usage_tracker.cpp'
+        'collection_index_usage_tracker.cpp',
+        'global_index_usage_tracker.cpp',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/commands/server_status_core',
     ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/index_names',
+    ]
 )
 
 env.Library(
@@ -452,15 +456,18 @@ env.Library(
 )
 
 env.Library(
-    target="write_concern_options",
+    target='write_concern_options',
     source=[
-        "write_concern_options.cpp",
+        'write_concern_options.cpp',
+        'write_concern_options.idl',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/bson/util/bson_extract',
-        '$BUILD_DIR/mongo/idl/basic_types',
         'read_write_concern_provenance',
     ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/server_options_core', # For serverGlobalParams
+    ]
 )
 
 env.Library(

src/mongo/db/auth/builtin_roles.cpp

@@ -768,9 +768,21 @@ const std::map<StringData, BuiltinRoleDefinition> kBuiltinRoles({
     {BUILTIN_ROLE_INTERNAL, {true, addInternalRolePrivileges}},
 });
 
+// $external is a virtual database used for X509, LDAP,
+// and other authentication mechanisms and not used for storage.
+// Therefore, granting privileges on this database does not make sense.
+bool isValidDB(StringData dbname) {
+    return NamespaceString::validDBName(dbname, NamespaceString::DollarInDbNameBehavior::Allow) &&
+        (dbname != NamespaceString::kExternalDb);
+}
+
 }  // namespace
 
 stdx::unordered_set<RoleName> auth::getBuiltinRoleNamesForDB(StringData dbname) {
+    if (!isValidDB(dbname)) {
+        return {};
+    }
+
     const bool isAdmin = dbname == ADMIN_DBNAME;
 
     stdx::unordered_set<RoleName> roleNames;
@@ -786,8 +798,7 @@ bool auth::addPrivilegesForBuiltinRole(const RoleName& roleName, PrivilegeVector
     auto role = roleName.getRole();
     auto dbname = roleName.getDB();
 
-    if (!NamespaceString::validDBName(dbname, NamespaceString::DollarInDbNameBehavior::Allow) ||
-        dbname == "$external") {
+    if (!isValidDB(dbname)) {
         return false;
     }
 
@@ -814,8 +825,7 @@ void auth::generateUniversalPrivileges(PrivilegeVector* privileges) {
 
 bool auth::isBuiltinRole(const RoleName& role) {
     auto dbname = role.getDB();
-    if (!NamespaceString::validDBName(dbname, NamespaceString::DollarInDbNameBehavior::Allow) ||
-        dbname == "$external") {
+    if (!isValidDB(dbname)) {
         return false;
     }
 

src/mongo/db/catalog/SConscript

@@ -309,6 +309,7 @@ env.Library(
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/concurrency/lock_manager',
+        '$BUILD_DIR/mongo/db/views/views',
         'collection',
         'collection_catalog',
     ],
@@ -499,6 +500,9 @@ env.Library(
         'rename_collection.cpp',
         'list_indexes.cpp',
     ],
+    LIBDEPS=[
+        'collection_catalog_helper',
+    ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/db_raii',

src/mongo/db/catalog/coll_mod.cpp

@@ -407,7 +407,7 @@ StatusWith<ParsedCollModRequest> parseCollModRequest(OperationContext* opCtx,
             } catch (const DBException& ex) {
                 return ex.toStatus();
             }
-        } else if (fieldName == "expireAfterSeconds") {
+        } else if (fieldName == "expireAfterSeconds" && !isView) {
             cmr.numModifications++;
             if (coll->getRecordStore()->keyFormat() != KeyFormat::String) {
                 return Status(ErrorCodes::InvalidOptions,

src/mongo/db/catalog/collection_catalog.cpp

@@ -346,7 +346,8 @@ CollectionCatalog::iterator::value_type CollectionCatalog::iterator::operator*()
         return CollectionPtr();
     }
 
-    return {_opCtx, _mapIter->second.get(), LookupCollectionForYieldRestore()};
+    return {
+        _opCtx, _mapIter->second.get(), LookupCollectionForYieldRestore(_mapIter->second->ns())};
 }
 
 Collection* CollectionCatalog::iterator::getWritableCollection(OperationContext* opCtx,
@@ -672,12 +673,12 @@ CollectionPtr CollectionCatalog::lookupCollectionByUUID(OperationContext* opCtx,
     }
 
     if (auto coll = UncommittedCollections::getForTxn(opCtx, uuid)) {
-        return {opCtx, coll.get(), LookupCollectionForYieldRestore()};
+        return {opCtx, coll.get(), LookupCollectionForYieldRestore(coll->ns())};
     }
 
     auto coll = _lookupCollectionByUUID(uuid);
     return (coll && coll->isCommitted())
-        ? CollectionPtr(opCtx, coll.get(), LookupCollectionForYieldRestore())
+        ? CollectionPtr(opCtx, coll.get(), LookupCollectionForYieldRestore(coll->ns()))
         : CollectionPtr();
 }
 
@@ -757,7 +758,7 @@ CollectionPtr CollectionCatalog::lookupCollectionByNamespace(OperationContext* o
     // If found=true above but we don't have a Collection pointer it is a drop or rename. But first
     // check UncommittedCollections in case we find a new collection there.
     if (auto coll = UncommittedCollections::getForTxn(opCtx, nss)) {
-        return {opCtx, coll.get(), LookupCollectionForYieldRestore()};
+        return {opCtx, coll.get(), LookupCollectionForYieldRestore(coll->ns())};
     }
 
     // Report the drop or rename as nothing new was created.
@@ -768,7 +769,7 @@ CollectionPtr CollectionCatalog::lookupCollectionByNamespace(OperationContext* o
     auto it = _collections.find(nss);
     auto coll = (it == _collections.end() ? nullptr : it->second);
     return (coll && coll->isCommitted())
-        ? CollectionPtr(opCtx, coll.get(), LookupCollectionForYieldRestore())
+        ? CollectionPtr(opCtx, coll.get(), LookupCollectionForYieldRestore(coll->ns()))
         : nullptr;
 }
 
@@ -1211,9 +1212,19 @@ void CollectionCatalogStasher::reset() {
 
 const Collection* LookupCollectionForYieldRestore::operator()(OperationContext* opCtx,
                                                               const UUID& uuid) const {
-    auto collection = CollectionCatalog::get(opCtx)->lookupCollectionByUUID(opCtx, uuid).get();
-    if (!collection)
+    auto collection = CollectionCatalog::get(opCtx)->lookupCollectionByUUIDForRead(opCtx, uuid);
+
+    // Collection dropped during yielding.
+    if (!collection) {
         return nullptr;
+    }
+
+    // Collection renamed during yielding.
+    // This check ensures that we are locked on the same namespace and that it is safe to return
+    // the C-style pointer to the Collection.
+    if (collection->ns() != _nss) {
+        return nullptr;
+    }
 
     // After yielding and reacquiring locks, the preconditions that were used to select our
     // ReadSource initially need to be checked again. We select a ReadSource based on replication
@@ -1225,7 +1236,7 @@ const Collection* LookupCollectionForYieldRestore::operator()(OperationContext*
         opCtx->recoveryUnit()->setTimestampReadSource(*newReadSource);
     }
 
-    return collection;
+    return collection.get();
 }
 
 BatchedCollectionCatalogWriter::BatchedCollectionCatalogWriter(OperationContext* opCtx)

src/mongo/db/catalog/collection_catalog.h

@@ -517,7 +517,11 @@ private:
  * restore implementation for CollectionPtr when acquired from the catalog.
  */
 struct LookupCollectionForYieldRestore {
+    explicit LookupCollectionForYieldRestore(const NamespaceString& nss) : _nss(nss) {}
     const Collection* operator()(OperationContext* opCtx, const UUID& uuid) const;
+
+private:
+    const NamespaceString _nss;
 };
 
 /**

src/mongo/db/catalog/collection_catalog_helper.cpp

@@ -31,6 +31,7 @@
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/collection_catalog.h"
 #include "mongo/db/concurrency/d_concurrency.h"
+#include "mongo/db/views/view_catalog.h"
 
 namespace mongo {
 
@@ -38,6 +39,26 @@ MONGO_FAIL_POINT_DEFINE(hangBeforeGettingNextCollection);
 
 namespace catalog {
 
+Status checkIfNamespaceExists(OperationContext* opCtx, const NamespaceString& nss) {
+    if (CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, nss)) {
+        return Status(ErrorCodes::NamespaceExists,
+                      str::stream() << "Collection " << nss.ns() << " already exists.");
+    }
+
+    auto view = ViewCatalog::get(opCtx)->lookup(opCtx, nss);
+    if (!view)
+        return Status::OK();
+
+    if (view->timeseries()) {
+        return Status(ErrorCodes::NamespaceExists,
+                      str::stream() << "A timeseries collection already exists. NS: " << nss);
+    }
+
+    return Status(ErrorCodes::NamespaceExists,
+                  str::stream() << "A view already exists. NS: " << nss);
+}
+
+
 void forEachCollectionFromDb(OperationContext* opCtx,
                              const TenantDatabaseName& tenantDbName,
                              LockMode collLockMode,

src/mongo/db/catalog/collection_catalog_helper.h

@@ -41,6 +41,15 @@ class CollectionCatalogEntry;
 
 namespace catalog {
 
+/**
+ * Returns ErrorCodes::NamespaceExists if a collection or any type of views exists on the given
+ * namespace 'nss'. Otherwise returns Status::OK().
+ *
+ * Note: If the caller calls this method without locking the collection, then the returned result
+ * could be stale right after this call.
+ */
+Status checkIfNamespaceExists(OperationContext* opCtx, const NamespaceString& nss);
+
 /**
  * Iterates through all the collections in the given database and runs the callback function on each
  * collection. If a predicate is provided, then the callback will only be executed against the

src/mongo/db/catalog/collection_catalog_test.cpp

@@ -26,6 +26,7 @@
  *    exception statement from all source files in the program, then also delete
  *    it in the license file.
  */
+
 #include "mongo/db/catalog/collection_catalog.h"
 
 #include <algorithm>
@@ -69,14 +70,15 @@ public:
         opCtx = makeOperationContext();
 
         std::shared_ptr<Collection> collection =
-            std::make_shared<CollectionMock>(TenantNamespace(boost::none, nss));
+            std::make_shared<CollectionMock>(colUUID, TenantNamespace(boost::none, nss));
         col = CollectionPtr(collection.get(), CollectionPtr::NoYieldTag{});
         // Register dummy collection in catalog.
         catalog.registerCollection(opCtx.get(), colUUID, std::move(collection));
     }
 
 protected:
-    CollectionCatalog catalog;
+    std::shared_ptr<CollectionCatalog> sharedCatalog = std::make_shared<CollectionCatalog>();
+    CollectionCatalog& catalog = *sharedCatalog;
     ServiceContext::UniqueOperationContext opCtx;
     NamespaceString nss;
     CollectionPtr col;
@@ -442,24 +444,76 @@ TEST_F(CollectionCatalogTest, InsertAfterLookup) {
 }
 
 TEST_F(CollectionCatalogTest, OnDropCollection) {
+    auto yieldableColl = catalog.lookupCollectionByUUID(opCtx.get(), colUUID);
+    ASSERT(yieldableColl);
+    ASSERT_EQUALS(yieldableColl, col);
+
+    // Yielding resets a CollectionPtr's internal state to be restored later, provided
+    // the collection has not been dropped or renamed.
+    ASSERT_EQ(yieldableColl->uuid(), colUUID);  // Correct collection UUID is required for restore.
+    yieldableColl.yield();
+    ASSERT_FALSE(yieldableColl);
+
+    // The global catalog is used to refresh the CollectionPtr's internal state, so we temporarily
+    // replace the global instance initialized in the service context test fixture with our own.
+    CollectionCatalogStasher catalogStasher(opCtx.get(), sharedCatalog);
+
+    // Before dropping collection, confirm that the CollectionPtr can be restored successfully.
+    yieldableColl.restore();
+    ASSERT(yieldableColl);
+    ASSERT_EQUALS(yieldableColl, col);
+
+    // Reset CollectionPtr for post-drop restore test.
+    yieldableColl.yield();
+    ASSERT_FALSE(yieldableColl);
+
     catalog.deregisterCollection(opCtx.get(), colUUID);
     // Ensure the lookup returns a null pointer upon removing the colUUID entry.
     ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), colUUID) == nullptr);
+
+    // After dropping the collection, we should fail to restore the CollectionPtr.
+    yieldableColl.restore();
+    ASSERT_FALSE(yieldableColl);
 }
 
 TEST_F(CollectionCatalogTest, RenameCollection) {
     auto uuid = UUID::gen();
     NamespaceString oldNss(nss.db(), "oldcol");
     std::shared_ptr<Collection> collShared =
-        std::make_shared<CollectionMock>(TenantNamespace(boost::none, oldNss));
+        std::make_shared<CollectionMock>(uuid, TenantNamespace(boost::none, oldNss));
     auto collection = collShared.get();
     catalog.registerCollection(opCtx.get(), uuid, std::move(collShared));
-    ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), uuid), collection);
+    auto yieldableColl = catalog.lookupCollectionByUUID(opCtx.get(), uuid);
+    ASSERT(yieldableColl);
+    ASSERT_EQUALS(yieldableColl, collection);
+
+    // Yielding resets a CollectionPtr's internal state to be restored later, provided
+    // the collection has not been dropped or renamed.
+    ASSERT_EQ(yieldableColl->uuid(), uuid);  // Correct collection UUID is required for restore.
+    yieldableColl.yield();
+    ASSERT_FALSE(yieldableColl);
+
+    // The global catalog is used to refresh the CollectionPtr's internal state, so we temporarily
+    // replace the global instance initialized in the service context test fixture with our own.
+    CollectionCatalogStasher catalogStasher(opCtx.get(), sharedCatalog);
+
+    // Before renaming collection, confirm that the CollectionPtr can be restored successfully.
+    yieldableColl.restore();
+    ASSERT(yieldableColl);
+    ASSERT_EQUALS(yieldableColl, collection);
+
+    // Reset CollectionPtr for post-rename restore test.
+    yieldableColl.yield();
+    ASSERT_FALSE(yieldableColl);
 
     TenantNamespace newNss(boost::none, NamespaceString(nss.db(), "newcol"));
     ASSERT_OK(collection->rename(opCtx.get(), newNss, false));
     ASSERT_EQ(collection->ns(), newNss.getNss());
     ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), uuid), collection);
+
+    // After renaming the collection, we should fail to restore the CollectionPtr.
+    yieldableColl.restore();
+    ASSERT_FALSE(yieldableColl);
 }
 
 TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDropped) {

src/mongo/db/catalog/collection_impl.cpp

@@ -2004,7 +2004,8 @@ std::vector<std::string> CollectionImpl::removeInvalidIndexOptions(OperationCont
             }
 
             indexesWithInvalidOptions.push_back(std::string(index.nameStringData()));
-            index.spec = index_key_validate::removeUnknownFields(oldSpec);
+            index.spec = index_key_validate::removeUnknownFields(
+                NamespaceString(md.tenantNs.getNss()), oldSpec);
         }
     });
 

src/mongo/db/catalog/collection_mock.h

@@ -40,10 +40,16 @@ namespace mongo {
  */
 class CollectionMock : public Collection {
 public:
-    CollectionMock(const TenantNamespace& tenantNs)
+    explicit CollectionMock(const TenantNamespace& tenantNs)
         : CollectionMock(tenantNs, std::unique_ptr<IndexCatalog>()) {}
+    CollectionMock(const UUID& uuid, const TenantNamespace& tenantNs)
+        : CollectionMock(uuid, tenantNs, std::unique_ptr<IndexCatalog>()) {}
     CollectionMock(const TenantNamespace& tenantNs, std::unique_ptr<IndexCatalog> indexCatalog)
-        : _tenantNs(tenantNs), _indexCatalog(std::move(indexCatalog)) {}
+        : CollectionMock(UUID::gen(), tenantNs, std::unique_ptr<IndexCatalog>()) {}
+    CollectionMock(const UUID& uuid,
+                   const TenantNamespace& tenantNs,
+                   std::unique_ptr<IndexCatalog> indexCatalog)
+        : _uuid(uuid), _tenantNs(tenantNs), _indexCatalog(std::move(indexCatalog)) {}
     CollectionMock(const TenantNamespace& tenantNs, RecordId catalogId)
         : _tenantNs(tenantNs), _catalogId(catalogId) {}
     ~CollectionMock() = default;

src/mongo/db/catalog/commit_quorum_options.cpp

@@ -68,12 +68,12 @@ Status CommitQuorumOptions::parse(const BSONElement& commitQuorumElement) {
     if (commitQuorumElement.isNumber()) {
         auto cNumNodes = commitQuorumElement.safeNumberLong();
         if (cNumNodes < 0 ||
-            cNumNodes > static_cast<decltype(cNumNodes)>(repl::ReplSetConfig::kMaxVotingMembers)) {
+            cNumNodes > static_cast<decltype(cNumNodes)>(repl::ReplSetConfig::kMaxMembers)) {
             return Status(
                 ErrorCodes::FailedToParse,
                 str::stream()
                     << "commitQuorum has to be a non-negative number and not greater than "
-                    << repl::ReplSetConfig::kMaxVotingMembers);
+                    << repl::ReplSetConfig::kMaxMembers);
         }
         numNodes = static_cast<decltype(numNodes)>(cNumNodes);
     } else if (commitQuorumElement.type() == String) {