SERVER-72021 Fix Enterprise SLES 12 Patch Failure

(cherry picked from commit 5fee6fff13)

README.md

@@ -90,3 +90,4 @@ Welcome to MongoDB!
   October 16, 2018, including patch fixes for prior versions, are published
   under the [Server Side Public License (SSPL) v1](LICENSE-Community.txt).
   See individual files for details.
+

SConstruct

@@ -3,6 +3,7 @@
 import atexit
 import copy
 import errno
+import functools
 import json
 import os
 import re
@@ -197,7 +198,7 @@ add_option(
 add_option(
     'install-action',
     choices=([*install_actions.available_actions] + ['default']),
-    default='default',
+    default='hardlink',
     help=
     'select mechanism to use to install files (advanced option to reduce disk IO and utilization)',
     nargs=1,
@@ -520,7 +521,7 @@ add_option(
     action="append",
     choices=["configure", "source"],
     const="source",
-    default=[],
+    default=build_profile.disable_warnings_as_errors,
     help=
     "Don't add a warnings-as-errors flag to compiler command lines in selected contexts; defaults to 'source' if no argument is provided",
     nargs="?",
@@ -1016,17 +1017,10 @@ env_vars.Add(
     converter=variable_shlex_converter,
 )
 
-default_destdir = '$BUILD_ROOT/install'
-if get_option('ninja') != 'disabled':
-    # Workaround for SERVER-53952 where issues wih different
-    # ninja files building to the same install dir. Different
-    # ninja files need to build to different install dirs.
-    default_destdir = '$BUILD_DIR/install'
-
 env_vars.Add(
     'DESTDIR',
     help='Where builds will install files',
-    default=default_destdir,
+    default='$BUILD_ROOT/install',
 )
 
 env_vars.Add(
@@ -1220,7 +1214,7 @@ env_vars.Add(
 env_vars.Add(
     'NINJA_BUILDDIR',
     help="Location for shared Ninja state",
-    default="$BUILD_DIR/ninja",
+    default="$BUILD_ROOT/ninja",
 )
 
 env_vars.Add(
@@ -1631,8 +1625,6 @@ unknown_vars = env_vars.UnknownVariables()
 if unknown_vars:
     env.FatalError("Unknown variables specified: {0}", ", ".join(list(unknown_vars.keys())))
 
-if get_option('install-action') != 'default' and get_option('ninja') != "disabled":
-    env.FatalError("Cannot use non-default install actions when generating Ninja.")
 install_actions.setup(env, get_option('install-action'))
 
 
@@ -2486,6 +2478,15 @@ for suboption in get_option('experimental-runtime-hardening'):
     elif suboption.startswith('+'):
         selected_experimental_runtime_hardenings.add(suboption[1:])
 
+# Disable floating-point contractions such as forming of fused multiply-add operations.
+if env.ToolchainIs('clang', 'gcc'):
+    env.Append(CCFLAGS=["-ffp-contract=off"])
+else:
+    # msvc defaults to /fp:precise. Visual Studio 2022 does not emit floating-point contractions
+    # with /fp:precise, but previous versions can. Disable contractions altogether by using
+    # /fp:strict.
+    env.Append(CCFLAGS=["/fp:strict"])
+
 if env.TargetOSIs('linux'):
     env.Append(LIBS=["m"])
     if not env.TargetOSIs('android'):
@@ -3251,7 +3252,7 @@ def doConfigure(myenv):
 
         conf.Finish()
 
-    def AddFlagIfSupported(env, tool, extension, flag, link, **mutation):
+    def CheckFlag(env, flag, tool, extension, link, **mutation):
         def CheckFlagTest(context, tool, extension, flag):
             if link:
                 if tool == 'C':
@@ -3327,34 +3328,33 @@ def doConfigure(myenv):
         )
         available = conf.CheckFlag()
         conf.Finish()
+        return available
+
+    def AddFlagIfSupported(env, flag, tool, extension, link, **mutation):
+
+        available = CheckFlag(env, flag, tool, extension, link, **mutation)
+
         if available:
             env.Append(**mutation)
         return available
 
-    def AddToCFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, False, CFLAGS=[flag])
+    conf_check_vars = {
+        'CFLAGS': {'tool': 'C', 'extension': '.c', 'link': False},
+        'CCFLAGS': {'tool': 'C', 'extension': '.c', 'link': False},
+        'CXXFLAGS': {'tool': 'C++', 'extension': '.cpp', 'link': False},
+        'LINKFLAGS': {'tool': 'C', 'extension': '.c', 'link': True},
+        'SHLINKFLAGS': {'tool': 'C', 'extension': '.c', 'link': True},
+    }
 
-    env.AddMethod(AddToCFLAGSIfSupported)
+    def var_func(env, flag, var, func):
+        kwargs = dict({var: [flag]}, **conf_check_vars[var])
+        return func(env, flag, **kwargs)
 
-    def AddToCCFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, False, CCFLAGS=[flag])
-
-    env.AddMethod(AddToCCFLAGSIfSupported)
-
-    def AddToCXXFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C++', '.cpp', flag, False, CXXFLAGS=[flag])
-
-    env.AddMethod(AddToCXXFLAGSIfSupported)
-
-    def AddToLINKFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, True, LINKFLAGS=[flag])
-
-    env.AddMethod(AddToLINKFLAGSIfSupported)
-
-    def AddToSHLINKFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, True, SHLINKFLAGS=[flag])
-
-    env.AddMethod(AddToSHLINKFLAGSIfSupported)
+    for var in conf_check_vars:
+        myenv.AddMethod(
+            functools.partial(var_func, var=var, func=AddFlagIfSupported), f"AddTo{var}IfSupported")
+        myenv.AddMethod(
+            functools.partial(var_func, var=var, func=CheckFlag), f"Check{var}Supported")
 
     if myenv.ToolchainIs('gcc', 'clang'):
         # This tells clang/gcc to use the gold linker if it is available - we prefer the gold linker
@@ -3378,20 +3378,20 @@ def doConfigure(myenv):
             #
             # We should revisit all of these issues the next time we upgrade our clang minimum.
             if get_option('separate-debug') == 'off' and get_option('link-model') != 'dynamic':
-                if not AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=lld'):
-                    AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=gold')
+                if not myenv.AddToLINKFLAGSIfSupported('-fuse-ld=lld'):
+                    myenv.AddToLINKFLAGSIfSupported('-fuse-ld=gold')
             else:
-                AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=gold')
+                myenv.AddToLINKFLAGSIfSupported('-fuse-ld=gold')
         elif link_model.startswith("dynamic") and linker_ld == 'bfd':
             # BFD is not supported due to issues with it causing warnings from some of
             # the third party libraries that mongodb is linked with:
             # https://jira.mongodb.org/browse/SERVER-49465
             myenv.FatalError(f"Linker {linker_ld} is not supported with dynamic link model builds.")
         else:
-            if not AddToLINKFLAGSIfSupported(myenv, f'-fuse-ld={linker_ld}'):
+            if not myenv.AddToLINKFLAGSIfSupported(f'-fuse-ld={linker_ld}'):
                 myenv.FatalError(f"Linker {linker_ld} could not be configured.")
 
-        if has_option('gcov') and AddToCCFLAGSIfSupported(myenv, '-fprofile-update=single'):
+        if has_option('gcov') and myenv.AddToCCFLAGSIfSupported('-fprofile-update=single'):
             myenv.AppendUnique(LINKFLAGS=['-fprofile-update=single'])
 
     detectCompiler = Configure(
@@ -3424,11 +3424,11 @@ def doConfigure(myenv):
 
     if myenv.ToolchainIs('clang', 'gcc'):
         # This warning was added in g++-4.8.
-        AddToCCFLAGSIfSupported(myenv, '-Wno-unused-local-typedefs')
+        myenv.AddToCCFLAGSIfSupported('-Wno-unused-local-typedefs')
 
         # Clang likes to warn about unused functions, which seems a tad aggressive and breaks
         # -Werror, which we want to be able to use.
-        AddToCCFLAGSIfSupported(myenv, '-Wno-unused-function')
+        myenv.AddToCCFLAGSIfSupported('-Wno-unused-function')
 
         # TODO: Note that the following two flags are added to CCFLAGS even though they are
         # really C++ specific. We need to do this because SCons passes CXXFLAGS *before*
@@ -3438,79 +3438,79 @@ def doConfigure(myenv):
         #
         # Clang likes to warn about unused private fields, but some of our third_party
         # libraries have such things.
-        AddToCCFLAGSIfSupported(myenv, '-Wno-unused-private-field')
+        myenv.AddToCCFLAGSIfSupported('-Wno-unused-private-field')
 
         # Prevents warning about using deprecated features (such as auto_ptr in c++11)
         # Using -Wno-error=deprecated-declarations does not seem to work on some compilers,
         # including at least g++-4.6.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-deprecated-declarations")
+        myenv.AddToCCFLAGSIfSupported("-Wno-deprecated-declarations")
 
         # As of clang-3.4, this warning appears in v8, and gets escalated to an error.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-constant-out-of-range-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-constant-out-of-range-compare")
 
         # As of clang in Android NDK 17, these warnings appears in boost and/or ICU, and get escalated to errors
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-constant-compare")
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-unsigned-zero-compare")
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-unsigned-enum-zero-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-constant-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-unsigned-zero-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-unsigned-enum-zero-compare")
 
         # New in clang-3.4, trips up things mostly in third_party, but in a few places in the
         # primary mongo sources as well.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-unused-const-variable")
+        myenv.AddToCCFLAGSIfSupported("-Wno-unused-const-variable")
 
         # Prevents warning about unused but set variables found in boost version 1.49
         # in boost/date_time/format_date_parser.hpp which does not work for compilers
         # GCC >= 4.6. Error explained in https://svn.boost.org/trac/boost/ticket/6136 .
-        AddToCCFLAGSIfSupported(myenv, "-Wno-unused-but-set-variable")
+        myenv.AddToCCFLAGSIfSupported("-Wno-unused-but-set-variable")
 
         # This has been suppressed in gcc 4.8, due to false positives, but not in clang.  So
         # we explicitly disable it here.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-missing-braces")
+        myenv.AddToCCFLAGSIfSupported("-Wno-missing-braces")
 
         # Suppress warnings about not consistently using override everywhere in a class. It seems
         # very pedantic, and we have a fair number of instances.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-inconsistent-missing-override")
+        myenv.AddToCCFLAGSIfSupported("-Wno-inconsistent-missing-override")
 
         # Don't issue warnings about potentially evaluated expressions
-        AddToCCFLAGSIfSupported(myenv, "-Wno-potentially-evaluated-expression")
+        myenv.AddToCCFLAGSIfSupported("-Wno-potentially-evaluated-expression")
 
         # Warn about moves of prvalues, which can inhibit copy elision.
-        AddToCXXFLAGSIfSupported(myenv, "-Wpessimizing-move")
+        myenv.AddToCXXFLAGSIfSupported("-Wpessimizing-move")
 
         # Disable warning about variables that may not be initialized
         # Failures are triggered in the case of boost::optional in GCC 4.8.x
         # TODO: re-evaluate when we move to GCC 5.3
         # see: http://stackoverflow.com/questions/21755206/how-to-get-around-gcc-void-b-4-may-be-used-uninitialized-in-this-funct
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-maybe-uninitialized")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-maybe-uninitialized")
 
         # Disable warning about templates that can't be implicitly instantiated. It is an attempt to
         # make a link error into an easier-to-debug compiler failure, but it triggers false
         # positives if explicit instantiation is used in a TU that can see the full definition. This
         # is a problem at least for the S2 headers.
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-undefined-var-template")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-undefined-var-template")
 
         # This warning was added in clang-4.0, but it warns about code that is required on some
         # platforms. Since the warning just states that 'explicit instantiation of [a template] that
         # occurs after an explicit specialization has no effect', it is harmless on platforms where
         # it isn't required
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-instantiation-after-specialization")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-instantiation-after-specialization")
 
         # This warning was added in clang-5 and flags many of our lambdas. Since it isn't actively
         # harmful to capture unused variables we are suppressing for now with a plan to fix later.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-unused-lambda-capture")
+        myenv.AddToCCFLAGSIfSupported("-Wno-unused-lambda-capture")
 
         # Enable sized deallocation support.
-        AddToCXXFLAGSIfSupported(myenv, '-fsized-deallocation')
+        myenv.AddToCXXFLAGSIfSupported('-fsized-deallocation')
 
         # This warning was added in Apple clang version 11 and flags many explicitly defaulted move
         # constructors and assignment operators for being implicitly deleted, which is not useful.
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-defaulted-function-deleted")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-defaulted-function-deleted")
 
         # SERVER-44856: Our windows builds complain about unused
         # exception parameters, but GCC and clang don't seem to do
         # that for us automatically. In the interest of making it more
         # likely to catch these errors early, add the (currently clang
         # only) flag that turns it on.
-        AddToCXXFLAGSIfSupported(myenv, "-Wunused-exception-parameter")
+        myenv.AddToCXXFLAGSIfSupported("-Wunused-exception-parameter")
 
         # TODO(SERVER-60151): Avoid the dilemma identified in
         # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100493. Unfortunately,
@@ -3522,7 +3522,7 @@ def doConfigure(myenv):
         # TODO(SERVER-60175): In fact we will want to explicitly opt
         # in to -Wdeprecated, since clang doesn't include it in -Wall.
         if get_option('cxx-std') == "20":
-            AddToCXXFLAGSIfSupported(myenv, '-Wno-deprecated')
+            myenv.AddToCXXFLAGSIfSupported('-Wno-deprecated')
 
         # Check if we can set "-Wnon-virtual-dtor" when "-Werror" is set. The only time we can't set it is on
         # clang 3.4, where a class with virtual function(s) and a non-virtual destructor throws a warning when
@@ -3568,18 +3568,18 @@ def doConfigure(myenv):
         # by -Wall), in order to enforce that -mXXX-version-min=YYY
         # will enforce that you don't use APIs from ZZZ.
         if env.TargetOSIs('darwin'):
-            AddToCCFLAGSIfSupported(env, '-Wunguarded-availability')
+            env.AddToCCFLAGSIfSupported('-Wunguarded-availability')
 
     if get_option('runtime-hardening') == "on":
         # Enable 'strong' stack protection preferentially, but fall back to 'all' if it is not
         # available. Note that we need to add these to the LINKFLAGS as well, since otherwise we
         # might not link libssp when we need to (see SERVER-12456).
         if myenv.ToolchainIs('gcc', 'clang'):
-            if AddToCCFLAGSIfSupported(myenv, '-fstack-protector-strong'):
+            if myenv.AddToCCFLAGSIfSupported('-fstack-protector-strong'):
                 myenv.Append(LINKFLAGS=[
                     '-fstack-protector-strong',
                 ], )
-            elif AddToCCFLAGSIfSupported(myenv, '-fstack-protector-all'):
+            elif myenv.AddToCCFLAGSIfSupported('-fstack-protector-all'):
                 myenv.Append(LINKFLAGS=[
                     '-fstack-protector-all',
                 ], )
@@ -3590,10 +3590,10 @@ def doConfigure(myenv):
                 ], )
 
             if 'stackclash' in selected_experimental_runtime_hardenings:
-                AddToCCFLAGSIfSupported(myenv, "-fstack-clash-protection")
+                myenv.AddToCCFLAGSIfSupported("-fstack-clash-protection")
 
             if 'controlflow' in selected_experimental_runtime_hardenings:
-                AddToCCFLAGSIfSupported(myenv, "-fcf-protection=full")
+                myenv.AddToCCFLAGSIfSupported("-fcf-protection=full")
 
         if myenv.ToolchainIs('clang'):
             # TODO: There are several interesting things to try here, but they each have
@@ -3636,7 +3636,7 @@ def doConfigure(myenv):
     if has_option('libc++'):
         if not myenv.ToolchainIs('clang'):
             myenv.FatalError('libc++ is currently only supported for clang')
-        if AddToCXXFLAGSIfSupported(myenv, '-stdlib=libc++'):
+        if myenv.AddToCXXFLAGSIfSupported('-stdlib=libc++'):
             myenv.Append(LINKFLAGS=['-stdlib=libc++'])
         else:
             myenv.ConfError('libc++ requested, but compiler does not support -stdlib=libc++')
@@ -3673,13 +3673,13 @@ def doConfigure(myenv):
             myenv.AppendUnique(CCFLAGS=['/std:c++20'])
     else:
         if get_option('cxx-std') == "17":
-            if not AddToCXXFLAGSIfSupported(myenv, '-std=c++17'):
+            if not myenv.AddToCXXFLAGSIfSupported('-std=c++17'):
                 myenv.ConfError('Compiler does not honor -std=c++17')
         elif get_option('cxx-std') == "20":
-            if not AddToCXXFLAGSIfSupported(myenv, '-std=c++20'):
+            if not myenv.AddToCXXFLAGSIfSupported('-std=c++20'):
                 myenv.ConfError('Compiler does not honor -std=c++20')
 
-        if not AddToCFLAGSIfSupported(myenv, '-std=c11'):
+        if not myenv.AddToCFLAGSIfSupported('-std=c11'):
             myenv.ConfError("C++17 mode selected for C++ files, but can't enable C11 for C files")
 
     if using_system_version_of_cxx_libraries():
@@ -4002,7 +4002,7 @@ def doConfigure(myenv):
 
         sanitizer_option = '-fsanitize=' + ','.join(sanitizer_list)
 
-        if AddToCCFLAGSIfSupported(myenv, sanitizer_option):
+        if myenv.AddToCCFLAGSIfSupported(sanitizer_option):
             myenv.Append(LINKFLAGS=[sanitizer_option])
             myenv.Append(CCFLAGS=['-fno-omit-frame-pointer'])
         else:
@@ -4013,7 +4013,7 @@ def doConfigure(myenv):
         if has_option('sanitize-coverage') and using_fsan:
             sanitize_coverage_list = get_option('sanitize-coverage')
             sanitize_coverage_option = '-fsanitize-coverage=' + sanitize_coverage_list
-            if AddToCCFLAGSIfSupported(myenv, sanitize_coverage_option):
+            if myenv.AddToCCFLAGSIfSupported(sanitize_coverage_option):
                 myenv.Append(LINKFLAGS=[sanitize_coverage_option])
             else:
                 myenv.ConfError('Failed to enable -fsanitize-coverage with flag: {0}',
@@ -4036,7 +4036,7 @@ def doConfigure(myenv):
         supportedDenyfiles = []
         denyfilesTestEnv = myenv.Clone()
         for denyfile in denyfiles:
-            if AddToCCFLAGSIfSupported(denyfilesTestEnv, f"-fsanitize-blacklist={denyfile}"):
+            if denyfilesTestEnv.AddToCCFLAGSIfSupported(f"-fsanitize-blacklist={denyfile}"):
                 supportedDenyfiles.append(denyfile)
         denyfilesTestEnv = None
         supportedDenyfiles = sorted(supportedDenyfiles)
@@ -4219,8 +4219,8 @@ def doConfigure(myenv):
             # have renamed the flag.
             # However, this flag cannot be included when using the fuzzer sanitizer
             # if we want to suppress errors to uncover new ones.
-            if not using_fsan and not AddToCCFLAGSIfSupported(myenv, "-fno-sanitize-recover"):
-                AddToCCFLAGSIfSupported(myenv, "-fno-sanitize-recover=undefined")
+            if not using_fsan and not myenv.AddToCCFLAGSIfSupported("-fno-sanitize-recover"):
+                myenv.AddToCCFLAGSIfSupported("-fno-sanitize-recover=undefined")
             myenv.AppendUnique(CPPDEFINES=['UNDEFINED_BEHAVIOR_SANITIZER'])
 
             # If anything is changed, added, or removed in ubsan_options, be
@@ -4242,7 +4242,7 @@ def doConfigure(myenv):
             # same as the correct link graph for a regular dynamic
             # build.
             if link_model == "dynamic":
-                if AddToCCFLAGSIfSupported(myenv, "-fno-sanitize=vptr"):
+                if myenv.AddToCCFLAGSIfSupported("-fno-sanitize=vptr"):
                     myenv.AppendUnique(LINKFLAGS=["-fno-sanitize=vptr"])
 
     if myenv.ToolchainIs('msvc') and optBuild != "off":
@@ -4259,21 +4259,28 @@ def doConfigure(myenv):
         # Usually, --gdb-index is too expensive in big static binaries, but for dynamic
         # builds it works well.
         if link_model.startswith("dynamic"):
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--gdb-index')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--gdb-index')
+
+        if link_model != 'dynamic':
+            # This will create an extra section where debug types can be referred from,
+            # reducing other section sizes. This helps most with big static links as there
+            # will be lots of duplicate debug type info.
+            myenv.AddToCCFLAGSIfSupported('-fdebug-types-section')
+            myenv.AddToLINKFLAGSIfSupported('-fdebug-types-section')
 
         # Our build is already parallel.
-        AddToLINKFLAGSIfSupported(myenv, '-Wl,--no-threads')
+        myenv.AddToLINKFLAGSIfSupported('-Wl,--no-threads')
 
         # Explicitly enable GNU build id's if the linker supports it.
-        AddToLINKFLAGSIfSupported(myenv, '-Wl,--build-id')
+        myenv.AddToLINKFLAGSIfSupported('-Wl,--build-id')
 
         # Explicitly use the new gnu hash section if the linker offers
         # it, except on android since older runtimes seem to not
         # support it. For that platform, use 'both'.
         if env.TargetOSIs('android'):
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--hash-style=both')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--hash-style=both')
         else:
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--hash-style=gnu')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--hash-style=gnu')
 
         # Try to have the linker tell us about ODR violations. Don't
         # use it when using clang with libstdc++, as libstdc++ was
@@ -4288,17 +4295,17 @@ def doConfigure(myenv):
                 env.FatalError(
                     'The --detect-odr-violations flag is expected to only be reliable with --opt=off'
                 )
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--detect-odr-violations')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--detect-odr-violations')
 
         # Disallow an executable stack. Also, issue a warning if any files are found that would
         # cause the stack to become executable if the noexecstack flag was not in play, so that we
         # can find them and fix them. We do this here after we check for ld.gold because the
         # --warn-execstack is currently only offered with gold.
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,-z,noexecstack")
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,--warn-execstack")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,-z,noexecstack")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,--warn-execstack")
 
         # If possible with the current linker, mark relocations as read-only.
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,-z,relro")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,-z,relro")
 
         # As far as we know these flags only apply on posix-y systems,
         # and not on Darwin.
@@ -4331,11 +4338,9 @@ def doConfigure(myenv):
             compress_type = "zlib-gabi"
             compress_flag = "compress-debug-sections"
 
-            AddToCCFLAGSIfSupported(
-                myenv,
+            myenv.AddToCCFLAGSIfSupported(
                 f"-Wa,--{compress_flag}={compress_type}"
-                if "as" in debug_compress else f"-Wa,--no{compress_flag}",
-            )
+                if "as" in debug_compress else f"-Wa,--no{compress_flag}", )
 
             # We shouldn't enable debug compression in the linker
             # (meaning our final binaries contain compressed debug
@@ -4371,22 +4376,16 @@ def doConfigure(myenv):
             conf.Finish()
 
             if have_shf_compressed and 'ld' in debug_compress:
-                AddToLINKFLAGSIfSupported(
-                    myenv,
-                    f"-Wl,--{compress_flag}={compress_type}",
-                )
+                myenv.AddToLINKFLAGSIfSupported(f"-Wl,--{compress_flag}={compress_type}", )
             else:
-                AddToLINKFLAGSIfSupported(
-                    myenv,
-                    f"-Wl,--{compress_flag}=none",
-                )
+                myenv.AddToLINKFLAGSIfSupported(f"-Wl,--{compress_flag}=none", )
 
         if "fnsi" in selected_experimental_optimizations:
-            AddToCCFLAGSIfSupported(myenv, "-fno-semantic-interposition")
+            myenv.AddToCCFLAGSIfSupported("-fno-semantic-interposition")
 
     # Avoid deduping symbols on OS X debug builds, as it takes a long time.
     if optBuild == "off" and myenv.ToolchainIs('clang') and env.TargetOSIs('darwin'):
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,-no_deduplicate")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,-no_deduplicate")
 
     # Apply any link time optimization settings as selected by the 'lto' option.
     if has_option('lto'):
@@ -4403,13 +4402,13 @@ def doConfigure(myenv):
         elif myenv.ToolchainIs('gcc', 'clang'):
             # For GCC and clang, the flag is -flto, and we need to pass it both on the compile
             # and link lines.
-            if not AddToCCFLAGSIfSupported(myenv, '-flto') or \
-                    not AddToLINKFLAGSIfSupported(myenv, '-flto'):
+            if not myenv.AddToCCFLAGSIfSupported('-flto') or \
+                    not myenv.AddToLINKFLAGSIfSupported('-flto'):
                 myenv.ConfError("Link time optimization requested, "
                                 "but selected compiler does not honor -flto")
 
             if myenv.TargetOSIs('darwin'):
-                AddToLINKFLAGSIfSupported(myenv, '-Wl,-object_path_lto,${TARGET}.lto')
+                myenv.AddToLINKFLAGSIfSupported('-Wl,-object_path_lto,${TARGET}.lto')
 
         else:
             myenv.ConfError("Don't know how to enable --lto on current toolchain")
@@ -4525,7 +4524,7 @@ def doConfigure(myenv):
     # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052
     if myenv.ToolchainIs('gcc') and (env['TARGET_ARCH'] in ['i386', 'x86_64']):
         if not 'builtin-memcmp' in selected_experimental_optimizations:
-            AddToCCFLAGSIfSupported(myenv, '-fno-builtin-memcmp')
+            myenv.AddToCCFLAGSIfSupported('-fno-builtin-memcmp')
 
     # pthread_setname_np was added in GLIBC 2.12, and Solaris 11.3
     if posix_system:
@@ -5179,7 +5178,7 @@ def doConfigure(myenv):
     myenv = conf.Finish()
 
     if env['TARGET_ARCH'] == "aarch64":
-        AddToCCFLAGSIfSupported(myenv, "-moutline-atomics")
+        myenv.AddToCCFLAGSIfSupported("-moutline-atomics")
 
     conf = Configure(myenv)
     usdt_enabled = get_option('enable-usdt-probes')
@@ -5337,6 +5336,54 @@ if get_option('ninja') != 'disabled':
 
     env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
 
+    if env.TargetOSIs('windows'):
+        # The /b option here will make sure that windows updates the mtime
+        # when copying the file. This allows to not need to use restat for windows
+        # copy commands.
+        copy_install_cmd = "cmd.exe /c copy /b $in $out 1>NUL"
+    else:
+        copy_install_cmd = "install $in $out"
+
+    if env.GetOption('install-action') == 'hardlink':
+        if env.TargetOSIs('windows'):
+            install_cmd = f"cmd.exe /c mklink /h $out $in 1>nul || {copy_install_cmd}"
+        else:
+            install_cmd = f"ln $in $out || {copy_install_cmd}"
+
+    elif env.GetOption('install-action') == 'symlink':
+
+        # macOS's ln and Windows mklink command do not support relpaths
+        # out of the box so we will  precompute during generation in a
+        # custom handler.
+        def symlink_install_action_function(_env, node):
+            # should only be one output and input for this case
+            output_file = _env.NinjaGetOutputs(node)[0]
+            input_file = _env.NinjaGetDependencies(node)[0]
+            try:
+                relpath = os.path.relpath(input_file, os.path.dirname(output_file))
+            except ValueError:
+                relpath = os.path.abspath(input_file)
+
+            return {
+                "outputs": [output_file],
+                "rule": "INSTALL",
+                "inputs": [input_file],
+                "implicit": _env.NinjaGetDependencies(node),
+                "variables": {"precious": node.precious, "relpath": relpath},
+            }
+
+        env.NinjaRegisterFunctionHandler("installFunc", symlink_install_action_function)
+
+        if env.TargetOSIs('windows'):
+            install_cmd = "cmd.exe /c mklink $out $relpath 1>nul"
+        else:
+            install_cmd = "ln -s $relpath $out"
+
+    else:
+        install_cmd = copy_install_cmd
+
+    env.NinjaRule("INSTALL", install_cmd, description="Installed $out", pool="install_pool")
+
     if env.TargetOSIs("windows"):
         # This is a workaround on windows for SERVER-48691 where the line length
         # in response files is too long:
@@ -5494,10 +5541,13 @@ if get_option('separate-debug') == "on" or env.TargetOSIs("windows"):
         )
     separate_debug(env)
 
-# TODO: SERVER-68475
-# temp fix for BF-25986, should be removed when better solution is found
 if env['SPLIT_DWARF'] == "auto":
-    env['SPLIT_DWARF'] = env.ToolchainIs('gcc') and not link_model == "dynamic"
+    # For static builds, splitting out the dwarf info reduces memory requirments, link time
+    # and binary size significantly. It's affect is less prominent in dynamic builds. The downside
+    # is .dwo files use absolute paths in the debug info, so it's not relocatable.
+    env['SPLIT_DWARF'] = (not link_model == "dynamic" and env.ToolchainIs('gcc', 'clang')
+                          and not env.TargetOSIs('darwin')
+                          and env.CheckCCFLAGSSupported('-gsplit-dwarf'))
 
 if env['SPLIT_DWARF']:
     env.Tool('split_dwarf')
@@ -5659,8 +5709,10 @@ env.AddPackageNameAlias(
     name="mh-debugsymbols",
 )
 
+env['RPATH_ESCAPED_DOLLAR_ORIGIN'] = '\\$$$$ORIGIN'
 
-def rpath_generator(env, source, target, for_signature):
+
+def prefix_libdir_rpath_generator(env, source, target, for_signature):
     # If the PREFIX_LIBDIR has an absolute path, we will use that directly as
     # RPATH because that indicates the final install destination of the libraries.
     prefix_libdir = env.subst('$PREFIX_LIBDIR')
@@ -5672,19 +5724,18 @@ def rpath_generator(env, source, target, for_signature):
     lib_rel = os.path.relpath(prefix_libdir, env.subst('$PREFIX_BINDIR'))
 
     if env['PLATFORM'] == 'posix':\
-        return [env.Literal(f"\\$$ORIGIN/{lib_rel}")]
+        return f"$RPATH_ESCAPED_DOLLAR_ORIGIN/{lib_rel}"
 
     if env['PLATFORM'] == 'darwin':
-        return [
-            f"@loader_path/{lib_rel}",
-        ]
+        return f"@loader_path/{lib_rel}"
 
 
-env['RPATH_GENERATOR'] = rpath_generator
+if get_option('link-model').startswith('dynamic'):
+    env['PREFIX_LIBDIR_RPATH_GENERATOR'] = prefix_libdir_rpath_generator
 
 if env['PLATFORM'] == 'posix':
     env.AppendUnique(
-        RPATH='$RPATH_GENERATOR',
+        RPATH=['$PREFIX_LIBDIR_RPATH_GENERATOR'],
         LINKFLAGS=[
             # Most systems *require* -z,origin to make origin work, but android
             # blows up at runtime if it finds DF_ORIGIN_1 in DT_FLAGS_1.
@@ -5704,12 +5755,12 @@ elif env['PLATFORM'] == 'darwin':
     # so we setup RPATH and LINKFLAGS ourselves.
     env['RPATHPREFIX'] = '-Wl,-rpath,'
     env['RPATHSUFFIX'] = ''
-    env['RPATH'] = '$RPATH_GENERATOR'
     env.AppendUnique(
         LINKFLAGS="${_concat(RPATHPREFIX, RPATH, RPATHSUFFIX, __env__)}",
         SHLINKFLAGS=[
             "-Wl,-install_name,@rpath/${TARGET.file}",
         ],
+        RPATH=['$PREFIX_LIBDIR_RPATH_GENERATOR'],
     )
 
 env.Default(env.Alias("install-default"))

buildscripts/errorcodes.py

@@ -45,6 +45,7 @@ _CODE_PATTERNS = [
         r"(?:StatusOK)?"
         r"(?:WithContext)?"
         r"\s*\(",
+        r"MONGO_UNREACHABLE_TASSERT\(",
         # DBException and AssertionException constructors
         r"(?:DB|Assertion)Exception\s*[({]",
         # Calls to all LOGV2* variants

buildscripts/evergreen_task_timeout.py

@@ -7,7 +7,7 @@ import math
 import os
 import shlex
 import sys
-from datetime import datetime, timedelta
+from datetime import timedelta
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -19,7 +19,7 @@ from evergreen import EvergreenApi, RetryingEvergreenApi
 
 from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, parse_evergreen_file)
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService, TimeoutSettings)
+from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService)
 from buildscripts.util.cmdutils import enable_logging
 from buildscripts.util.taskname import determine_task_base_name
 
@@ -369,9 +369,6 @@ def main():
 
     options = parser.parse_args()
 
-    end_date = datetime.now()
-    start_date = end_date - HISTORY_LOOKBACK
-
     timeout_override = timedelta(seconds=options.timeout) if options.timeout else None
     exec_timeout_override = timedelta(
         seconds=options.exec_timeout) if options.exec_timeout else None
@@ -386,7 +383,6 @@ def main():
         binder.bind(
             EvergreenApi,
             RetryingEvergreenApi.get_api(config_file=os.path.expanduser(options.evg_api_config)))
-        binder.bind(TimeoutSettings, TimeoutSettings(start_date=start_date, end_date=end_date))
         binder.bind(TimeoutOverrides, timeout_overrides)
         binder.bind(EvergreenProjectConfig,
                     parse_evergreen_file(os.path.expanduser(options.evg_project_config)))

buildscripts/generate_version_expansions.py

@@ -61,8 +61,8 @@ def generate_version_expansions():
             raise ValueError("Unable to parse version from stdin and no version.json provided")
 
     if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
+        expansions["suffix"] = "v6.1-latest"
+        expansions["src_suffix"] = "v6.1-latest"
         expansions["is_release"] = "false"
     else:
         expansions["suffix"] = version_line

buildscripts/idl/gen_all_feature_flag_list.py

@@ -30,7 +30,6 @@ Generate a file containing a list of disabled feature flags.
 Used by resmoke.py to run only feature flag tests.
 """
 
-import argparse
 import os
 import sys
 
@@ -43,6 +42,7 @@ sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../
 
 # pylint: disable=wrong-import-position
 import buildscripts.idl.lib as lib
+from buildscripts.idl.idl import parser
 
 
 def is_third_party_idl(idl_path: str) -> bool:
@@ -56,13 +56,14 @@ def is_third_party_idl(idl_path: str) -> bool:
     return False
 
 
-def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
+def gen_all_feature_flags(idl_dir: str = os.getcwd()):
     """Generate a list of all feature flags."""
     all_flags = []
     for idl_path in sorted(lib.list_idls(idl_dir)):
         if is_third_party_idl(idl_path):
             continue
-        for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags:
+        doc = parser.parse_file(open(idl_path), idl_path)
+        for feature_flag in doc.spec.feature_flags:
             if feature_flag.default.literal != "true":
                 all_flags.append(feature_flag.name)
 
@@ -72,18 +73,17 @@ def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
     return list(set(all_flags) - set(force_disabled_flags))
 
 
+def gen_all_feature_flags_file(filename: str = lib.ALL_FEATURE_FLAG_FILE):
+    """Output generated list of feature flags to specified file."""
+    flags = gen_all_feature_flags()
+    with open(filename, "w") as output_file:
+        output_file.write("\n".join(flags))
+        print("Generated: ", os.path.realpath(output_file.name))
+
+
 def main():
     """Run the main function."""
-    arg_parser = argparse.ArgumentParser(description=__doc__)
-    arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append",
-                            help="Directory to search for IDL import files")
-
-    args = arg_parser.parse_args()
-
-    flags = gen_all_feature_flags(os.getcwd(), args.import_dirs)
-    with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file:
-        for flag in flags:
-            output_file.write("%s\n" % flag)
+    gen_all_feature_flags_file()
 
 
 if __name__ == '__main__':

buildscripts/idl/idl/parser.py

@@ -1026,7 +1026,7 @@ def _propagate_globals(spec):
         idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type)
 
 
-def _parse(stream, error_file_name):
+def parse_file(stream, error_file_name):
     # type: (Any, str) -> syntax.IDLParsedSpec
     """
     Parse a YAML document into an idl.syntax tree.
@@ -1130,7 +1130,7 @@ def parse(stream, input_file_name, resolver):
     """
     # pylint: disable=too-many-locals
 
-    root_doc = _parse(stream, input_file_name)
+    root_doc = parse_file(stream, input_file_name)
 
     if root_doc.errors:
         return root_doc
@@ -1167,7 +1167,7 @@ def parse(stream, input_file_name, resolver):
 
         # Parse imported file
         with resolver.open(resolved_file_name) as file_stream:
-            parsed_doc = _parse(file_stream, resolved_file_name)
+            parsed_doc = parse_file(file_stream, resolved_file_name)
 
         # Check for errors
         if parsed_doc.errors:

buildscripts/resmoke_tests_runtime_validate.py

@@ -3,7 +3,6 @@
 import json
 import sys
 from collections import namedtuple
-from datetime import datetime, timedelta
 from statistics import mean
 from typing import Dict, List
 
@@ -13,7 +12,8 @@ import structlog
 from buildscripts.resmokelib.testing.report import TestInfo, TestReport
 from buildscripts.resmokelib.utils import get_task_name_without_suffix
 from buildscripts.util.cmdutils import enable_logging
-from evergreen import RetryingEvergreenApi, TestStats
+
+from buildscripts.util.teststats import HistoricTaskData, HistoricalTestInformation
 
 LOGGER = structlog.get_logger("buildscripts.resmoke_tests_runtime_validate")
 
@@ -34,17 +34,12 @@ def parse_resmoke_report(report_file: str) -> List[TestInfo]:
     return [test_info for test_info in test_report.test_infos if "jstests" in test_info.test_file]
 
 
-def get_historic_stats(evg_api_config: str, project_id: str, test_files: List[str], task_name: str,
-                       build_variant: str) -> List[TestStats]:
+def get_historic_stats(project_id: str, task_name: str,
+                       build_variant: str) -> List[HistoricalTestInformation]:
     """Get historic test stats."""
-    evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
-    before_date = datetime.today()
-    after_date = before_date - timedelta(days=LOOK_BACK_NUM_DAYS)
     base_task_name = get_task_name_without_suffix(task_name, build_variant).replace(
         BURN_IN_PREFIX, "")
-    return evg_api.test_stats_by_project(project_id=project_id, after_date=after_date,
-                                         before_date=before_date, tests=test_files,
-                                         tasks=[base_task_name], variants=[build_variant])
+    return HistoricTaskData.get_stats_from_s3(project_id, base_task_name, build_variant)
 
 
 def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]:
@@ -63,13 +58,10 @@ def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]:
 @click.command()
 @click.option("--resmoke-report-file", type=str, required=True,
               help="Location of resmoke's report JSON file.")
-@click.option("--evg-api-config", type=str, required=True,
-              help="Location of evergreen api configuration.")
 @click.option("--project-id", type=str, required=True, help="Evergreen project id.")
 @click.option("--build-variant", type=str, required=True, help="Evergreen build variant name.")
 @click.option("--task-name", type=str, required=True, help="Evergreen task name.")
-def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_variant: str,
-         task_name: str) -> None:
+def main(resmoke_report_file: str, project_id: str, build_variant: str, task_name: str) -> None:
     """Compare resmoke tests runtime with historic stats."""
     enable_logging(verbose=False)
 
@@ -79,10 +71,9 @@ def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_v
         for test_info in current_test_infos
     ])
 
-    historic_stats = get_historic_stats(evg_api_config, project_id, list(current_stats_map.keys()),
-                                        task_name, build_variant)
+    historic_stats = get_historic_stats(project_id, task_name, build_variant)
     historic_stats_map = make_stats_map([
-        _TestData(test_stats.test_file, test_stats.avg_duration_pass)
+        _TestData(test_stats.test_name, test_stats.avg_duration_pass)
         for test_stats in historic_stats
     ])
 

buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml

@@ -52,6 +52,8 @@
 
         # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary.
         - jstests/core/geo_s2ordering.js
+        - jstests/fle2/**/*.js
+        - src/mongo/db/modules/*/jstests/fle2/**/*.js
 
 
 - name: reconfig_kill_primary_jscore_passthrough_exclude_files
@@ -120,6 +122,8 @@
 
         # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary.
         - jstests/core/geo_s2ordering.js
+        - jstests/fle2/**/*.js
+        - src/mongo/db/modules/*/jstests/fle2/**/*.js
 
 
 - name: kill_primary_jscore_passthrough_exclude_with_any_tags

buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml

@@ -3,8 +3,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
-  - jstests/fle2/**/*.js
-  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Has conditional logic for standalone servers and replica sets, but can't distinguish the two
   # when connected to mongos.

buildscripts/resmokeconfig/suites/cqf_passthrough.yml

@@ -55,6 +55,8 @@ selector:
   # Transactions are not supported on MongoDB standalone nodes, so we do not run these tests in the
   # 'core' suite. Instead we run them against a 1-node replica set in the 'core_txns' suite.
   - jstests/core/txns/**/*.js
+  # TODO SERVER-67506
+  - jstests/core/cover_null_queries.js
 
 executor:
   archive:

buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml

@@ -256,9 +256,6 @@ selector:
   - jstests/core/index_check6.js
   - jstests/core/index_check7.js
   - jstests/core/index_decimal.js
-  - jstests/core/index_elemmatch1.js
-  - jstests/core/index_elemmatch2.js
-  - jstests/core/index_elemmatch2.js
   - jstests/core/index_filter_commands.js
   - jstests/core/index_filter_on_hidden_index.js
   - jstests/core/index_multiple_compatibility.js
@@ -506,8 +503,6 @@ selector:
   - jstests/core/in7.js
   - jstests/core/index13.js
   - jstests/core/index_check2.js
-  - jstests/core/index_elemmatch1.js
-  - jstests/core/index_elemmatch2.js
   - jstests/core/indexl.js
   - jstests/core/json_schema/misc_validation.js
   - jstests/core/ne_array.js
@@ -756,7 +751,6 @@ selector:
   - jstests/core/geonear_key.js
   - jstests/core/getmore_invalidated_documents.js
   - jstests/core/hidden_index.js
-  - jstests/core/index_elemmatch2.js
   - jstests/core/index_partial_2dsphere.js
   - jstests/core/json_schema/misc_validation.js
   - jstests/core/list_collections_filter.js

buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml

@@ -7,8 +7,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
-  - jstests/fle2/**/*.js
-  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Transactions do not support retryability of individual operations.
   # TODO: Remove this once it is supported (SERVER-33952).

buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml

@@ -3,8 +3,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
-  - jstests/fle2/**/*.js
-  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Transactions do not support retryability of individual operations.
   # TODO: Remove this once it is supported (SERVER-33952).

buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml

@@ -3,8 +3,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
-  - jstests/fle2/**/*.js
-  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Transactions do not support retryability of individual operations.
   # TODO: Remove this once it is supported (SERVER-33952).

buildscripts/resmokelib/config.py

@@ -92,8 +92,9 @@ DEFAULTS = {
     "report_failure_status": "fail",
     "report_file": None,
     "run_all_feature_flag_tests": False,
-    "run_all_feature_flags_no_tests": False,
+    "run_no_feature_flag_tests": False,
     "additional_feature_flags": None,
+    "additional_feature_flags_file": None,
     "seed": int(time.time() * 256),  # Taken from random.py code in Python 2.7.
     "service_executor": None,
     "shell_conn_string": None,
@@ -379,8 +380,11 @@ INSTALL_DIR = None
 # Whether to run tests for feature flags.
 RUN_ALL_FEATURE_FLAG_TESTS = None
 
-# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true.
-RUN_ALL_FEATURE_FLAGS = None
+# Whether to run the tests with enabled feature flags
+RUN_NO_FEATURE_FLAG_TESTS = None
+
+# the path to a file containing feature flags
+ADDITIONAL_FEATURE_FLAGS_FILE = None
 
 # List of enabled feature flags.
 ENABLED_FEATURE_FLAGS = []

buildscripts/resmokelib/configure_resmoke.py

@@ -15,6 +15,7 @@ import shlex
 
 import pymongo.uri_parser
 
+from buildscripts.idl import gen_all_feature_flag_list
 from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE
 
 from buildscripts.resmokelib import config as _config
@@ -52,14 +53,9 @@ def _validate_options(parser, args):
             "Cannot use --replayFile with additional test files listed on the command line invocation."
         )
 
-    if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests:
-        if not os.path.isfile(ALL_FEATURE_FLAG_FILE):
-            parser.error(
-                "To run tests with all feature flags, the %s file must exist and be placed in"
-                " your working directory. The file can be downloaded from the artifacts tarball"
-                " in Evergreen. Alternatively, if you know which feature flags you want to enable,"
-                " you can use the --additionalFeatureFlags command line argument" %
-                ALL_FEATURE_FLAG_FILE)
+    if args.additional_feature_flags_file and not os.path.isfile(
+            args.additional_feature_flags_file):
+        parser.error("The specified additional feature flags file does not exist.")
 
     def get_set_param_errors(process_params):
         agg_set_params = collections.defaultdict(list)
@@ -185,28 +181,36 @@ be invoked as either:
 - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""")
         raise RuntimeError(err)
 
+    def process_feature_flag_file(path):
+        with open(path) as fd:
+            return fd.read().split()
+
     def setup_feature_flags():
         _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests")
-        _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests")
+        _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests")
+        _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file")
 
-        # Running all feature flag tests implies running the fixtures with feature flags.
         if _config.RUN_ALL_FEATURE_FLAG_TESTS:
-            _config.RUN_ALL_FEATURE_FLAGS = True
+            print("Generating: ", ALL_FEATURE_FLAG_FILE)
+            gen_all_feature_flag_list.gen_all_feature_flags_file()
 
         all_ff = []
         enabled_feature_flags = []
         try:
-            with open(ALL_FEATURE_FLAG_FILE) as fd:
-                all_ff = fd.read().split()
+            all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE)
         except FileNotFoundError:
             # If we ask resmoke to run with all feature flags, the feature flags file
             # needs to exist.
-            if _config.RUN_ALL_FEATURE_FLAGS:
+            if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS:
                 raise
 
-        if _config.RUN_ALL_FEATURE_FLAGS:
+        if _config.RUN_ALL_FEATURE_FLAG_TESTS:
             enabled_feature_flags = all_ff[:]
 
+        if _config.ADDITIONAL_FEATURE_FLAGS_FILE:
+            enabled_feature_flags.extend(
+                process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE))
+
         # Specify additional feature flags from the command line.
         # Set running all feature flag tests to True if this options is specified.
         additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags"))
@@ -231,7 +235,7 @@ be invoked as either:
     _config.EXCLUDE_WITH_ANY_TAGS.extend(
         utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
 
-    if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS:
+    if _config.RUN_NO_FEATURE_FLAG_TESTS:
         # Don't run any feature flag tests.
         _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags)
     else:

buildscripts/resmokelib/logging/buildlogger.py

@@ -297,6 +297,7 @@ class BuildloggerServer(object):
                 "builder": builder,
                 "buildnum": build_num,
                 "task_id": _config.EVERGREEN_TASK_ID,
+                "execution": _config.EVERGREEN_EXECUTION,
             })
 
         return response["id"]
@@ -315,6 +316,7 @@ class BuildloggerServer(object):
                 "command": test_command,
                 "phase": self.config.get("build_phase", "unknown"),
                 "task_id": _config.EVERGREEN_TASK_ID,
+                "execution": _config.EVERGREEN_EXECUTION,
             })
 
         return response["id"]

buildscripts/resmokelib/run/__init__.py

@@ -835,15 +835,19 @@ class RunPlugin(PluginInterface):
         )
 
         parser.add_argument(
-            "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests",
-            action="store_true", help=
-            "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites"
-        )
+            "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true",
+            help=("Do not run any tests tagged with enabled feature flags."
+                  " This argument has precedence over --runAllFeatureFlagTests"
+                  "; used for multiversion suites"))
 
         parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags",
                             action="append", metavar="featureFlag1, featureFlag2, ...",
                             help="Additional feature flags")
 
+        parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file",
+                            action="store", metavar="FILE",
+                            help="The path to a file with feature flags, delimited by newlines.")
+
         parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size",
                             help=argparse.SUPPRESS)
 
@@ -1191,15 +1195,6 @@ def to_local_args(input_args=None):  # pylint: disable=too-many-branches,too-man
     if origin_suite is not None:
         setattr(parsed_args, "suite_files", origin_suite)
 
-    # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on
-    # all_feature_flags.txt which may not exist in the local dev environment.
-    run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None)
-    if run_all_feature_flag_tests is not None:
-        setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS)
-        del parsed_args.run_all_feature_flag_tests
-
-    del parsed_args.run_all_feature_flags_no_tests
-
     # The top-level parser has one subparser that contains all subcommand parsers.
     command_subparser = [
         action for action in parser._actions  # pylint: disable=protected-access

buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py

@@ -276,16 +276,15 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
             secondary.await_ready()
 
             client = secondary.mongo_client()
-            minvalid_doc = client.local["replset.minvalid"].find_one()
             oplog_truncate_after_doc = client.local["replset.oplogTruncateAfterPoint"].find_one()
             recovery_timestamp_res = client.admin.command("replSetTest",
                                                           getLastStableRecoveryTimestamp=True)
             latest_oplog_doc = client.local["oplog.rs"].find_one(sort=[("$natural",
                                                                         pymongo.DESCENDING)])
 
-            self.logger.info("Checking invariants: minValid: {}, oplogTruncateAfterPoint: {},"
+            self.logger.info("Checking replication invariants. oplogTruncateAfterPoint: {},"
                              " stable recovery timestamp: {}, latest oplog doc: {}".format(
-                                 minvalid_doc, oplog_truncate_after_doc, recovery_timestamp_res,
+                                 oplog_truncate_after_doc, recovery_timestamp_res,
                                  latest_oplog_doc))
 
             null_ts = bson.Timestamp(0, 0)
@@ -299,13 +298,6 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
                 raise errors.ServerFailure(
                     "Latest oplog entry had no 'ts' field: {}".format(latest_oplog_doc))
 
-            # The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default
-            # it to null.
-            oplog_truncate_after_ts = null_ts
-            if oplog_truncate_after_doc is not None:
-                oplog_truncate_after_ts = oplog_truncate_after_doc.get(
-                    "oplogTruncateAfterPoint", null_ts)
-
             # The "lastStableRecoveryTimestamp" field is present if the storage engine supports
             # "recover to a timestamp". If it's a null timestamp on a durable storage engine, that
             # means we do not yet have a stable checkpoint timestamp and must be restarting at the
@@ -328,94 +320,6 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
                                                recovery_timestamp, latest_oplog_entry_ts,
                                                recovery_timestamp_res, latest_oplog_doc))
 
-            if minvalid_doc is not None:
-                applied_through_ts = minvalid_doc.get("begin", {}).get("ts", null_ts)
-                minvalid_ts = minvalid_doc.get("ts", null_ts)
-
-                # The "appliedThrough" value should always equal the "last stable recovery
-                # timestamp", AKA the stable checkpoint for durable engines, on server restart.
-                #
-                # The written "appliedThrough" time is updated with the latest timestamp at the end
-                # of each batch application, and batch boundaries are the only valid stable
-                # timestamps on secondaries. Therefore, a non-null appliedThrough timestamp must
-                # equal the checkpoint timestamp, because any stable timestamp that the checkpoint
-                # could use includes an equal persisted appliedThrough timestamp.
-                if (recovery_timestamp != null_ts and applied_through_ts != null_ts
-                        and (not recovery_timestamp == applied_through_ts)):
-                    raise errors.ServerFailure(
-                        "The condition last stable recovery timestamp ({}) == appliedThrough ({})"
-                        " doesn't hold: minValid document={},"
-                        " getLastStableRecoveryTimestamp result={}, last oplog entry={}".format(
-                            recovery_timestamp, applied_through_ts, minvalid_doc,
-                            recovery_timestamp_res, latest_oplog_doc))
-
-                if applied_through_ts == null_ts:
-                    # We clear "appliedThrough" to represent having applied through the top of the
-                    # oplog in PRIMARY state or immediately after "rollback via refetch".
-                    # If we are using a storage engine that supports "recover to a timestamp,"
-                    # then we will have a "last stable recovery timestamp" and we should use that
-                    # as our "appliedThrough" (similarly to why we assert their equality above).
-                    # If both are null, then we are in PRIMARY state on a storage engine that does
-                    # not support "recover to a timestamp" or in RECOVERING immediately after
-                    # "rollback via refetch". Since we do not update "minValid" in PRIMARY state,
-                    # we leave "appliedThrough" as null so that the invariants below hold, rather
-                    # than substituting the latest oplog entry for the "appliedThrough" value.
-                    applied_through_ts = recovery_timestamp
-
-                if minvalid_ts == null_ts:
-                    # The server treats the "ts" field in the minValid document as missing when its
-                    # value is the null timestamp.
-                    minvalid_ts = applied_through_ts
-
-                if latest_oplog_entry_ts == null_ts:
-                    # If the oplog is empty, we treat the "minValid" as the latest oplog entry.
-                    latest_oplog_entry_ts = minvalid_ts
-
-                if oplog_truncate_after_ts == null_ts:
-                    # The server treats the "oplogTruncateAfterPoint" field as missing when its
-                    # value is the null timestamp. When it is null, the oplog is complete and
-                    # should not be truncated, so it is effectively the top of the oplog.
-                    oplog_truncate_after_ts = latest_oplog_entry_ts
-
-                # Check the ordering invariants before the secondary has reconciled the end of
-                # its oplog.
-                # The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of
-                # oplog entries before they are written to the oplog. Thus, it can be ahead
-                # of the top of the oplog before any oplog entries are written, and behind it
-                # after some are written. Thus, we cannot compare it to the top of the oplog.
-
-                # appliedThrough <= minValid
-                # appliedThrough represents the end of the previous batch, so it is always the
-                # earliest.
-                if applied_through_ts > minvalid_ts:
-                    raise errors.ServerFailure(
-                        "The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid"
-                        " document={}, latest oplog entry={}".format(
-                            applied_through_ts, minvalid_ts, minvalid_doc, latest_oplog_doc))
-
-                # minValid <= oplogTruncateAfterPoint
-                # This is true because this hook is never run after a rollback. Thus, we only
-                # move "minValid" to the end of each batch after the batch is written to the oplog.
-                # We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from
-                # the end of the previous batch to the end of the current batch. Thus "minValid"
-                # must be less than or equal to the "oplogTruncateAfterPoint".
-                if minvalid_ts > oplog_truncate_after_ts:
-                    raise errors.ServerFailure(
-                        "The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't"
-                        " hold: minValid document={}, oplogTruncateAfterPoint document={},"
-                        " latest oplog entry={}".format(minvalid_ts, oplog_truncate_after_ts,
-                                                        minvalid_doc, oplog_truncate_after_doc,
-                                                        latest_oplog_doc))
-
-                # minvalid <= latest oplog entry
-                # "minValid" is set to the end of a batch after the batch is written to the oplog.
-                # Thus it is always less than or equal to the top of the oplog.
-                if minvalid_ts > latest_oplog_entry_ts:
-                    raise errors.ServerFailure(
-                        "The condition minValid <= top of oplog ({} <= {}) doesn't"
-                        " hold: minValid document={}, latest oplog entry={}".format(
-                            minvalid_ts, latest_oplog_entry_ts, minvalid_doc, latest_oplog_doc))
-
             try:
                 secondary.teardown()
             except errors.ServerFailure:

buildscripts/resmokelib/utils/globstar.py

@@ -1,11 +1,9 @@
 """Filename globbing utility."""
 
 import glob as _glob
-import os
 import os.path
 import re
 
-_GLOBSTAR = "**"
 _CONTAINS_GLOB_PATTERN = re.compile("[*?[]")
 
 
@@ -35,155 +33,6 @@ def iglob(globbed_pathname):
     expanded to match zero or more subdirectories.
     """
 
-    parts = _split_path(globbed_pathname)
-    parts = _canonicalize(parts)
-
-    index = _find_globstar(parts)
-    if index == -1:
-        for pathname in _glob.iglob(globbed_pathname):
-            # Normalize 'pathname' so exact string comparison can be used later.
-            yield os.path.normpath(pathname)
-        return
-
-    # **, **/, or **/a
-    if index == 0:
-        expand = _expand_curdir
-
-    # a/** or a/**/ or a/**/b
-    else:
-        expand = _expand
-
-    prefix_parts = parts[:index]
-    suffix_parts = parts[index + 1:]
-
-    prefix = os.path.join(*prefix_parts) if prefix_parts else os.curdir
-    suffix = os.path.join(*suffix_parts) if suffix_parts else ""
-
-    for (kind, path) in expand(prefix):
-        if not suffix_parts:
-            yield path
-
-        # Avoid following symlinks to avoid an infinite loop
-        elif suffix_parts and kind == "dir" and not os.path.islink(path):
-            path = os.path.join(path, suffix)
-            for pathname in iglob(path):
-                yield pathname
-
-
-def _split_path(pathname):
-    """Return 'pathname' as a list of path components."""
-
-    parts = []
-
-    while True:
-        (dirname, basename) = os.path.split(pathname)
-        parts.append(basename)
-        if pathname == dirname:
-            parts.append(dirname)
-            break
-        if not dirname:
-            break
-        pathname = dirname
-
-    parts.reverse()
-    return parts
-
-
-def _canonicalize(parts):
-    """Return a copy of 'parts' with consecutive "**"s coalesced.
-
-    Raise a ValueError for unsupported uses of "**".
-    """
-
-    res = []
-
-    prev_was_globstar = False
-    for part in parts:
-        if part == _GLOBSTAR:
-            # Skip consecutive **'s
-            if not prev_was_globstar:
-                prev_was_globstar = True
-                res.append(part)
-        elif _GLOBSTAR in part:  # a/b**/c or a/**b/c
-            raise ValueError("Can only specify glob patterns of the form a/**/b")
-        else:
-            prev_was_globstar = False
-            res.append(part)
-
-    return res
-
-
-def _find_globstar(parts):
-    """Return the index of the first occurrence of "**" in 'parts'.
-
-    Return -1 if "**" is not found in the list.
-    """
-
-    for (idx, part) in enumerate(parts):
-        if part == _GLOBSTAR:
-            return idx
-    return -1
-
-
-def _list_dir(pathname):
-    """Return a pair of subdirectory names and filenames contained within the 'pathname' directory.
-
-    If 'pathname' does not exist, then None is returned.
-    """
-
-    try:
-        (_root, dirs, files) = next(os.walk(pathname))
-        return (dirs, files)
-    except StopIteration:
-        return None  # 'pathname' directory does not exist
-
-
-def _expand(pathname):
-    """Emit tuples of the form ("dir", dirname) and ("file", filename).
-
-    The result is for all directories and files contained within the 'pathname' directory.
-    """
-
-    res = _list_dir(pathname)
-    if res is None:
-        return
-
-    (dirs, files) = res
-
-    # Zero expansion
-    if os.path.basename(pathname):
-        yield ("dir", os.path.join(pathname, ""))
-
-    for fname in files:
-        path = os.path.join(pathname, fname)
-        yield ("file", path)
-
-    for dname in dirs:
-        path = os.path.join(pathname, dname)
-        for xpath in _expand(path):
-            yield xpath
-
-
-def _expand_curdir(pathname):
-    """Emit tuples of the form ("dir", dirname) and ("file", filename).
-
-    The result is for all directories and files contained within the 'pathname' directory.
-
-    The returned pathnames omit a "./" prefix.
-    """
-
-    res = _list_dir(pathname)
-    if res is None:
-        return
-
-    (dirs, files) = res
-
-    # Zero expansion
-    yield ("dir", "")
-
-    for fname in files:
-        yield ("file", fname)
-
-    for dname in dirs:
-        for xdir in _expand(dname):
-            yield xdir
+    for pathname in _glob.iglob(globbed_pathname, recursive=True):
+        # Normalize 'pathname' so exact string comparison can be used later.
+        yield os.path.normpath(pathname)

buildscripts/scons_metrics/metrics.py

@@ -1,5 +1,6 @@
 """SCons metrics."""
 import re
+import os
 from typing import Optional, NamedTuple, List, Pattern, AnyStr
 
 from buildscripts.util.cedar_report import CedarMetric, CedarTestReport
@@ -105,7 +106,7 @@ class SconsMetrics:  # pylint: disable=too-many-instance-attributes
             self.total_command_execution_time = self._parse_float(
                 TOTAL_COMMAND_EXECUTION_TIME_REGEX, self.raw_report)
 
-        if cache_debug_log_file:
+        if os.path.exists(cache_debug_log_file):
             with open(cache_debug_log_file, "r") as fh:
                 self.final_cache_hit_ratio = self._parse_float(CACHE_HIT_RATIO_REGEX, fh.read())
         else:

buildscripts/scons_metrics/report.py

@@ -27,21 +27,6 @@ def main(scons_stdout_log_file: str, scons_cache_debug_log_file: str,
     scons_cache_debug_log_file = os.path.abspath(scons_cache_debug_log_file)
     cedar_report_file = os.path.abspath(cedar_report_file)
 
-    # This is a special file which must be available to use scons cache in evergreen. Here
-    # we are assuming that if this file is not present then this a static host which
-    # can not use scons cache in evergreen so therefore we will disable colleting cache metrics.
-    # Otherwise we assume the cache is available and if the cache log file is not found,
-    # something is not right and we exit with failure.
-    if not os.path.exists('/etc/mongodb-build-system-id'):
-        print(
-            "System is a static host and not connected to AWS cache dir. Skipping scons cache metrics."
-        )
-        scons_cache_debug_log_file = None
-    else:
-        if not os.path.exists(scons_cache_debug_log_file):
-            print(f"Could not find SCons cache debug log file '{scons_cache_debug_log_file}'.")
-            sys.exit(1)
-
     if not os.path.exists(scons_stdout_log_file):
         print(f"Could not find SCons stdout log file '{scons_stdout_log_file}'.")
         sys.exit(1)

buildscripts/task_generation/suite_split.py

@@ -172,7 +172,6 @@ class SuiteSplitService:
     @inject.autoparams()
     def __init__(
             self,
-            evg_api: EvergreenApi,
             resmoke_proxy: ResmokeProxyService,
             config: SuiteSplitConfig,
             split_strategy: SplitStrategy,
@@ -181,11 +180,9 @@ class SuiteSplitService:
         """
         Initialize the suite split service.
 
-        :param evg_api: Evergreen API client.
         :param resmoke_proxy: Resmoke Proxy service.
         :param config: Configuration options of how to split suites.
         """
-        self.evg_api = evg_api
         self.resmoke_proxy = resmoke_proxy
         self.config = config
         self.split_strategy = split_strategy
@@ -201,25 +198,15 @@ class SuiteSplitService:
         if self.config.default_to_fallback:
             return self.calculate_fallback_suites(params)
 
-        try:
-            evg_stats = HistoricTaskData.from_evg(self.evg_api, self.config.evg_project,
-                                                  self.config.start_date, self.config.end_date,
-                                                  params.task_name, params.build_variant)
-            if not evg_stats:
-                LOGGER.debug("No test history, using fallback suites")
-                # This is probably a new suite, since there is no test history, just use the
-                # fallback values.
-                return self.calculate_fallback_suites(params)
+        evg_stats = HistoricTaskData.from_s3(self.config.evg_project, params.task_name,
+                                             params.build_variant)
+
+        if evg_stats:
             return self.calculate_suites_from_evg_stats(evg_stats, params)
-        except requests.HTTPError as err:
-            if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE:
-                # Evergreen may return a 503 when the service is degraded.
-                # We fall back to splitting the tests into a fixed number of suites.
-                LOGGER.warning("Received 503 from Evergreen, "
-                               "dividing the tests evenly among suites")
-                return self.calculate_fallback_suites(params)
-            else:
-                raise
+
+        LOGGER.debug("No test history, using fallback suites")
+        # Since there is no test history this is probably a new suite, just use the fallback values.
+        return self.calculate_fallback_suites(params)
 
     def calculate_fallback_suites(self, params: SuiteSplitParameters) -> GeneratedSuite:
         """Divide tests into a fixed number of suites."""

buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp

@@ -32,3 +32,4 @@ iassert(27, "words");
 iasserted(28, "words");
 iassertNoTrace(29, "words");
 iassertedNoTrace(30, "words");
+MONGO_UNREACHABLE_TASSERT(31);

buildscripts/tests/task_generation/test_suite_split.py

@@ -8,7 +8,7 @@ import requests
 import buildscripts.task_generation.suite_split as under_test
 from buildscripts.task_generation.suite_split_strategies import greedy_division, \
     round_robin_fallback
-from buildscripts.util.teststats import TestRuntime
+from buildscripts.util.teststats import TestRuntime, HistoricalTestInformation
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
 
@@ -19,10 +19,9 @@ def mock_evg_error(mock_evg_api, error_code=requests.codes.SERVICE_UNAVAILABLE):
     return mock_evg_api
 
 
-def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None):
+def build_mock_service(split_config=None, resmoke_proxy=None):
 
     return under_test.SuiteSplitService(
-        evg_api=evg_api if evg_api else MagicMock(),
         resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(),
         config=split_config if split_config else MagicMock(),
         split_strategy=greedy_division,
@@ -31,7 +30,12 @@ def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None):
 
 
 def tst_stat_mock(file, duration, pass_count):
-    return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count)
+    return HistoricalTestInformation(
+        test_name=file,
+        num_pass=pass_count,
+        num_fail=0,
+        avg_duration_pass=duration,
+    )
 
 
 def build_mock_split_config(target_resmoke_time=None, max_sub_suites=None):
@@ -115,15 +119,16 @@ class TestGeneratedSuite(unittest.TestCase):
 
 
 class TestSplitSuite(unittest.TestCase):
-    def test_calculate_suites(self):
+    @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
+    def test_calculate_suites(self, get_stats_from_s3_mock):
         mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)]
         split_config = build_mock_split_config(target_resmoke_time=10)
         split_params = build_mock_split_params()
 
         suite_split_service = build_mock_service(split_config=split_config)
-        suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats
+        get_stats_from_s3_mock.return_value = mock_test_stats
         suite_split_service.resmoke_proxy.list_tests.return_value = [
-            stat.test_file for stat in mock_test_stats
+            stat.test_name for stat in mock_test_stats
         ]
         suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
 
@@ -137,32 +142,15 @@ class TestSplitSuite(unittest.TestCase):
         for sub_suite in suite.sub_suites:
             self.assertEqual(10, len(sub_suite.test_list))
 
-    def test_calculate_suites_fallback_on_error(self):
-        n_tests = 100
-        max_sub_suites = 4
-        split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
-        split_params = build_mock_split_params()
-
-        suite_split_service = build_mock_service(split_config=split_config)
-        mock_evg_error(suite_split_service.evg_api)
-        suite_split_service.resmoke_proxy.list_tests.return_value = [
-            f"test_{i}.js" for i in range(n_tests)
-        ]
-
-        suite = suite_split_service.split_suite(split_params)
-
-        self.assertEqual(max_sub_suites, len(suite))
-        for sub_suite in suite.sub_suites:
-            self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
-
-    def test_calculate_suites_uses_fallback_on_no_results(self):
+    @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
+    def test_calculate_suites_uses_fallback_on_no_results(self, get_stats_from_s3_mock):
         n_tests = 100
         max_sub_suites = 5
         split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
         split_params = build_mock_split_params()
 
         suite_split_service = build_mock_service(split_config=split_config)
-        suite_split_service.evg_api.test_stats_by_project.return_value = []
+        get_stats_from_s3_mock.return_value = []
         suite_split_service.resmoke_proxy.list_tests.return_value = [
             f"test_{i}.js" for i in range(n_tests)
         ]
@@ -173,7 +161,9 @@ class TestSplitSuite(unittest.TestCase):
         for sub_suite in suite.sub_suites:
             self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
 
-    def test_calculate_suites_uses_fallback_if_only_results_are_filtered(self):
+    @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
+    def test_calculate_suites_uses_fallback_if_only_results_are_filtered(
+            self, get_stats_from_s3_mock):
         n_tests = 100
         max_sub_suites = 10
         mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)]
@@ -182,7 +172,7 @@ class TestSplitSuite(unittest.TestCase):
         split_params = build_mock_split_params()
 
         suite_split_service = build_mock_service(split_config=split_config)
-        suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats
+        get_stats_from_s3_mock.return_value = mock_test_stats
         suite_split_service.resmoke_proxy.list_tests.return_value = [
             f"test_{i}.js" for i in range(n_tests)
         ]
@@ -198,31 +188,17 @@ class TestSplitSuite(unittest.TestCase):
         for sub_suite in suite.sub_suites:
             self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
 
-    def test_calculate_suites_fail_on_unexpected_error(self):
-        n_tests = 100
-        max_sub_suites = 4
-        split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
-        split_params = build_mock_split_params()
-
-        suite_split_service = build_mock_service(split_config=split_config)
-        mock_evg_error(suite_split_service.evg_api, error_code=requests.codes.INTERNAL_SERVER_ERROR)
-        suite_split_service.resmoke_proxy.list_tests.return_value = [
-            f"test_{i}.js" for i in range(n_tests)
-        ]
-
-        with self.assertRaises(requests.HTTPError):
-            suite_split_service.split_suite(split_params)
-
-    def test_calculate_suites_will_filter_specified_tests(self):
+    @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
+    def test_calculate_suites_will_filter_specified_tests(self, get_stats_from_s3_mock):
         mock_test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)]
         split_config = build_mock_split_config(target_resmoke_time=10)
         split_params = build_mock_split_params(
             test_filter=lambda t: t in {"test_1.js", "test_2.js"})
 
         suite_split_service = build_mock_service(split_config=split_config)
-        suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats
+        get_stats_from_s3_mock.return_value = mock_test_stats
         suite_split_service.resmoke_proxy.list_tests.return_value = [
-            stat.test_file for stat in mock_test_stats
+            stat.test_name for stat in mock_test_stats
         ]
         suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
 

buildscripts/tests/test_errorcodes.py

@@ -26,7 +26,7 @@ class TestErrorcodes(unittest.TestCase):
             captured_error_codes.append(code)
 
         errorcodes.parse_source_files(accumulate_files, TESTDATA_DIR + 'regex_matching/')
-        self.assertEqual(30, len(captured_error_codes))
+        self.assertEqual(31, len(captured_error_codes))
 
     def test_dup_checking(self):
         """Test dup checking."""

buildscripts/tests/timeouts/test_timeout_service.py

@@ -1,41 +1,42 @@
 """Unit tests for timeout_service.py."""
 import random
 import unittest
-from datetime import datetime, timedelta
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from requests.exceptions import HTTPError
-from evergreen import EvergreenApi
 
 import buildscripts.timeouts.timeout_service as under_test
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.util.teststats import HistoricTaskData
+from buildscripts.util.teststats import HistoricTaskData, HistoricTestInfo
 
 # pylint: disable=missing-docstring,no-self-use,invalid-name,protected-access
 
+NS = "buildscripts.timeouts.timeout_service"
 
-def build_mock_service(evg_api=None, resmoke_proxy=None):
-    end_date = datetime.now()
-    start_date = end_date - timedelta(weeks=2)
-    timeout_settings = under_test.TimeoutSettings(
-        end_date=end_date,
-        start_date=start_date,
-    )
+
+def ns(relative_name):  # pylint: disable=invalid-name
+    """Return a full name from a name relative to the test module"s name space."""
+    return NS + "." + relative_name
+
+
+def build_mock_service(resmoke_proxy=None):
     return under_test.TimeoutService(
-        evg_api=evg_api if evg_api else MagicMock(spec_set=EvergreenApi),
-        resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService),
-        timeout_settings=timeout_settings)
+        resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService))
 
 
 def tst_stat_mock(file, duration, pass_count):
-    return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count)
+    return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count, hooks=[])
+
+
+def tst_runtime_mock(file, duration, pass_count):
+    return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count)
 
 
 class TestGetTimeoutEstimate(unittest.TestCase):
-    def test_no_stats_should_return_default_timeout(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
-        mock_evg_api.test_stats_by_project.return_value = []
-        timeout_service = build_mock_service(evg_api=mock_evg_api)
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_no_stats_should_return_default_timeout(self, from_s3_mock: MagicMock):
+        timeout_service = build_mock_service()
+        from_s3_mock.return_value = []
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",
@@ -48,13 +49,17 @@ class TestGetTimeoutEstimate(unittest.TestCase):
 
         self.assertFalse(timeout.is_specified())
 
-    def test_a_test_with_missing_history_should_cause_a_default_timeout(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
-        test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)]
-        mock_evg_api.test_stats_by_project.return_value = test_stats
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_a_test_with_missing_history_should_cause_a_default_timeout(
+            self, from_s3_mock: MagicMock):
+        test_stats = [
+            HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=60, num_pass=1, hooks=[])
+            for i in range(30)
+        ]
+        from_s3_mock.return_value = HistoricTaskData(test_stats)
         mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
         mock_resmoke_proxy.list_tests.return_value = ["test_with_no_stats.js"]
-        timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
+        timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",
@@ -67,14 +72,19 @@ class TestGetTimeoutEstimate(unittest.TestCase):
 
         self.assertFalse(timeout.is_specified())
 
-    def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
-        test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)]
-        test_stats.append(tst_stat_mock("zero.js", 0.0, 1))
-        mock_evg_api.test_stats_by_project.return_value = test_stats
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(
+            self, from_s3_mock: MagicMock):
+        test_stats = [
+            HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=60, num_pass=1, hooks=[])
+            for i in range(30)
+        ]
+        test_stats.append(
+            HistoricTestInfo(test_name="zero.js", avg_duration=0.0, num_pass=1, hooks=[]))
+        from_s3_mock.return_value = HistoricTaskData(test_stats)
         mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
-        mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats]
-        timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
+        mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats]
+        timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",
@@ -87,15 +97,19 @@ class TestGetTimeoutEstimate(unittest.TestCase):
 
         self.assertFalse(timeout.is_specified())
 
-    def test_all_tests_with_runtime_history_should_use_custom_timeout(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_all_tests_with_runtime_history_should_use_custom_timeout(self,
+                                                                      from_s3_mock: MagicMock):
         n_tests = 30
         test_runtime = 600
-        test_stats = [tst_stat_mock(f"test_{i}.js", test_runtime, 1) for i in range(n_tests)]
-        mock_evg_api.test_stats_by_project.return_value = test_stats
+        test_stats = [
+            HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=test_runtime, num_pass=1,
+                             hooks=[]) for i in range(n_tests)
+        ]
+        from_s3_mock.return_value = HistoricTaskData(test_stats)
         mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
-        mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats]
-        timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
+        mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats]
+        timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",
@@ -149,10 +163,10 @@ class TestGetTaskHookOverhead(unittest.TestCase):
 
 
 class TestLookupHistoricStats(unittest.TestCase):
-    def test_no_stats_from_evergreen_should_return_none(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
-        mock_evg_api.test_stats_by_project.return_value = []
-        timeout_service = build_mock_service(evg_api=mock_evg_api)
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_no_stats_from_evergreen_should_return_none(self, from_s3_mock: MagicMock):
+        from_s3_mock.return_value = None
+        timeout_service = build_mock_service()
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",
@@ -165,10 +179,10 @@ class TestLookupHistoricStats(unittest.TestCase):
 
         self.assertIsNone(stats)
 
-    def test_errors_from_evergreen_should_return_none(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
-        mock_evg_api.test_stats_by_project.side_effect = HTTPError("failed to connect")
-        timeout_service = build_mock_service(evg_api=mock_evg_api)
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_errors_from_evergreen_should_return_none(self, from_s3_mock: MagicMock):
+        from_s3_mock.side_effect = HTTPError("failed to connect")
+        timeout_service = build_mock_service()
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",
@@ -181,11 +195,11 @@ class TestLookupHistoricStats(unittest.TestCase):
 
         self.assertIsNone(stats)
 
-    def test_stats_from_evergreen_should_return_the_stats(self):
-        mock_evg_api = MagicMock(spec_set=EvergreenApi)
+    @patch(ns("HistoricTaskData.from_s3"))
+    def test_stats_from_evergreen_should_return_the_stats(self, from_s3_mock: MagicMock):
         test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)]
-        mock_evg_api.test_stats_by_project.return_value = test_stats
-        timeout_service = build_mock_service(evg_api=mock_evg_api)
+        from_s3_mock.return_value = HistoricTaskData(test_stats)
+        timeout_service = build_mock_service()
         timeout_params = under_test.TimeoutParams(
             evg_project="my project",
             build_variant="bv",

buildscripts/tests/util/test_teststats.py

@@ -80,7 +80,7 @@ class TestHistoricTaskData(unittest.TestCase):
     @staticmethod
     def _make_evg_result(test_file="dir/test1.js", num_pass=0, duration=0):
         return Mock(
-            test_file=test_file,
+            test_name=test_file,
             task_name="task1",
             variant="variant1",
             distro="distro1",

buildscripts/timeouts/timeout_service.py

@@ -1,5 +1,4 @@
 """Service for determining task timeouts."""
-from datetime import datetime
 from typing import Any, Dict, NamedTuple, Optional
 
 import inject
@@ -31,29 +30,17 @@ class TimeoutParams(NamedTuple):
     is_asan: bool
 
 
-class TimeoutSettings(NamedTuple):
-    """Settings for determining timeouts."""
-
-    start_date: datetime
-    end_date: datetime
-
-
 class TimeoutService:
     """A service for determining task timeouts."""
 
     @inject.autoparams()
-    def __init__(self, evg_api: EvergreenApi, resmoke_proxy: ResmokeProxyService,
-                 timeout_settings: TimeoutSettings) -> None:
+    def __init__(self, resmoke_proxy: ResmokeProxyService) -> None:
         """
         Initialize the service.
 
-        :param evg_api: Evergreen API client.
         :param resmoke_proxy: Proxy to query resmoke.
-        :param timeout_settings: Settings for how timeouts are calculated.
         """
-        self.evg_api = evg_api
         self.resmoke_proxy = resmoke_proxy
-        self.timeout_settings = timeout_settings
 
     def get_timeout_estimate(self, timeout_params: TimeoutParams) -> TimeoutEstimate:
         """
@@ -129,7 +116,8 @@ class TimeoutService:
             return n_expected_runs * avg_clean_every_n_runtime
         return 0.0
 
-    def lookup_historic_stats(self, timeout_params: TimeoutParams) -> Optional[HistoricTaskData]:
+    @staticmethod
+    def lookup_historic_stats(timeout_params: TimeoutParams) -> Optional[HistoricTaskData]:
         """
         Lookup historic test results stats for the given task.
 
@@ -137,10 +125,8 @@ class TimeoutService:
         :return: Historic test results if they exist.
         """
         try:
-            evg_stats = HistoricTaskData.from_evg(
-                self.evg_api, timeout_params.evg_project, self.timeout_settings.start_date,
-                self.timeout_settings.end_date, timeout_params.task_name,
-                timeout_params.build_variant)
+            evg_stats = HistoricTaskData.from_s3(
+                timeout_params.evg_project, timeout_params.task_name, timeout_params.build_variant)
             if not evg_stats:
                 LOGGER.warning("No historic runtime information available")
                 return None

buildscripts/util/teststats.py

@@ -1,15 +1,32 @@
 """Utility to support parsing a TestStat."""
 from collections import defaultdict
 from dataclasses import dataclass
-from datetime import datetime
 from itertools import chain
+from json import JSONDecodeError
 from typing import NamedTuple, List, Callable, Optional
-
-from evergreen import EvergreenApi, TestStats
+import requests
+from requests.adapters import HTTPAdapter, Retry
 
 from buildscripts.util.testname import split_test_hook_name, is_resmoke_hook, get_short_name_from_test_file
 
 TASK_LEVEL_HOOKS = {"CleanEveryN"}
+TESTS_STATS_S3_LOCATION = "https://mongo-test-stats.s3.amazonaws.com"
+
+
+class HistoricalTestInformation(NamedTuple):
+    """
+    Container for information about the historical runtime of a test.
+
+    test_name: Name of test.
+    avg_duration_pass: Average of runtime of test that passed.
+    num_pass: Number of times the test has passed.
+    num_fail: Number of times the test has failed.
+    """
+
+    test_name: str
+    num_pass: int
+    num_fail: int
+    avg_duration_pass: float
 
 
 class TestRuntime(NamedTuple):
@@ -74,9 +91,9 @@ class HistoricHookInfo(NamedTuple):
     avg_duration: float
 
     @classmethod
-    def from_test_stats(cls, test_stats: TestStats) -> "HistoricHookInfo":
+    def from_test_stats(cls, test_stats: HistoricalTestInformation) -> "HistoricHookInfo":
         """Create an instance from a test_stats object."""
-        return cls(hook_id=test_stats.test_file, num_pass=test_stats.num_pass,
+        return cls(hook_id=test_stats.test_name, num_pass=test_stats.num_pass,
                    avg_duration=test_stats.avg_duration_pass)
 
     def test_name(self) -> str:
@@ -101,10 +118,10 @@ class HistoricTestInfo(NamedTuple):
     hooks: List[HistoricHookInfo]
 
     @classmethod
-    def from_test_stats(cls, test_stats: TestStats,
+    def from_test_stats(cls, test_stats: HistoricalTestInformation,
                         hooks: List[HistoricHookInfo]) -> "HistoricTestInfo":
         """Create an instance from a test_stats object."""
-        return cls(test_name=test_stats.test_file, num_pass=test_stats.num_pass,
+        return cls(test_name=test_stats.test_name, num_pass=test_stats.num_pass,
                    avg_duration=test_stats.avg_duration_pass, hooks=hooks)
 
     def normalized_test_name(self) -> str:
@@ -137,46 +154,59 @@ class HistoricTaskData(object):
         """Initialize the TestStats with raw results from the Evergreen API."""
         self.historic_test_results = historic_test_results
 
-    # pylint: disable=too-many-arguments
-    @classmethod
-    def from_evg(cls, evg_api: EvergreenApi, project: str, start_date: datetime, end_date: datetime,
-                 task: str, variant: str) -> "HistoricTaskData":
+    @staticmethod
+    def get_stats_from_s3(project: str, task: str, variant: str) -> List[HistoricalTestInformation]:
         """
-        Retrieve test stats from evergreen for a given task.
+        Retrieve test stats from s3 for a given task.
+
+        :param project: Project to query.
+        :param task: Task to query.
+        :param variant: Build variant to query.
+        :return: A list of the Test stats for the specified task.
+        """
+        session = requests.Session()
+        retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
+        session.mount('https://', HTTPAdapter(max_retries=retries))
+
+        response = session.get(f"{TESTS_STATS_S3_LOCATION}/{project}/{variant}/{task}")
+
+        try:
+            data = response.json()
+            return [HistoricalTestInformation(**item) for item in data]
+        except JSONDecodeError:
+            return []
+
+    @classmethod
+    def from_s3(cls, project: str, task: str, variant: str) -> "HistoricTaskData":
+        """
+        Retrieve test stats from s3 for a given task.
 
-        :param evg_api: Evergreen API client.
         :param project: Project to query.
-        :param start_date: Start date to query.
-        :param end_date: End date to query.
         :param task: Task to query.
         :param variant: Build variant to query.
         :return: Test stats for the specified task.
         """
-        days = (end_date - start_date).days
-        historic_stats = evg_api.test_stats_by_project(
-            project, after_date=start_date, before_date=end_date, tasks=[task], variants=[variant],
-            group_by="test", group_num_days=days)
-
-        return cls.from_stats_list(historic_stats)
+        historical_test_data = cls.get_stats_from_s3(project, task, variant)
+        return cls.from_stats_list(historical_test_data)
 
     @classmethod
-    def from_stats_list(cls, historic_stats: List[TestStats]) -> "HistoricTaskData":
+    def from_stats_list(
+            cls, historical_test_data: List[HistoricalTestInformation]) -> "HistoricTaskData":
         """
         Build historic task data from a list of historic stats.
 
-        :param historic_stats: List of historic stats to build from.
+        :param historical_test_data: A list of information about the runtime of a test.
         :return: Historic task data from the list of stats.
         """
-
         hooks = defaultdict(list)
-        for hook in [stat for stat in historic_stats if is_resmoke_hook(stat.test_file)]:
+        for hook in [stat for stat in historical_test_data if is_resmoke_hook(stat.test_name)]:
             historical_hook = HistoricHookInfo.from_test_stats(hook)
             hooks[historical_hook.test_name()].append(historical_hook)
 
         return cls([
             HistoricTestInfo.from_test_stats(stat,
-                                             hooks[get_short_name_from_test_file(stat.test_file)])
-            for stat in historic_stats if not is_resmoke_hook(stat.test_file)
+                                             hooks[get_short_name_from_test_file(stat.test_name)])
+            for stat in historical_test_data if not is_resmoke_hook(stat.test_name)
         ])
 
     def get_tests_runtimes(self) -> List[TestRuntime]:

debian/mongod.1

@@ -19,6 +19,10 @@ more details, see \fBDisable TLS 1.0\f1\&.
 .SH OPTIONS
 .RS
 .IP \(bu 2
+MongoDB removes the \fB\-\-cpu\f1 command\-line option.
+.RE
+.RS
+.IP \(bu 2
 MongoDB removes the \fB\-\-serviceExecutor\f1 command\-line option and the
 corresponding \fBnet.serviceExecutor\f1 configuration option.
 .RE
@@ -194,10 +198,15 @@ link\-local IPv6 address (https://en.wikipedia.org/wiki/Link\-local_address#IPv6
 zone index (https://en.wikipedia.org/wiki/IPv6_address#Scoped_literal_IPv6_addresses_(with_zone_index))
 to that address (i.e. \fBfe80::<address>%<adapter\-name>\f1).
 .PP
-When possible, use a logical DNS hostname instead of an ip address,
-particularly when configuring replica set members or sharded cluster
-members. The use of logical DNS hostnames avoids configuration
-changes due to ip address changes.
+To avoid configuration updates due to IP address changes, use DNS
+hostnames instead of IP addresses. It is particularly important to
+use a DNS hostname instead of an IP address when configuring replica
+set members or sharded cluster members.
+.PP
+Use hostnames instead of IP addresses to configure clusters across a
+split network horizon. Starting in MongoDB 5.0, nodes that are only
+configured with an IP address will fail startup validation and will
+not start.
 .PP
 Before binding to a non\-localhost (e.g. publicly accessible)
 IP address, ensure you have secured your cluster from unauthorized
@@ -677,13 +686,6 @@ connect to the \fBmongod\f1\f1 using the appropriate \fBuser\f1
 prior to restarting \fBmongod\f1\f1 without \fB\-\-transitionToAuth\f1\f1\&.
 .RE
 .PP
-\fBmongod \-\-cpu\f1
-.RS
-.PP
-Forces the \fBmongod\f1\f1 process to report the percentage of CPU time in
-write lock, every four seconds.
-.RE
-.PP
 \fBmongod \-\-sysinfo\f1
 .RS
 .PP
@@ -721,6 +723,8 @@ For additional ways to shut down, see also \fBStop mongod\f1 Processes\f1\&.
 \fBmongod \-\-redactClientLogData\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A \fBmongod\f1\f1 running with \fB\-\-redactClientLogData\f1\f1 redacts any message accompanying a given
 log event before logging. This prevents the \fBmongod\f1\f1 from writing
 potentially sensitive data stored on the database to the diagnostic log.
@@ -935,6 +939,8 @@ For the corresponding configuration file setting, see
 \fBmongod \-\-ldapServers\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The LDAP server against which the \fBmongod\f1\f1 authenticates users or
 determines what actions a user is authorized to perform on a given
 database. If the LDAP server specified has any replicated instances,
@@ -975,6 +981,8 @@ server is unavailable.
 \fBmongod \-\-ldapQueryUser\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The identity with which \fBmongod\f1\f1 binds as, when connecting to or
 performing queries on an LDAP server.
 .PP
@@ -1000,28 +1008,27 @@ instead of \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1\&. You
 both \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapBindWithOSDefaults\f1\f1 at the same time.
 .RE
 .PP
-\fBmongod \-\-ldapQueryPassword\f1
-.RS
+\fIAvailable in MongoDB Enterprise only.\f1
 .PP
 The password used to bind to an LDAP server when using
 \fB\-\-ldapQueryUser\f1\f1\&. You must use \fB\-\-ldapQueryPassword\f1\f1 with
 \fB\-\-ldapQueryUser\f1\f1\&.
 .PP
-If unset, \fBmongod\f1\f1 will not attempt to bind to the LDAP server.
+If not set, \fBmongod\f1\f1 does not attempt to bind to the LDAP server.
 .PP
-This setting can be configured on a running \fBmongod\f1\f1 using
+You can configure this setting on a running \fBmongod\f1\f1 using
 \fBsetParameter\f1\f1\&.
 .PP
 Starting in MongoDB 4.4, the \fBldapQueryPassword\f1
 \fBsetParameter\f1\f1 command accepts either a string or
-an array of strings. If set to an array, each password is tried
-until one succeeds. This can be used to perform a rollover of the
-LDAP account password without downtime for MongoDB.
+an array of strings. If \fBldapQueryPassword\f1 is set to an array, MongoDB tries
+each password in order until one succeeds. Use a password array to roll over the
+LDAP account password without downtime.
 .PP
 Windows MongoDB deployments can use \fB\-\-ldapBindWithOSDefaults\f1\f1
-instead of \fB\-\-ldapQueryPassword\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1\&. You cannot specify
-both \fB\-\-ldapQueryPassword\f1\f1 and \fB\-\-ldapBindWithOSDefaults\f1\f1 at the same time.
-.RE
+instead of \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1\&.
+You cannot specify both \fB\-\-ldapQueryPassword\f1\f1 and
+\fB\-\-ldapBindWithOSDefaults\f1\f1 at the same time.
 .PP
 \fBmongod \-\-ldapBindWithOSDefaults\f1
 .RS
@@ -1052,6 +1059,8 @@ Use \fB\-\-ldapBindWithOSDefaults\f1\f1 to replace \fB\-\-ldapQueryUser\f1\f1 an
 .PP
 \fIDefault\f1: simple
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The method \fBmongod\f1\f1 uses to authenticate to an LDAP server.
 Use with \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1 to
 connect to the LDAP server.
@@ -1074,6 +1083,8 @@ using \fBDIGEST\-MD5\f1 mechanism.
 .PP
 \fIDefault\f1: DIGEST\-MD5
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A comma\-separated list of SASL mechanisms \fBmongod\f1\f1 can
 use when authenticating to the LDAP server. The \fBmongod\f1\f1 and the
 LDAP server must agree on at least one mechanism. The \fBmongod\f1\f1
@@ -1145,6 +1156,8 @@ For Windows, please see the Windows SASL documentation (https://msdn.microsoft.c
 .PP
 \fIDefault\f1: tls
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 By default, \fBmongod\f1\f1 creates a TLS/SSL secured connection to the LDAP
 server.
 .PP
@@ -1173,6 +1186,8 @@ credentials between \fBmongod\f1\f1 and the LDAP server.
 .PP
 \fIDefault\f1: 10000
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The amount of time in milliseconds \fBmongod\f1\f1 should wait for an LDAP server
 to respond to a request.
 .PP
@@ -1188,6 +1203,8 @@ This setting can be configured on a running \fBmongod\f1\f1 using
 \fBmongod \-\-ldapUserToDNMapping\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 Maps the username provided to \fBmongod\f1\f1 for authentication to a LDAP
 Distinguished Name (DN). You may need to use \fB\-\-ldapUserToDNMapping\f1\f1 to transform a
 username into an LDAP DN in the following scenarios:
@@ -1352,6 +1369,8 @@ This setting can be configured on a running \fBmongod\f1\f1 using the
 \fBmongod \-\-ldapAuthzQueryTemplate\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A relative LDAP query URL formatted conforming to RFC4515 (https://tools.ietf.org/search/rfc4515) and RFC4516 (https://tools.ietf.org/html/rfc4516) that \fBmongod\f1\f1 executes to obtain
 the LDAP groups to which the authenticated user belongs to. The query is
 relative to the host or hosts specified in \fB\-\-ldapServers\f1\f1\&.
@@ -1459,7 +1478,7 @@ To specify the \fBWiredTiger Storage Engine\f1\&.
 .IP \(bu 4
 To specify the \fBIn\-Memory Storage Engine\f1\&.
 .IP
-Available in MongoDB Enterprise only.
+\fIAvailable in MongoDB Enterprise only.\f1
 .RE
 .RE
 .PP
@@ -2015,10 +2034,18 @@ and is always set to \fBtrue\f1\&. In earlier versions of MongoDB,
 \fB\-\-enableMajorityReadConcern\f1\f1 was configurable.
 .PP
 If you are using a three\-member primary\-secondary\-arbiter (PSA)
-architecture, the write concern \fB"majority"\f1\f1 can cause
-performance issues if a secondary is unavailable or lagging. See
-\fBMitigate Performance Issues with PSA Replica Set\f1 for advice on how to mitigate these
-issues.
+architecture, consider the following:
+.RS
+.IP \(bu 2
+The write concern \fB"majority"\f1\f1 can cause
+performance issues if a secondary is unavailable or lagging. For
+advice on how to mitigate these issues, see
+\fBMitigate Performance Issues with PSA Replica Set\f1\&.
+.IP \(bu 2
+If you are using a global default \fB"majority"\f1\f1
+and the write concern is less than the size of the majority,
+your queries may return stale (not fully replicated) data.
+.RE
 .RE
 .SS SHARDED CLUSTER OPTIONS
 .PP
@@ -3331,7 +3358,17 @@ This is the default profiler level.
 \fB1\f1
 .IP \(bu 4
 The profiler collects data for operations that take longer
-than the value of \fBslowms\f1\&.
+than the value of \fBslowms\f1 or that match a \fBfilter\f1\&.
+.IP
+When a filter is set:
+.RS
+.IP \(bu 6
+The \fBslowms\f1 and \fBsampleRate\f1 options are not used for
+profiling.
+.IP \(bu 6
+The profiler only captures operations that match the
+\fBfilter\f1\&.
+.RE
 .RE
 .IP \(bu 2
 .RS
@@ -3361,9 +3398,7 @@ that run for longer than this threshold are considered \fIslow\f1\&.
 .PP
 When \fBlogLevel\f1\f1 is set to \fB0\f1, MongoDB records \fIslow\f1
 operations to the diagnostic log at a rate determined by
-\fBslowOpSampleRate\f1\f1\&. Starting in MongoDB
-4.2, the secondaries of replica sets log \fBall oplog entry messages
-that take longer than the slow operation threshold to apply\f1 regardless of the sample rate.
+\fBslowOpSampleRate\f1\f1\&.
 .PP
 At higher \fBlogLevel\f1\f1 settings, all operations appear in
 the diagnostic log regardless of their latency with the following
@@ -3395,6 +3430,43 @@ diagnostic log and, if enabled, the profiler.
 .RE
 .SS AUDIT OPTIONS
 .PP
+\fBmongod \-\-auditCompressionMode\f1
+.RS
+.PP
+Specifies the compression mode for \fBaudit log encryption\f1\&. You must also enable audit log
+encryption using either \fB\-\-auditEncryptionKeyUID\f1\f1 or
+\fB\-\-auditLocalKeyFile\f1\f1\&.
+.PP
+\fB\-\-auditCompressionMode\f1\f1 can be set to one of these values:
+.RS
+.IP \(bu 2
+.RS
+.IP \(bu 4
+Value
+.IP \(bu 4
+Description
+.RE
+.IP \(bu 2
+.RS
+.IP \(bu 4
+\fBzstd\f1
+.IP \(bu 4
+Use the \fBzstd\f1 algorithm to compress the audit log.
+.RE
+.IP \(bu 2
+.RS
+.IP \(bu 4
+\fBnone\f1 \fI(default)\f1
+.IP \(bu 4
+Do not compress the audit log.
+.RE
+.RE
+.PP
+Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)\&.
+MongoDB Enterprise and Atlas have different configuration
+requirements.
+.RE
+.PP
 \fBmongod \-\-auditDestination\f1
 .RS
 .PP
@@ -3445,6 +3517,20 @@ Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-e
 and MongoDB Atlas (https://cloud.mongodb.com/user#/atlas/login)\&.
 .RE
 .PP
+\fBmongod \-\-auditEncryptionKeyUID\f1
+.RS
+.PP
+Specifies the unique identifier of the Key Management
+Interoperability Protocol (KMIP) key for \fBaudit log encryption\f1\&.
+.PP
+You cannot use \fB\-\-auditEncryptionKeyUID\f1\f1 and
+\fB\-\-auditLocalKeyFile\f1\f1 together.
+.PP
+Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)\&.
+MongoDB Enterprise and Atlas have different configuration
+requirements.
+.RE
+.PP
 \fBmongod \-\-auditFormat\f1
 .RS
 .PP
@@ -3483,6 +3569,25 @@ Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-e
 and MongoDB Atlas (https://cloud.mongodb.com/user#/atlas/login)\&.
 .RE
 .PP
+\fBmongod \-\-auditLocalKeyFile\f1
+.RS
+.PP
+Specifies the path and file name for a local audit key file for
+\fBaudit log encryption\f1\&.
+.PP
+Only use \fB\-\-auditLocalKeyFile\f1\f1 for testing because the key is
+not secured. To secure the key, use
+\fB\-\-auditEncryptionKeyUID\f1\f1 and an external Key
+Management Interoperability Protocol (KMIP) server.
+.PP
+You cannot use \fB\-\-auditLocalKeyFile\f1\f1 and
+\fB\-\-auditEncryptionKeyUID\f1\f1 together.
+.PP
+Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)\&.
+MongoDB Enterprise and Atlas have different configuration
+requirements.
+.RE
+.PP
 \fBmongod \-\-auditPath\f1
 .RS
 .PP
@@ -3517,6 +3622,37 @@ the configuration file.
 Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)
 and MongoDB Atlas (https://cloud.mongodb.com/user#/atlas/login)\&.
 .RE
+.SS SNMP OPTIONS
+.PP
+MongoDB Enterprise on macOS does \fInot\f1 include support for SNMP due
+to SERVER\-29352 (https://jira.mongodb.org/browse/SERVER\-29352)\&.
+.PP
+\fBmongod \-\-snmp\-disabled\f1
+.RS
+.PP
+Disables SNMP access to \fBmongod\f1\f1\&. The option is incompatible
+with \fB\-\-snmp\-subagent\f1\f1 and \fB\-\-snmp\-master\f1\f1\&.
+.RE
+.PP
+\fBmongod \-\-snmp\-subagent\f1
+.RS
+.PP
+Runs SNMP as a subagent. The option is incompatible with \fB\-\-snmp\-disabled\f1\f1\&.
+.RE
+.PP
+\fBmongod \-\-snmp\-master\f1
+.RS
+.PP
+Runs SNMP as a master. The option is incompatible with \fB\-\-snmp\-disabled\f1\f1\&.
+.RE
+.RS
+.IP \(bu 2
+\fBMonitor MongoDB With SNMP on Linux\f1
+.IP \(bu 2
+\fBMonitor MongoDB Windows with SNMP\f1
+.IP \(bu 2
+\fBTroubleshoot SNMP\f1
+.RE
 .SS INMEMORY OPTIONS
 .PP
 \fBmongod \-\-inMemorySizeGB\f1
@@ -3792,8 +3928,36 @@ KMIP server.
 .PP
 Starting in 4.0, on macOS or Windows, you can use a certificate
 from the operating system\(aqs secure store instead of a PEM key
-file. See \fB\-\-kmipClientCertificateSelector\f1\f1\&. When using the secure store, you do not
-need to, but can, also specify the \fB\-\-kmipServerCAFile\f1\f1\&.
+file. See \fB\-\-kmipClientCertificateSelector\f1\f1\&. When using the secure
+store, you do not need to, but can, also specify the \fB\-\-kmipServerCAFile\f1\f1\&.
+.RE
+.PP
+\fBmongod \-\-kmipActivateKeys\f1
+.RS
+.PP
+\fIDefault\f1: true
+.PP
+Activates all newly created KMIP keys upon creation and then periodically
+checks those keys are in an active state.
+.PP
+When \fB\-\-kmipActivateKeys\f1 is \fBtrue\f1 and you have existing keys on a
+KMIP server, the key must be activated first or the \fBmongod\f1\f1 node
+will fail to start.
+.PP
+If the key being used by the mongod transitions into a non\-active state,
+the \fBmongod\f1\f1 node will shut down unless \fBkmipActivateKeys\f1 is
+false. To ensure you have an active key, rotate the KMIP master key by
+using \fB\-\-kmipRotateMasterKey\f1\f1\&.
+.RE
+.PP
+\fBmongod \-\-kmipKeyStatePollingSeconds\f1
+.RS
+.PP
+\fIDefault\f1: 900 seconds
+.PP
+Frequency in seconds at which mongod polls the KMIP server for active keys.
+.PP
+To disable disable polling, set the value to \fB\-1\f1\&.
 .RE
 .PP
 \fBmongod \-\-eseDatabaseKeyRollover\f1

debian/mongod.conf

@@ -6,8 +6,6 @@
 # Where and how to store data.
 storage:
   dbPath: /var/lib/mongodb
-  journal:
-    enabled: true
 #  engine:
 #  wiredTiger:
 

debian/mongoldap.1

@@ -1,6 +1,8 @@
 .TH mongoldap 1
 .SH MONGOLDAP
+\fIMongoDB Enterprise\f1
 .SH SYNOPSIS
+.PP
 Starting in version 3.4, MongoDB Enterprise provides
 \fBmongoldap\f1\f1 for testing MongoDB\(aqs LDAP \fBconfiguration
 options\f1 against a running LDAP server or set
@@ -174,6 +176,18 @@ configuration files are valid, the output might be as follows:
   [OK] Successfully acquired the following roles:
   ...
 .EE
+.SH BEHAVIOR
+.PP
+Starting in MonogoDB 5.1, \fBmongoldap\f1 supports prefixing LDAP
+server with \fBsrv:\f1 and \fBsrv_raw:\f1\&.
+.PP
+If your connection string specifies \fB"srv:<DNS_NAME>"\f1, \fBmongoldap\f1
+verifies that \fB"_ldap._tcp.gc._msdcs.<DNS_NAME>"\f1 exists for SRV to
+support Active Directory. If not found, it verifies
+\fB"_ldap._tcp.<DNS_NAME>"\f1 exists for SRV. If an SRV record cannot be
+found, \fBmongoldap\f1 warns you to use \fB"srv_raw:<DNS_NAME>"\f1 instead.
+\fBmongoldap\f1 does the reverse check for \fB"srv_raw:<DNS_NAME>"\f1 by
+checking for \fB"_ldap._tcp.<DNS NAME>"\f1\&.
 .SH OPTIONS
 .PP
 \fBmongoldap \-\-config\f1, \fBmongoldap \-f\f1
@@ -235,6 +249,8 @@ If unset, \fBmongoldap\f1\f1 cannot use \fBLDAP authentication or authorization\
 \fBmongoldap \-\-ldapQueryUser\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The identity with which \fBmongoldap\f1\f1 binds as, when connecting to or
 performing queries on an LDAP server.
 .PP
@@ -260,22 +276,27 @@ instead of \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1\&. You
 both \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapBindWithOSDefaults\f1\f1 at the same time.
 .RE
 .PP
-\fBmongoldap \-\-ldapQueryPassword\f1
-.RS
+\fIAvailable in MongoDB Enterprise only.\f1
 .PP
 The password used to bind to an LDAP server when using
 \fB\-\-ldapQueryUser\f1\f1\&. You must use \fB\-\-ldapQueryPassword\f1\f1 with
 \fB\-\-ldapQueryUser\f1\f1\&.
 .PP
-If unset, \fBmongoldap\f1\f1 will not attempt to bind to the LDAP server.
+If not set, \fBmongoldap\f1\f1 does not attempt to bind to the LDAP server.
 .PP
-This setting can be configured on a running \fBmongoldap\f1\f1 using
+You can configure this setting on a running \fBmongoldap\f1\f1 using
 \fBsetParameter\f1\f1\&.
 .PP
+Starting in MongoDB 4.4, the \fBldapQueryPassword\f1
+\fBsetParameter\f1\f1 command accepts either a string or
+an array of strings. If \fBldapQueryPassword\f1 is set to an array, MongoDB tries
+each password in order until one succeeds. Use a password array to roll over the
+LDAP account password without downtime.
+.PP
 Windows MongoDB deployments can use \fB\-\-ldapBindWithOSDefaults\f1\f1
-instead of \fB\-\-ldapQueryPassword\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1\&. You cannot specify
-both \fB\-\-ldapQueryPassword\f1\f1 and \fB\-\-ldapBindWithOSDefaults\f1\f1 at the same time.
-.RE
+instead of \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1\&.
+You cannot specify both \fB\-\-ldapQueryPassword\f1\f1 and
+\fB\-\-ldapBindWithOSDefaults\f1\f1 at the same time.
 .PP
 \fBmongoldap \-\-ldapBindWithOSDefaults\f1
 .RS
@@ -306,6 +327,8 @@ Use \fB\-\-ldapBindWithOSDefaults\f1\f1 to replace \fB\-\-ldapQueryUser\f1\f1 an
 .PP
 \fIDefault\f1: simple
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The method \fBmongoldap\f1\f1 uses to authenticate to an LDAP
 server. Use with \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1 to connect to the LDAP server.
 .PP
@@ -345,6 +368,8 @@ using \fBDIGEST\-MD5\f1 mechanism.
 .PP
 \fIDefault\f1: DIGEST\-MD5
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A comma\-separated list of SASL mechanisms \fBmongoldap\f1\f1 can
 use when authenticating to the LDAP server. The \fBmongoldap\f1\f1 and the
 LDAP server must agree on at least one mechanism. The \fBmongoldap\f1\f1
@@ -416,6 +441,8 @@ For Windows, please see the Windows SASL documentation (https://msdn.microsoft.c
 .PP
 \fIDefault\f1: tls
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 By default, \fBmongoldap\f1\f1 creates a TLS/SSL secured connection to the LDAP
 server.
 .PP
@@ -444,6 +471,8 @@ credentials between \fBmongoldap\f1\f1 and the LDAP server.
 .PP
 \fIDefault\f1: 10000
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The amount of time in milliseconds \fBmongoldap\f1\f1 should wait for an LDAP server
 to respond to a request.
 .PP
@@ -459,6 +488,8 @@ This setting can be configured on a running \fBmongoldap\f1\f1 using
 \fBmongoldap \-\-ldapUserToDNMapping\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 Maps the username provided to \fBmongoldap\f1\f1 for authentication to a LDAP
 Distinguished Name (DN). You may need to use \fB\-\-ldapUserToDNMapping\f1\f1 to transform a
 username into an LDAP DN in the following scenarios:
@@ -623,6 +654,8 @@ This setting can be configured on a running \fBmongoldap\f1\f1 using the
 \fBmongoldap \-\-ldapAuthzQueryTemplate\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A relative LDAP query URL formatted conforming to RFC4515 (https://tools.ietf.org/search/rfc4515) and RFC4516 (https://tools.ietf.org/html/rfc4516) that \fBmongoldap\f1\f1 executes to obtain
 the LDAP groups to which the authenticated user belongs to. The query is
 relative to the host or hosts specified in \fB\-\-ldapServers\f1\f1\&.

debian/mongos.1

@@ -20,8 +20,8 @@ Starting in version 4.0, MongoDB disables support for TLS 1.0
 encryption on systems where TLS 1.1+ is available. For
 more details, see \fBDisable TLS 1.0\f1\&.
 .IP \(bu 2
-Starting in MongoDB 4.0, the \fBmongos\f1\f1 binary will crash when
-attempting to connect to \fBmongod\f1\f1 instances whose
+The \fBmongos\f1\f1 binary will crash when attempting to connect
+to \fBmongod\f1\f1 instances whose
 \fBfeature compatibility version (fCV)\f1 is greater than
 that of the \fBmongos\f1\f1\&. For example, you cannot connect
 a MongoDB 4.0 version \fBmongos\f1\f1 to a 4.2
@@ -191,10 +191,15 @@ link\-local IPv6 address (https://en.wikipedia.org/wiki/Link\-local_address#IPv6
 zone index (https://en.wikipedia.org/wiki/IPv6_address#Scoped_literal_IPv6_addresses_(with_zone_index))
 to that address (i.e. \fBfe80::<address>%<adapter\-name>\f1).
 .PP
-When possible, use a logical DNS hostname instead of an ip address,
-particularly when configuring replica set members or sharded cluster
-members. The use of logical DNS hostnames avoids configuration
-changes due to ip address changes.
+To avoid configuration updates due to IP address changes, use DNS
+hostnames instead of IP addresses. It is particularly important to
+use a DNS hostname instead of an IP address when configuring replica
+set members or sharded cluster members.
+.PP
+Use hostnames instead of IP addresses to configure clusters across a
+split network horizon. Starting in MongoDB 5.0, nodes that are only
+configured with an IP address will fail startup validation and will
+not start.
 .PP
 Before binding to a non\-localhost (e.g. publicly accessible)
 IP address, ensure you have secured your cluster from unauthorized
@@ -349,6 +354,8 @@ If you specify \fBreopen\f1, you must also use \fB\-\-logappend\f1\f1\&.
 \fBmongos \-\-redactClientLogData\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A \fBmongos\f1\f1 running with \fB\-\-redactClientLogData\f1\f1 redacts any message accompanying a given
 log event before logging. This prevents the \fBmongos\f1\f1 from writing
 potentially sensitive data stored on the database to the diagnostic log.
@@ -726,7 +733,7 @@ port of different members of the replica set.
 Specifies the ping time, in milliseconds, that \fBmongos\f1\f1 uses
 to determine which secondary replica set members to pass read
 operations from clients. The default value of \fB15\f1 corresponds to
-the default value in all of the client drivers (https://docs.mongodb.com/drivers/)\&.
+the default value in all of the client drivers (https://www.mongodb.com/docs/drivers/)\&.
 .PP
 When \fBmongos\f1\f1 receives a request that permits reads to
 \fBsecondary\f1 members, the \fBmongos\f1\f1 will:
@@ -1773,6 +1780,43 @@ available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-e
 .RE
 .SS AUDIT OPTIONS
 .PP
+\fBmongos \-\-auditCompressionMode\f1
+.RS
+.PP
+Specifies the compression mode for \fBaudit log encryption\f1\&. You must also enable audit log
+encryption using either \fB\-\-auditEncryptionKeyUID\f1\f1 or
+\fB\-\-auditLocalKeyFile\f1\f1\&.
+.PP
+\fB\-\-auditCompressionMode\f1\f1 can be set to one of these values:
+.RS
+.IP \(bu 2
+.RS
+.IP \(bu 4
+Value
+.IP \(bu 4
+Description
+.RE
+.IP \(bu 2
+.RS
+.IP \(bu 4
+\fBzstd\f1
+.IP \(bu 4
+Use the \fBzstd\f1 algorithm to compress the audit log.
+.RE
+.IP \(bu 2
+.RS
+.IP \(bu 4
+\fBnone\f1 \fI(default)\f1
+.IP \(bu 4
+Do not compress the audit log.
+.RE
+.RE
+.PP
+Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)\&.
+MongoDB Enterprise and Atlas have different configuration
+requirements.
+.RE
+.PP
 \fBmongos \-\-auditDestination\f1
 .RS
 .PP
@@ -1823,6 +1867,20 @@ Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-e
 and MongoDB Atlas (https://cloud.mongodb.com/user#/atlas/login)\&.
 .RE
 .PP
+\fBmongos \-\-auditEncryptionKeyUID\f1
+.RS
+.PP
+Specifies the unique identifier of the Key Management
+Interoperability Protocol (KMIP) key for \fBaudit log encryption\f1\&.
+.PP
+You cannot use \fB\-\-auditEncryptionKeyUID\f1\f1 and
+\fB\-\-auditLocalKeyFile\f1\f1 together.
+.PP
+Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)\&.
+MongoDB Enterprise and Atlas have different configuration
+requirements.
+.RE
+.PP
 \fBmongos \-\-auditFormat\f1
 .RS
 .PP
@@ -1861,6 +1919,25 @@ Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-e
 and MongoDB Atlas (https://cloud.mongodb.com/user#/atlas/login)\&.
 .RE
 .PP
+\fBmongos \-\-auditLocalKeyFile\f1
+.RS
+.PP
+Specifies the path and file name for a local audit key file for
+\fBaudit log encryption\f1\&.
+.PP
+Only use \fB\-\-auditLocalKeyFile\f1\f1 for testing because the key is
+not secured. To secure the key, use
+\fB\-\-auditEncryptionKeyUID\f1\f1 and an external Key
+Management Interoperability Protocol (KMIP) server.
+.PP
+You cannot use \fB\-\-auditLocalKeyFile\f1\f1 and
+\fB\-\-auditEncryptionKeyUID\f1\f1 together.
+.PP
+Available only in MongoDB Enterprise (http://www.mongodb.com/products/mongodb\-enterprise\-advanced?tck=docs_server)\&.
+MongoDB Enterprise and Atlas have different configuration
+requirements.
+.RE
+.PP
 \fBmongos \-\-auditPath\f1
 .RS
 .PP
@@ -1934,6 +2011,8 @@ only and not the profiler since profiling is not available on
 \fBmongos \-\-ldapServers\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The LDAP server against which the \fBmongos\f1\f1 authenticates users or
 determines what actions a user is authorized to perform on a given
 database. If the LDAP server specified has any replicated instances,
@@ -1974,6 +2053,8 @@ server is unavailable.
 \fBmongos \-\-ldapQueryUser\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The identity with which \fBmongos\f1\f1 binds as, when connecting to or
 performing queries on an LDAP server.
 .PP
@@ -2002,6 +2083,8 @@ both \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapBindWithOSDefaults\f1\f1 at the s
 \fBmongos \-\-ldapQueryPassword\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The password used to bind to an LDAP server when using
 \fB\-\-ldapQueryUser\f1\f1\&. You must use \fB\-\-ldapQueryPassword\f1\f1 with
 \fB\-\-ldapQueryUser\f1\f1\&.
@@ -2045,6 +2128,8 @@ Use \fB\-\-ldapBindWithOSDefaults\f1\f1 to replace \fB\-\-ldapQueryUser\f1\f1 an
 .PP
 \fIDefault\f1: simple
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The method \fBmongos\f1\f1 uses to authenticate to an LDAP server.
 Use with \fB\-\-ldapQueryUser\f1\f1 and \fB\-\-ldapQueryPassword\f1\f1 to
 connect to the LDAP server.
@@ -2067,6 +2152,8 @@ using \fBDIGEST\-MD5\f1 mechanism.
 .PP
 \fIDefault\f1: DIGEST\-MD5
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 A comma\-separated list of SASL mechanisms \fBmongos\f1\f1 can
 use when authenticating to the LDAP server. The \fBmongos\f1\f1 and the
 LDAP server must agree on at least one mechanism. The \fBmongos\f1\f1
@@ -2138,6 +2225,8 @@ For Windows, please see the Windows SASL documentation (https://msdn.microsoft.c
 .PP
 \fIDefault\f1: tls
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 By default, \fBmongos\f1\f1 creates a TLS/SSL secured connection to the LDAP
 server.
 .PP
@@ -2166,6 +2255,8 @@ credentials between \fBmongos\f1\f1 and the LDAP server.
 .PP
 \fIDefault\f1: 10000
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 The amount of time in milliseconds \fBmongos\f1\f1 should wait for an LDAP server
 to respond to a request.
 .PP
@@ -2181,6 +2272,8 @@ This setting can be configured on a running \fBmongos\f1\f1 using
 \fBmongos \-\-ldapUserToDNMapping\f1
 .RS
 .PP
+\fIAvailable in MongoDB Enterprise only.\f1
+.PP
 Maps the username provided to \fBmongos\f1\f1 for authentication to a LDAP
 Distinguished Name (DN). You may need to use \fB\-\-ldapUserToDNMapping\f1\f1 to transform a
 username into an LDAP DN in the following scenarios:

etc/evergreen.yml

@@ -57,6 +57,7 @@ include:
 - filename: etc/evergreen_yml_components/variants/task_generation.yml
 - filename: etc/evergreen_yml_components/variants/sanitizer.yml
 - filename: etc/evergreen_yml_components/variants/in_memory.yml
+- filename: etc/evergreen_yml_components/variants/ninja.yml
 
 variables:
 - &libfuzzertests
@@ -191,8 +192,6 @@ variables:
     - ubuntu2004-package
   - name: selinux_rhel7_enterprise
   - name: .publish
-    distros:
-    - rhel70-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 
@@ -274,21 +273,6 @@ buildvariants:
     - rhel80-large
   - name: generate_buildid_to_debug_symbols_mapping
 
-- <<: *linux-64-debug-required-template
-  name: linux-64-debug-wtdevelop
-  display_name: "~ Linux DEBUG WiredTiger develop"
-  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
-  modules:
-  - wtdevelop
-  expansions:
-    use_wt_develop: true
-    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
-    compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-free-mon=on --enable-http-client=on --link-model=dynamic
-    scons_cache_mode: all
-    scons_cache_scope: shared
-    num_scons_link_jobs_available: 0.99
-    test_flags: --excludeWithAnyTags=requires_http_client
-
 - name: tla-plus
   display_name: TLA+
   run_on:
@@ -877,7 +861,7 @@ buildvariants:
     # To force disable feature flags even on the all feature flags variant, please use this file:
     # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_windows_tls
       --excludeWithAnyTags=incompatible_with_shard_merge
     external_auth_jobs_max: 1
@@ -936,21 +920,6 @@ buildvariants:
 #      distros:
 #        - windows-vsCurrent-xlarge
 
-- name: enterprise-windows-ninja
-  display_name: "Ninja Build: Enterprise Windows"
-  cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
-  modules:
-  - enterprise
-  expansions:
-    compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include" LIBPATH="c:/sasl/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5")  --win-version-min=win10
-  tasks:
-  - name: compile_ninja_next_TG
-    distros:
-    - windows-vsCurrent-large
-  - name: compile_ninja_TG
-    distros:
-    - windows-vsCurrent-large
-
 - name: enterprise-windows-cxx20-debug-experimental
   display_name: "~ Enterprise Windows C++20 DEBUG"
   cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
@@ -1070,20 +1039,6 @@ buildvariants:
   - name: unittest_shell_hang_analyzer_gen
   - name: generate_buildid_to_debug_symbols_mapping
 
-- name: macos-enterprise-ninja
-  display_name: "Ninja Build: macOS Enterprise"
-  cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
-  modules:
-  - enterprise
-  run_on:
-  - macos-1100
-  expansions:
-    compile_env: DEVELOPER_DIR=/Applications/Xcode13.app
-    compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars
-  tasks:
-  - name: compile_ninja_next_TG
-  - name: compile_ninja_TG
-
 - name: enterprise-macos-rosetta-2
   display_name: "Enterprise macOS Via Rosetta 2"
   cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
@@ -1166,12 +1121,8 @@ buildvariants:
   - name: replica_sets_jscore_passthrough
   - name: sasl
   - name: push
-    distros:
-    - rhel70-small
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - rhel70-small
 
 ###########################################
 #          Embedded SDK buildvariants     #
@@ -1255,6 +1206,8 @@ buildvariants:
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
     burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
   tasks:
   - name: compile_test_and_package_parallel_core_stream_TG
@@ -1382,26 +1335,6 @@ buildvariants:
     <<: *enterprise-rhel-80-64-bit-dynamic-required-expansions
     compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-glibcxx-debug --dbg=on --allocator=system
 
-- name: enterprise-rhel-80-64-bit-dynamic-required-ninja
-  display_name: "Ninja Build: Enterprise RHEL 8.0"
-  cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
-  modules:
-  - enterprise
-  run_on:
-  - rhel80-small
-  stepback: false
-  expansions:
-    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
-    repo_edition: enterprise
-    has_packages: false
-  tasks:
-  - name: compile_ninja_next_TG
-    distros:
-    - rhel80-xlarge
-  - name: compile_ninja_TG
-    distros:
-    - rhel80-xlarge
-
 - &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-template
   name: enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required
   display_name: "! Shared Library Enterprise RHEL 8.0 (all feature flags)"
@@ -1429,7 +1362,7 @@ buildvariants:
     # To force disable feature flags even on the all feature flags variant, please use this file:
     # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_shard_merge
   tasks: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-tasks
   - name: cqf
@@ -1565,6 +1498,8 @@ buildvariants:
     max_sub_suites: 5
     large_distro_name: rhel80-medium
     burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
@@ -1790,7 +1725,7 @@ buildvariants:
     <<: *enterprise-rhel-80-64-bit-multiversion-expansions-template
     # No feature flag tests since they aren't compatible with the older binaries.
     test_flags: >-
-      --runAllFeatureFlagsNoTests
+      --runNoFeatureFlagTests
       --excludeWithAnyTags=incompatible_with_shard_merge
   tasks:
   - name: compile_test_and_package_parallel_core_stream_TG
@@ -2231,7 +2166,7 @@ buildvariants:
     # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
     large_distro_name: rhel80-build
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_shard_merge
     separate_debug: off
   tasks:
@@ -2389,7 +2324,7 @@ buildvariants:
     test_flags: >-
       --excludeWithAnyTags=requires_ocsp_stapling
       --excludeWithAnyTags=incompatible_with_shard_merge
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
     multiversion_platform: rhel80
     multiversion_edition: enterprise
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under UBSAN build.
@@ -2581,7 +2516,7 @@ buildvariants:
     test_flags: >-
       --excludeWithAnyTags=requires_ocsp_stapling
       --excludeWithAnyTags=incompatible_with_shard_merge
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under {A,UB}SAN build.
     hang_analyzer_dump_core: false
     scons_cache_scope: shared
@@ -3045,7 +2980,7 @@ buildvariants:
     target_resmoke_time: 30
     max_sub_suites: 3
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=resource_intensive
       --excludeWithAnyTags=incompatible_with_shard_merge
 
@@ -3105,17 +3040,6 @@ buildvariants:
     - windows-vsCurrent-large
   - name: .benchmarks !benchmarks_orphaned
 
-- <<: *enterprise-windows-nopush-template
-  name: enterprise-windows-wtdevelop
-  display_name: "~ Enterprise Windows WiredTiger develop"
-  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
-  modules:
-  - enterprise
-  - wtdevelop
-  expansions:
-    <<: *enterprise-windows-nopush-expansions-template
-    use_wt_develop: true
-
 ### QO & QE Patch-Specific Build Variants ###
 - <<: *enterprise-windows-nopush-template
   name: windows-compile-query-patch-only

etc/evergreen_nightly.yml

@@ -6,14 +6,386 @@ include:
 - filename: etc/evergreen_yml_components/variants/atlas.yml
 - filename: etc/evergreen_yml_components/variants/misc_release.yml
 ### Comment out when using this file for a LTS or Rapid release branch. ###
-- filename: etc/evergreen_yml_components/variants/ibm.yml
+#- filename: etc/evergreen_yml_components/variants/ibm.yml
 ### Uncomment when using this file for a LTS release branch. ###
-# - filename: etc/evergreen_yml_components/variants/in_memory.yml
+#- filename: etc/evergreen_yml_components/variants/in_memory.yml
 ### Uncomment when using this file for a LTS or Rapid release branch. ###
-# - filename: etc/evergreen_yml_components/variants/sanitizer.yml
+- filename: etc/evergreen_yml_components/variants/sanitizer.yml
+### Uncomment when using this file for a LTS or Rapid release branch. ###
+- filename: etc/evergreen_yml_components/variants/ninja.yml
 
 
 parameters:
 - key: evergreen_config_file_path
   value: "etc/evergreen_nightly.yml"
   description: "path to this file"
+
+buildvariants:
+- &linux-64-debug-required-template
+  name: linux-64-debug-required
+  display_name: "! Shared Library Linux DEBUG"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  run_on:
+  - rhel80-medium
+  expansions:
+    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
+    compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-free-mon=on --enable-http-client=on --link-model=dynamic
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    test_flags: --excludeWithAnyTags=requires_http_client
+    target_resmoke_time: 15
+    max_sub_suites: 5
+    num_scons_link_jobs_available: 0.99
+    large_distro_name: rhel80-medium
+  tasks:
+  - name: compile_test_and_package_parallel_core_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: compile_test_and_package_parallel_unittest_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: compile_test_and_package_parallel_dbtest_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: .aggregation !.encrypt !.feature_flag_guarded
+  - name: .auth !.audit !.multiversion
+  - name: .causally_consistent !.wo_snapshot
+  - name: .change_streams !.secondary_reads
+  - name: .clustered_collections
+  - name: .misc_js
+  - name: disk_wiredtiger
+  - name: free_monitoring
+  - name: .integration !.audit
+  - name: .jscore .common
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: query_golden_classic
+  - name: libunwind_tests
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: .ocsp
+  - name: .read_write_concern
+  - name: .replica_sets !.encrypt !.ignore_non_generated_replica_sets_jscore_passthrough !.fcbis
+  - name: replica_sets_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+  - name: .retry
+  - name: .read_only
+  - name: session_jscore_passthrough
+  - name: sharded_multi_stmt_txn_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot
+  - name: sharding_gen
+  - name: sharding_opportunistic_secondary_targeting_gen
+  - name: .stitch
+  - name: server_discovery_and_monitoring_json_test_TG
+    distros:
+    - rhel80-large
+  - name: server_selection_json_test_TG
+    distros:
+    - rhel80-large
+  - name: generate_buildid_to_debug_symbols_mapping
+
+- &enterprise-windows-required-template
+  name: enterprise-windows-required
+  display_name: "! Enterprise Windows"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - windows-vsCurrent-small
+  expansions:
+    burn_in_tests_build_variant: enterprise-windows-required
+    exe: ".exe"
+    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug msi archive-mh archive-mh-debug
+    content_type: application/zip
+    compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include" LIBPATH="c:/sasl/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8") --win-version-min=win10
+    num_scons_link_jobs_available: 0.2
+    python: '/cygdrive/c/python/python37/python.exe'
+    ext: zip
+    scons_cache_scope: shared
+    multiversion_platform: windows
+    multiversion_edition: enterprise
+    jstestfuzz_num_generated_files: 35
+    target_resmoke_time: 20
+    max_sub_suites: 5
+    large_distro_name: windows-vsCurrent-large
+    push_path: windows
+    push_bucket: downloads.10gen.com
+    push_name: windows
+    push_arch: x86_64-enterprise
+    test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls
+    external_auth_jobs_max: 1
+  tasks:
+  - name: compile_test_and_package_serial_TG
+    distros:
+    - windows-vsCurrent-xlarge
+  - name: compile_build_tools_next_TG
+    distros:
+    - windows-vsCurrent-xlarge
+  - name: burn_in_tests_gen
+  - name: audit
+  - name: auth_audit_gen
+  - name: buildscripts_test
+  - name: causally_consistent_jscore_txns_passthrough
+    distros:
+    - windows-vsCurrent-large
+  - name: .crypt
+    distros:
+    - windows-vsCurrent-large
+  - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
+  - name: external_auth
+  - name: external_auth_aws
+  - name: external_auth_windows
+    distros:
+    - windows-2016-dc
+  - name: .jscore .common !.sharding
+  - name: jsCore_auth
+  - name: jsCore_ese
+  - name: jsCore_txns_large_txns_format
+  - name: .jstestfuzz .common
+  - name: mqlrun
+  - name: noPassthrough_gen
+  - name: noPassthroughWithMongod_gen
+  - name: .replica_sets .common !.ignore_non_generated_replica_sets_jscore_passthrough
+  - name: .replica_sets .multi_oplog !.ignore_non_generated_replica_sets_jscore_passthrough
+  - name: replica_sets_jscore_passthrough
+    distros:
+    - windows-vsCurrent-large
+  - name: replica_sets_ese_gen
+  - name: sasl
+  - name: server_discovery_and_monitoring_json_test_TG
+  - name: server_selection_json_test_TG
+  - name: .sharding .txns
+  - name: sharding_auth_gen
+  - name: sharding_auth_audit_gen
+  - name: sharding_ese_gen
+  - name: sharding_opportunistic_secondary_targeting_gen
+  - name: unittest_shell_hang_analyzer_gen
+
+- &enterprise-rhel-80-64-bit-dynamic-required-template
+  name: enterprise-rhel-80-64-bit-dynamic-required
+  display_name: "! Shared Library Enterprise RHEL 8.0"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-small
+  expansions: &enterprise-rhel-80-64-bit-dynamic-required-expansions
+    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug
+    compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
+    crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique"
+    multiversion_platform: rhel80
+    multiversion_edition: enterprise
+    has_packages: false
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    jstestfuzz_num_generated_files: 40
+    jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 10
+    max_sub_suites: 5
+    idle_timeout_factor: 1.5
+    exec_timeout_factor: 1.5
+    large_distro_name: rhel80-medium
+    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    num_scons_link_jobs_available: 0.99
+  tasks:
+  - name: compile_test_and_package_parallel_core_stream_TG
+    distros:
+    - rhel80-xlarge
+  #  - name: compile_test_and_package_parallel_unittest_stream_with_recording_TG
+  #    distros:
+  #      - rhel80-xlarge
+  - name: compile_test_and_package_parallel_unittest_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: compile_test_and_package_parallel_dbtest_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: .lint
+  - name: lint_fuzzer_sanity_patch
+  - name: test_api_version_compatibility
+  - name: burn_in_tests_gen
+  - name: check_feature_flag_tags
+  - name: check_for_todos
+  - name: .aggfuzzer !.feature_flag_guarded
+  - name: .aggregation !.feature_flag_guarded
+  - name: audit
+  - name: .auth
+  #  - name: burn_in_tags_gen
+  - name: buildscripts_test
+  - name: resmoke_end2end_tests
+  - name: unittest_shell_hang_analyzer_gen
+  - name: .causally_consistent !.sharding
+  - name: .change_streams
+  - name: .change_stream_fuzzer
+  - name: .misc_js
+  - name: .concurrency !.large !.ubsan !.no_txns !.debug_only
+  - name: .concurrency .large !.ubsan !.no_txns !.debug_only
+    distros:
+    - rhel80-medium
+  - name: config_fuzzer_concurrency
+  - name: config_fuzzer_simulate_crash_concurrency_replication
+    distros:
+    - rhel80-large
+  - name: config_fuzzer_concurrency_replication
+    distros:
+    - rhel80-large
+  - name: config_fuzzer_jsCore
+  - name: config_fuzzer_replica_sets_jscore_passthrough
+    distros:
+    - rhel80-large
+  - name: disk_wiredtiger
+  - name: .encrypt
+  - name: idl_tests
+  - name: initial_sync_fuzzer_gen
+  - name: .integration
+    distros:
+    - rhel80-medium
+  - name: jsCore
+    distros:
+    - rhel80-xlarge
+  - name: .jscore .common !jsCore
+  - name: jsCore_minimum_batch_size
+  - name: jsCore_txns_large_txns_format
+  - name: json_schema
+  - name: .jstestfuzz !.flow_control  # Flow control jstestfuzz take longer.
+  - name: libunwind_tests
+  - name: .multiversion_sanity_check
+  - name: mqlrun
+  - name: .multi_shard
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: multiversion_gen
+  - name: .query_fuzzer
+  - name: .random_multiversion_ds
+  - name: .read_write_concern .large
+    distros:
+    - rhel80-medium
+  - name: .read_write_concern !.large
+  - name: .replica_sets !.encrypt !.auth
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_api_version_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_passthrough_gen
+  - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: replica_sets_reconfig_kill_primary_jscore_passthrough_gen
+    distros:
+    - rhel80-xlarge
+  - name: retryable_writes_jscore_passthrough_gen
+  - name: retryable_writes_jscore_stepdown_passthrough_gen
+    distros:
+    - rhel80-medium
+  - name: .read_only
+  - name: .rollbackfuzzer
+  - name: sasl
+  - name: search
+  - name: search_auth
+  - name: search_ssl
+  - name: session_jscore_passthrough
+  - name: .sharding .jscore !.wo_snapshot !.multi_stmt
+  - name: sharding_api_version_jscore_passthrough_gen
+  - name: .sharding .txns
+  - name: .sharding .common
+  - name: sharding_opportunistic_secondary_targeting_gen
+  - name: .stitch
+  - name: .crypt
+    distros:
+    - rhel80-xlarge
+  - name: crypt_build_debug_and_test
+    distros:
+    - rhel80-xlarge
+  - name: .updatefuzzer
+  - name: secondary_reads_passthrough_gen
+  - name: server_discovery_and_monitoring_json_test_TG
+  - name: .serverless
+    distros:
+    - rhel80-xlarge
+  - name: server_selection_json_test_TG
+    distros:
+    - rhel80-xlarge
+  - name: generate_buildid_to_debug_symbols_mapping
+
+- name: enterprise-rhel80-dynamic-clang-tidy-required
+  display_name: "! Enterprise Clang Tidy"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+    - enterprise
+  run_on:
+    - rhel80-xxlarge
+  expansions:
+    lang_environment: LANG=C
+    compile_flags: --link-model=dynamic -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_${clang_tidy_toolchain}_clang.vars
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    show_scons_timings: false
+    clang_tidy_toolchain: v3
+    clang_tidy_file: .clang-tidy-extra-checks
+  tasks:
+  - name: clang_tidy_TG
+
+- &rhel80-debug-aubsan-lite-required-template
+  name: rhel80-debug-aubsan-lite-required
+  display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-build
+  stepback: true
+  expansions:
+    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
+    lang_environment: LANG=C
+    # If you add anything to san_options, make sure the appropriate changes are
+    # also made to SConstruct.
+    san_options: >-
+      UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
+      LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1"
+      ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:strict_string_checks=true:detect_invalid_pointer_pairs=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
+    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
+    resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under {A,UB}SAN build.
+    hang_analyzer_dump_core: false
+    scons_cache_scope: shared
+    scons_cache_mode: all
+    max_sub_suites: 3
+    num_scons_link_jobs_available: 0.99
+    separate_debug: off
+    large_distro_name: rhel80-build
+  tasks:
+  - name: compile_test_and_package_parallel_core_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: compile_test_and_package_parallel_unittest_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: compile_test_and_package_parallel_dbtest_stream_TG
+    distros:
+    - rhel80-xlarge
+  - name: jsCore
+  - name: jsCore_txns
+  - name: unittest_shell_hang_analyzer_gen
+  - name: generate_buildid_to_debug_symbols_mapping
+
+- <<: *enterprise-rhel-80-64-bit-dynamic-required-template
+  name: commit-queue
+  display_name: "~ Commit Queue"
+  cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter
+  stepback: false
+  tasks:
+  - name: compile_test_and_package_parallel_core_stream_TG
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: compile_test_and_package_parallel_unittest_stream_TG
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: compile_test_and_package_parallel_dbtest_stream_TG
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: jsCore
+    distros:
+    - rhel80-xlarge-commitqueue
+  - name: .lint
+  - name: test_api_version_compatibility
+  - name: validate_commit_message
+  - name: check_feature_flag_tags

etc/evergreen_timeouts.yml

@@ -35,6 +35,8 @@ overrides:
   enterprise-rhel-80-64-bit-dynamic-required:
   - task: replica_sets_large_txns_format
     exec_timeout: 120  # 2 hours.
+  - task: config_fuzzer_replica_sets_jscore_passthrough
+    exec_timeout: 150  # 2.5 hours.
 
   enterprise-rhel80-debug-tsan:
   - task: run_unittests
@@ -111,8 +113,6 @@ overrides:
   windows:
   - task: replica_sets
     exec_timeout: 180  # 3 hours.
-  - task: replica_sets_jscore_passthrough
-    exec_timeout: 150  # 2.5 hours.
 
   windows-debug-suggested:
   - task: replica_sets_initsync_jscore_passthrough

etc/evergreen_yml_components/definitions.yml

@@ -66,7 +66,7 @@ modules:
 - name: enterprise
   repo: git@github.com:10gen/mongo-enterprise-modules.git
   prefix: src/mongo/db/modules
-  branch: master
+  branch: v6.1
 
 - name: wtdevelop
   repo: git@github.com:wiredtiger/wiredtiger.git
@@ -102,6 +102,19 @@ variables:
     vars:
       resmoke_args: --help
 
+- &gen_burn_in_task_template
+  name: gen_burn_in_task_template
+  depends_on:
+  - name: version_gen
+    variant: generate-tasks-for-version
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+  - name: archive_dist_test_debug
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      resmoke_args: --help
+
 - &benchmark_template
   name: benchmark_template
   depends_on:
@@ -304,7 +317,6 @@ variables:
   - enterprise-windows-debug-unoptimized
   - enterprise-windows-inmem
   - enterprise-windows-required
-  - enterprise-windows-wtdevelop
   - ubuntu1804-debug-asan
   - ubuntu1804-debug-ubsan
   - ubuntu1804-debug-aubsan-lite-required
@@ -578,7 +590,7 @@ functions:
   "get buildnumber": &get_buildnumber
     command: keyval.inc
     params:
-      key: "${build_variant}_master"
+      key: "${build_variant}_v6.1"
       destination: "builder_num"
 
   "run diskstats": &run_diskstats
@@ -1034,11 +1046,45 @@ functions:
       files:
       - src/generated_resmoke_config/*.json
 
+  "generate version burn in":
+  - *f_expansions_write
+  - *configure_evergreen_api_credentials
+  - command: subprocess.exec
+    type: test
+    params:
+      binary: bash
+      args:
+      - "./src/evergreen/generate_version_burn_in.sh"
+  - command: archive.targz_pack
+    params:
+      target: generate_tasks_config.tgz
+      source_dir: src/generated_resmoke_config
+      include:
+      - "*"
+  - command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: generate_tasks_config.tgz
+      remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz
+      bucket: mciuploads
+      permissions: public-read
+      content_type: ${content_type|application/gzip}
+      display_name: Generated Burn In Task Config - Execution ${execution}
+      optional: true
+  - command: generate.tasks
+    params:
+      optional: true
+      files:
+      - src/generated_resmoke_config/*.json
+
   "generate resmoke tasks":
   - *fetch_artifacts
   - *f_expansions_write
   - *kill_processes
   - *cleanup_environment
+  - *get_version_expansions
+  - *apply_version_expansions
   - *fetch_venv
   - *adjust_venv
   - *f_expansions_write
@@ -1179,6 +1225,15 @@ functions:
       args:
       - "src/evergreen/scons_compile.sh"
 
+  "ninja compile":
+  - *f_expansions_write
+  - command: subprocess.exec
+    type: test
+    params:
+      binary: bash
+      args:
+      - "./src/evergreen/ninja_compile.sh"
+
   "generate version expansions": &generate_version_expansions
     command: subprocess.exec
     params:
@@ -2298,16 +2353,88 @@ tasks:
     vars:
       generating_for_ninja: true
       separate_debug: off
-      task_install_action:
-        default
       task_compile_flags: >-
         --ninja
   - *f_expansions_write
-  - command: subprocess.exec
-    params:
-      binary: bash
-      args:
-      - "./src/evergreen/ninja_compile.sh"
+  - func: "ninja compile"
+    vars:
+      ninja_file: "build.ninja"
+
+- name: compile_ninja_default_profile
+  tags: []
+  depends_on:
+  - name: version_expansions_gen
+    variant: generate-tasks-for-version
+  commands:
+  - func: "scons compile"
+    vars:
+      generating_for_ninja: true
+      separate_debug: off
+      task_compile_flags: >-
+        --build-profile=default
+        --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+        --ninja
+  - *f_expansions_write
+  - func: "ninja compile"
+    vars:
+      ninja_file: "build.ninja"
+
+- name: compile_ninja_opt_profile
+  tags: []
+  depends_on:
+  - name: version_expansions_gen
+    variant: generate-tasks-for-version
+  commands:
+  - func: "scons compile"
+    vars:
+      generating_for_ninja: true
+      separate_debug: off
+      task_compile_flags: >-
+        --build-profile=opt
+        CCACHE=
+        ICECC=
+  - *f_expansions_write
+  - func: "ninja compile"
+    vars:
+      ninja_file: "opt.ninja"
+
+- name: compile_ninja_fast_profile
+  tags: []
+  depends_on:
+  - name: version_expansions_gen
+    variant: generate-tasks-for-version
+  commands:
+  - func: "scons compile"
+    vars:
+      generating_for_ninja: true
+      separate_debug: off
+      task_compile_flags: >-
+        --build-profile=fast
+        CCACHE=
+        ICECC=
+  - *f_expansions_write
+  - func: "ninja compile"
+    vars:
+      ninja_file: "fast.ninja"
+
+- name: compile_ninja_san_profile
+  tags: []
+  depends_on:
+  - name: version_expansions_gen
+    variant: generate-tasks-for-version
+  commands:
+  - func: "scons compile"
+    vars:
+      generating_for_ninja: true
+      separate_debug: off
+      task_compile_flags: >-
+        --build-profile=san
+        CCACHE=
+        ICECC=
+  - *f_expansions_write
+  - func: "ninja compile"
+    vars:
+      ninja_file: "san.ninja"
 
 - name: compile_ninja_next
   tags: []
@@ -2319,17 +2446,13 @@ tasks:
     vars:
       generating_for_ninja: true
       separate_debug: off
-      task_install_action:
-        default
       task_compile_flags: >-
         --build-tools=next
         --ninja
   - *f_expansions_write
-  - command: subprocess.exec
-    params:
-      binary: bash
-      args:
-      - "./src/evergreen/ninja_compile.sh"
+  - func: "ninja compile"
+    vars:
+      ninja_file: "build.ninja"
 
 - name: compile_build_tools_next
   tags: []
@@ -2445,6 +2568,8 @@ tasks:
   commands:
   - func: "scons compile"
     vars:
+      task_compile_flags: >-
+        --build-profile=compiledb
       targets: compiledb
       compiling_for_test: true
   - command: subprocess.exec
@@ -2895,7 +3020,7 @@ tasks:
       task_compile_flags: >-
         --allocator=system
         --enable-free-mon=off
-        --enterprise-features=fle
+        --enterprise-features=fle,search
         --js-engine=none
         --link-model=dynamic-sdk
         --enable-http-client=off
@@ -2933,7 +3058,7 @@ tasks:
         --opt=off
         --allocator=system
         --enable-free-mon=off
-        --enterprise-features=fle
+        --enterprise-features=fle,search
         --js-engine=none
         --link-model=dynamic-sdk
         DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION'
@@ -2968,15 +3093,14 @@ tasks:
     vars:
       targets: archive-mongo-crypt-shlib-test
       compiling_for_test: true
-      task_install_action:
-        default
       task_compile_flags: >-
         --allocator=system
         --enable-free-mon=off
-        --enterprise-features=fle
+        --enterprise-features=fle,search
         --js-engine=none
         --link-model=static
         DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION'
+        RPATH='$$RPATH_ESCAPED_DOLLAR_ORIGIN/../lib'
   - command: s3.put
     params:
       aws_key: ${aws_key}
@@ -3169,7 +3293,7 @@ tasks:
       args:
       - "src/evergreen/check_idl_compat.sh"
 
-- <<: *gen_task_template
+- <<: *gen_burn_in_task_template
   name: burn_in_tests_gen
   tags: []
   commands:
@@ -4234,7 +4358,7 @@ tasks:
   commands:
   - func: "generate resmoke tasks"
 
-- <<: *gen_task_template
+- <<: *gen_burn_in_task_template
   name: burn_in_tags_gen
   tags: []
   commands:
@@ -4252,6 +4376,18 @@ tasks:
   - func: "upload pip requirements"
   - func: "generate version"
 
+- name: version_burn_in_gen
+  commands:
+  - command: manifest.load
+  - *git_get_project
+  - *f_expansions_write
+  - *add_git_tag
+  - *kill_processes
+  - *cleanup_environment
+  - func: "set up venv"
+  - func: "upload pip requirements"
+  - func: "generate version burn in"
+
 - name: version_expansions_gen
   commands:
   - command: manifest.load
@@ -6572,6 +6708,7 @@ tasks:
 
 
 - name: publish_packages
+  run_on: rhel80-small
   tags: ["publish"]
   # This should prevent this task from running in patch builds, where we
   # don't want to publish packages.
@@ -6586,6 +6723,7 @@ tasks:
   commands:
   - command: manifest.load
   - func: "git get project and add git tag"
+  - func: "get and apply version expansions"
   - func: "f_expansions_write"
   - func: "kill processes"
   - func: "cleanup environment"
@@ -6606,6 +6744,7 @@ tasks:
       - "./src/evergreen/packages_publish.sh"
 
 - name: push
+  run_on: rhel80-small
   tags: ["publish"]
   patchable: false
   depends_on:
@@ -7085,6 +7224,7 @@ tasks:
          'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}}
 
 - name: crypt_push
+  run_on: rhel80-small
   tags: ["publish_crypt"]
   patchable: false
   stepback: false
@@ -7487,6 +7627,26 @@ task_groups:
       content_type: text/plain
       display_name: build.ninja
 
+- <<: *compile_task_group_template
+  name: compile_ninja_default_profile_TG
+  tasks:
+  - compile_ninja_default_profile
+
+- <<: *compile_task_group_template
+  name: compile_ninja_opt_profile_TG
+  tasks:
+  - compile_ninja_opt_profile
+
+- <<: *compile_task_group_template
+  name: compile_ninja_san_profile_TG
+  tasks:
+  - compile_ninja_san_profile
+
+- <<: *compile_task_group_template
+  name: compile_ninja_fast_profile_TG
+  tasks:
+  - compile_ninja_fast_profile
+
 - <<: *compile_task_group_template
   name: server_discovery_and_monitoring_json_test_TG
   tasks:

etc/evergreen_yml_components/variants/atlas.yml

@@ -74,6 +74,4 @@ buildvariants:
     - ubuntu2004-package
   - name: selinux_rhel7_enterprise
   - name: .publish
-    distros:
-    - rhel70-small
   - name: generate_buildid_to_debug_symbols_mapping

etc/evergreen_yml_components/variants/ibm.yml

@@ -49,11 +49,7 @@ buildvariants:
   - name: .stitch
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - rhel70-small
   - name: .publish
-    distros:
-    - rhel70-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-rhel-72-s390x
@@ -104,11 +100,7 @@ buildvariants:
   - name: .stitch
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - rhel70-small
   - name: .publish
-    distros:
-    - rhel70-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-rhel-83-s390x
@@ -159,9 +151,5 @@ buildvariants:
   - name: .stitch
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - rhel80-small
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping

etc/evergreen_yml_components/variants/misc_release.yml

@@ -159,8 +159,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - amazon2-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-amazon2
@@ -218,8 +216,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - amazon2-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: amazon2-arm64
@@ -268,8 +264,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-amazon2-arm64
@@ -377,8 +371,6 @@ buildvariants:
     distros:
     - amazon2-arm64-large
   - name: .publish_crypt
-    distros:
-    - amazon2-arm64-large
   - name: secondary_reads_passthrough_gen
   - name: server_discovery_and_monitoring_json_test_TG
   - name: .serverless
@@ -391,8 +383,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: debian10
@@ -445,8 +435,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - debian10-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-debian10-64
@@ -500,8 +488,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - debian10-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: debian11
@@ -554,8 +540,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - debian11-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-debian11-64
@@ -609,8 +593,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - debian11-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: rhel70
@@ -662,8 +644,6 @@ buildvariants:
     - ubuntu2004-package
   - name: selinux_rhel7_org
   - name: .publish
-    distros:
-    - rhel70-small
   - name: generate_buildid_to_debug_symbols_mapping
 
   # This variant compiles on RHEL 7.0 and runs tests on RHEL 7.6
@@ -751,8 +731,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-rhel-80-64-bit
@@ -915,8 +893,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-rhel-82-arm64
@@ -974,8 +950,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - rhel80-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: suse12
@@ -1025,8 +999,6 @@ buildvariants:
   - name: .ssl
   - name: .stitch
   - name: .publish
-    distros:
-    - suse12-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-suse12-64
@@ -1069,13 +1041,13 @@ buildvariants:
   - name: sharding_auth_gen
   - name: .stitch
   - name: .crypt
+    distros:
+    - suse12-sp5-large
   - name: .publish_crypt
   - name: test_packages
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - suse12-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-suse15-64
@@ -1118,8 +1090,6 @@ buildvariants:
   - name: .crypt
   - name: .publish_crypt
   - name: .publish
-    distros:
-    - suse15-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: suse15
@@ -1168,8 +1138,6 @@ buildvariants:
   - name: .ssl
   - name: .stitch
   - name: .publish
-    distros:
-    - suse15-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 
@@ -1227,8 +1195,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu1804-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-ubuntu1804-64
@@ -1298,8 +1264,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu1804-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-ubuntu1804-arm64
@@ -1351,14 +1315,10 @@ buildvariants:
   - name: .stitch
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - ubuntu1804-test
   - name: test_packages
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu1804-test
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: ubuntu1804-arm64
@@ -1393,8 +1353,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu1804-test
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: ubuntu2004
@@ -1445,8 +1403,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu2004-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-ubuntu2004-64
@@ -1502,8 +1458,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu2004-small
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: enterprise-ubuntu2004-arm64
@@ -1558,8 +1512,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu2004-test
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: ubuntu2004-arm64
@@ -1593,8 +1545,6 @@ buildvariants:
     distros:
     - ubuntu2004-package
   - name: .publish
-    distros:
-    - ubuntu2004-test
   - name: generate_buildid_to_debug_symbols_mapping
 
 - name: windows
@@ -1646,7 +1596,7 @@ buildvariants:
   - name: .query_fuzzer
   - name: .read_write_concern
   - name: replica_sets_gen
-  - name: replica_sets_jscore_passthrough
+  - name: replica_sets_jscore_passthrough_gen
   - name: .sharding .jscore !.wo_snapshot !.multi_stmt
   - name: .sharding .txns
   - name: .sharding .common !.csrs
@@ -1654,8 +1604,6 @@ buildvariants:
   - name: .stitch
   - name: .updatefuzzer
   - name: push
-    distros:
-    - rhel70-small
 
 - name: enterprise-windows
   display_name: "Enterprise Windows"
@@ -1704,8 +1652,6 @@ buildvariants:
     distros:
     - windows-vsCurrent-large
   - name: .publish_crypt
-    distros:
-    - rhel70-small
   - name: disk_wiredtiger
   - name: .jscore .common !.auth !.feature_flag_guarded
   - name: json_schema
@@ -1724,8 +1670,6 @@ buildvariants:
   - name: .stitch
   - name: .updatefuzzer
   - name: push
-    distros:
-    - rhel70-small
 
   ###########################################
   #           macOS buildvariants           #
@@ -1778,8 +1722,6 @@ buildvariants:
   - name: .stitch
   - name: unittest_shell_hang_analyzer_gen
   - name: push
-    distros:
-    - rhel70-small
 
 - name: macos-arm64
   display_name: macOS arm64
@@ -1829,8 +1771,6 @@ buildvariants:
   - name: .stitch
   - name: unittest_shell_hang_analyzer_gen
   - name: push
-    distros:
-    - rhel70-small
 
 - name: enterprise-macos
   display_name: Enterprise macOS
@@ -1865,12 +1805,8 @@ buildvariants:
   - name: replica_sets_jscore_passthrough
   - name: sasl
   - name: push
-    distros:
-    - rhel70-small
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - rhel70-small
 
 - name: enterprise-macos-arm64
   display_name: Enterprise macOS arm64
@@ -1904,9 +1840,5 @@ buildvariants:
   - name: replica_sets_jscore_passthrough
   - name: sasl
   - name: push
-    distros:
-    - rhel70-small
   - name: .crypt
   - name: .publish_crypt
-    distros:
-    - rhel70-small

etc/evergreen_yml_components/variants/ninja.yml

@@ -0,0 +1,74 @@
+buildvariants:
+
+- name: enterprise-windows-ninja
+  display_name: "Ninja Build: Enterprise Windows"
+  cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
+  modules:
+  - enterprise
+  expansions:
+    compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5")  --win-version-min=win10
+  tasks:
+  - name: compile_ninja_next_TG
+    distros:
+    - windows-vsCurrent-large
+  - name: compile_ninja_TG
+    distros:
+    - windows-vsCurrent-large
+
+- name: macos-enterprise-ninja
+  display_name: "Ninja Build: macOS Enterprise"
+  cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
+  modules:
+  - enterprise
+  run_on:
+  - macos-1100
+  expansions:
+    compile_env: DEVELOPER_DIR=/Applications/Xcode13.app
+    compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars
+  tasks:
+  - name: compile_ninja_next_TG
+  - name: compile_ninja_TG
+
+- name: ubuntu1804-ninja-build-profiles
+  display_name: "Ninja Build Profiles: Ubuntu 18.04"
+  cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to build profiles
+  modules:
+  - enterprise
+  run_on:
+  - ubuntu1804-small
+  stepback: false
+  expansions:
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo)
+    repo_edition: enterprise
+    has_packages: false
+  tasks:
+  - name: compile_ninja_default_profile_TG
+    distros:
+    - ubuntu1804-xlarge
+  - name: compile_ninja_opt_profile_TG
+    distros:
+    - ubuntu1804-xlarge
+  - name: compile_ninja_san_profile_TG
+    distros:
+    - ubuntu1804-xlarge
+  - name: compile_ninja_fast_profile_TG
+    distros:
+    - ubuntu1804-xlarge
+
+- name: enterprise-rhel-80-64-bit-dynamic-required-ninja
+  display_name: "Ninja Build: Enterprise RHEL 8.0"
+  cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
+  modules:
+  - enterprise
+  run_on:
+  - rhel80-small
+  expansions:
+    compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
+    has_packages: false
+  tasks:
+  - name: compile_ninja_next_TG
+    distros:
+    - rhel80-xlarge
+  - name: compile_ninja_TG
+    distros:
+    - rhel80-xlarge

etc/evergreen_yml_components/variants/task_generation.yml

@@ -10,4 +10,5 @@ buildvariants:
   - rhel80-medium
   tasks:
   - name: version_gen
+  - name: version_burn_in_gen
   - name: version_expansions_gen

etc/expansions.default.yml

@@ -37,5 +37,4 @@ install_directory: /data/mongo-install-directory
 num_scons_link_jobs_available: "0.25"
 scons_cache_mode: nolinked
 show_scons_timings: "true"
-task_install_action: hardlink
 separate_debug: "on"

etc/perf.yml

@@ -40,7 +40,6 @@ variables:
     - mongo-tools
     - dsi
     - genny
-    - signal-processing
     - workloads
     - linkbench
     - linkbench2
@@ -63,10 +62,6 @@ modules:
     repo: git@github.com:10gen/genny.git
     prefix: ${workdir}/src
     branch: microbenchmarks-stable
-  - name: signal-processing
-    repo: git@github.com:10gen/signal-processing.git
-    prefix: ${workdir}/src
-    branch: master
   - name: workloads
     repo: git@github.com:10gen/workloads.git
     prefix: ${workdir}/src
@@ -104,7 +99,7 @@ modules:
   - name: enterprise
     repo: git@github.com:10gen/mongo-enterprise-modules.git
     prefix: src/mongo/db/modules
-    branch: master
+    branch: v6.1
   - name: mongo-tools
     repo: git@github.com:mongodb/mongo-tools.git
     prefix: mongo-tools/src/github.com/mongodb
@@ -139,7 +134,6 @@ functions:
         revisions:
           dsi: ${dsi_rev}
           genny: ${genny_rev}
-          signal-processing: ${signal-processing_rev}
           linkbench: ${linkbench_rev}
           linkbench2: ${linkbench2_rev}
           workloads: ${workloads_rev}
@@ -326,7 +320,7 @@ functions:
           set -o errexit
           set -o verbose
           source "${workdir}/compile_venv/bin/activate"
-          python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
+          python ./buildscripts/idl/gen_all_feature_flag_list.py
           mkdir -p mongodb/feature_flags
           cp ./all_feature_flags.txt mongodb/feature_flags
     - command: shell.exec
@@ -336,10 +330,7 @@ functions:
           set -o errexit
           set -o verbose
           source "${workdir}/compile_venv/bin/activate"
-          # TODO: SERVER-68475
-          # remove this when a better solution is found
-          extra_args="SPLIT_DWARF_DWP_FILES=1"
-          python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
+          python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args install-core install-jstestshell SPLIT_DWARF=0 MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
           mkdir -p mongodb/jstests/hooks
           if [ -d jstests/hooks ]
           then
@@ -376,7 +367,6 @@ functions:
         revisions:
           dsi: ${dsi_rev}
           genny: ${genny_rev}
-          signal-processing: ${signal-processing_rev}
           linkbench: ${linkbench_rev}
           linkbench2: ${linkbench2_rev}
           workloads: ${workloads_rev}

etc/pip/components/compile.req

@@ -4,3 +4,4 @@ packaging <= 21.3
 regex <= 2021.11.10
 requirements_parser <= 0.3.1
 setuptools
+ninja>=1.10.0

etc/scons/experimental_unified_ninja.vars

@@ -1,6 +1,9 @@
+# This file is now empty and the settings have been made the default.
+# This file exists only to prevent breakage when used with existing command line invocations.
+
 # Configures the build for building with a unified ninja
 # Each configuration will share a ninja log
 # This allows the output binaries of each configuration to share a common directory
 
-NINJA_BUILDDIR="$BUILD_ROOT/ninja"
-DESTDIR="$BUILD_ROOT/install"
+# NINJA_BUILDDIR="$BUILD_ROOT/ninja"
+# DESTDIR="$BUILD_ROOT/install"

etc/system_perf.yml

@@ -56,7 +56,6 @@ variables:
     # - mongo
     - dsi
     - genny
-    - signal-processing
     - workloads
     - linkbench
     - linkbench2
@@ -82,10 +81,6 @@ modules:
     repo: git@github.com:10gen/genny.git
     prefix: ${workdir}/src
     branch: master
-  - name: signal-processing
-    repo: git@github.com:10gen/signal-processing.git
-    prefix: ${workdir}/src
-    branch: master
   - name: workloads
     repo: git@github.com:10gen/workloads.git
     prefix: ${workdir}/src
@@ -101,8 +96,7 @@ modules:
   - name: tsbs
     repo: git@github.com:gregorynoma/tsbs.git
     prefix: ${workdir}/src
-    branch: master
-    ref: 027f4b2d83267ae69d2740855654b4a7cac72498
+    branch: production
   - name: mongo-perf
     repo: git@github.com:mongodb/mongo-perf.git
     prefix: ${workdir}/src
@@ -127,7 +121,7 @@ modules:
   - name: enterprise
     repo: git@github.com:10gen/mongo-enterprise-modules.git
     prefix: src/mongo/db/modules
-    branch: master
+    branch: v6.1
   - name: mongo-tools
     repo: git@github.com:mongodb/mongo-tools.git
     prefix: mongo-tools/src/github.com/mongodb
@@ -166,7 +160,6 @@ functions:
         revisions:
           dsi: ${dsi_rev}
           genny: ${genny_rev}
-          signal-processing: ${signal-processing_rev}
           linkbench: ${linkbench_rev}
           linkbench2: ${linkbench2_rev}
           tsbs: ${tsbs_rev}
@@ -356,7 +349,7 @@ functions:
           set -o errexit
           set -o verbose
           source "${workdir}/compile_venv/bin/activate"
-          python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
+          python ./buildscripts/idl/gen_all_feature_flag_list.py
           mkdir -p mongodb/feature_flags
           cp ./all_feature_flags.txt mongodb/feature_flags
     - command: shell.exec
@@ -366,10 +359,7 @@ functions:
           set -o errexit
           set -o verbose
           source "${workdir}/compile_venv/bin/activate"
-          # TODO: SERVER-68475
-          # remove this when a better solution is found
-          extra_args="SPLIT_DWARF_DWP_FILES=1"
-          python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
+          python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args install-core install-jstestshell SPLIT_DWARF=0 MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
           mkdir -p mongodb/jstests/hooks
           if [ -d jstests/hooks ]
           then
@@ -406,7 +396,6 @@ functions:
         revisions:
           dsi: ${dsi_rev}
           genny: ${genny_rev}
-          signal-processing: ${signal-processing_rev}
           linkbench: ${linkbench_rev}
           linkbench2: ${linkbench2_rev}
           tsbs: ${tsbs_rev}
@@ -506,43 +495,6 @@ tasks:
             mongo-tools: ${mongo-tools_rev}
       - func: "compile mongodb"
 
-  - name: renew_ssl_cert
-    commands:
-      - command: git.get_project
-        params:
-          directory: *src_dir
-          revisions:
-            dsi: ${dsi_rev}
-      # Run the script to generate ssl cert files
-      - command: shell.exec
-        params:
-          script: AWS_ACCESS_KEY_ID=${terraform_key} AWS_SECRET_ACCESS_KEY=${terraform_secret} ./src/dsi/run-dsi generate_ssl_cert
-      # Upload files for further DSI usage
-      - command: s3.put
-        params:
-          aws_key: ${aws_key}
-          aws_secret: ${aws_secret}
-          local_file: member.pem
-          # path to the remote file is intended to be static
-          remote_file: dsi/ssl/member.pem
-          bucket: mciuploads
-          # the visibility has to be public for consumption by DSI
-          permissions: public-read
-          content_type: text/plain
-          display_name: member.pem
-      - command: s3.put
-        params:
-          aws_key: ${aws_key}
-          aws_secret: ${aws_secret}
-          local_file: root.crt
-          # path to the remote file is intended to be static
-          remote_file: dsi/ssl/root.crt
-          bucket: mciuploads
-          # the visibility has to be public for consumption by DSI
-          permissions: public-read
-          content_type: text/plain
-          display_name: root.crt
-
   - name: linkbench
     priority: 5
     commands:
@@ -957,7 +909,7 @@ tasks:
           test_control: "initialsync-logkeeper"
           mongodb_setup: "initialsync-logkeeper-short"
           # Logkeeper dataset with FCV set to 6.0
-          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.0.tgz"
+          mongodb_dataset: "https://dsi-donot-remove.s3-us-west-2.amazonaws.com/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.1.tgz"
 
   - name: initialsync-logkeeper-short-fcbis
     priority: 5
@@ -967,7 +919,7 @@ tasks:
           test_control: "initialsync-logkeeper"
           mongodb_setup: "initialsync-logkeeper-short-fcbis"
           # Logkeeper dataset with FCV set to 6.0
-          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.0.tgz"
+          mongodb_dataset: "https://dsi-donot-remove.s3-us-west-2.amazonaws.com/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.1.tgz"
 
   - name: initialsync-logkeeper
     priority: 5
@@ -999,7 +951,7 @@ tasks:
           test_control: "initialsync-logkeeper-short-s3-update"
           mongodb_setup: "initialsync-logkeeper-short-s3-update"
           # Update this to Logkeeper dataset with FCV set to latest after each LTS release.
-          mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.0.tgz"
+          mongodb_dataset: "https://dsi-donot-remove.s3-us-west-2.amazonaws.com/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.1.tgz"
 
   - name: initialsync-logkeeper-snapshot-update
     priority: 5
@@ -1171,41 +1123,6 @@ buildvariants:
       - name: tpch_10_normalized
       - name: tpch_10_denormalized
 
-  - name: linux-standalone-all-feature-flags
-    display_name: Linux Standalone (all feature flags)
-    cron: "0 0 * * *" # Everyday at 00:00
-    modules: *modules
-    expansions:
-      mongodb_setup: standalone-all-feature-flags
-      infrastructure_provisioning: single
-      platform: linux
-      project_dir: *project_dir
-      authentication: enabled
-      storageEngine: wiredTiger
-    run_on:
-      - "rhel70-perf-single"
-    depends_on: *_compile_amazon2
-    tasks:
-      - name: schedule_patch_auto_tasks
-      - name: schedule_variant_auto_tasks
-      - name: industry_benchmarks
-      - name: ycsb_60GB
-      - name: ycsb_60GB.long
-      - name: crud_workloads
-      - name: bestbuy_agg
-      - name: bestbuy_agg_merge_different_db
-      - name: bestbuy_agg_merge_same_db
-      - name: bestbuy_agg_merge_wordcount
-      - name: bestbuy_query
-      - name: cursor_manager
-      - name: map_reduce_workloads
-      - name: tpcc
-      - name: tpch_1_normalized
-      - name: tpch_1_denormalized
-      - name: tpch_10_normalized
-      - name: tpch_10_denormalized
-      - name: column_store_index_charts_events_1G
-
   - name: linux-standalone-classic-query-engine
     display_name: Linux Standalone (Classic Query Engine)
     cron: "0 0 * * 4" # 00:00 on Thursday
@@ -1658,26 +1575,6 @@ buildvariants:
       - name: industry_benchmarks
       - name: linkbench
 
-  - name: linux-shard-lite-all-feature-flags
-    display_name: Linux Shard Lite (all feature flags)
-    cron: "0 0 * * 4" # 00:00 on Thursday
-    modules: *modules
-    expansions:
-      mongodb_setup: shard-lite-all-feature-flags
-      infrastructure_provisioning: shard-lite
-      platform: linux
-      project_dir: *project_dir
-      authentication: enabled
-      storageEngine: wiredTiger
-    run_on:
-      - "rhel70-perf-shard-lite"
-    depends_on: *_compile_amazon2
-    tasks:
-      - name: schedule_patch_auto_tasks
-      - name: schedule_variant_auto_tasks
-      - name: change_streams_preimage_throughput
-      - name: change_streams_preimage_latency
-
   - name: linux-shard-single
     display_name: Linux Shard Single
     cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday
@@ -1869,67 +1766,6 @@ buildvariants:
       - name: sb_large_scale
       - name: sb_timeseries
 
-  # Note that the "disabled-feature-flags" part of the name is kept to avoid breaking
-  # history even though the display name is "all feature flags"
-  - name: linux-3-node-replSet-disabled-feature-flags
-    display_name: Linux 3-Node ReplSet (all feature flags)
-    cron: "0 0 * * 4" # 00:00 on Thursday
-    modules: *modules
-    expansions:
-      mongodb_setup: replica-all-feature-flags
-      infrastructure_provisioning: replica
-      platform: linux
-      project_dir: *project_dir
-      authentication: enabled
-      storageEngine: wiredTiger
-    run_on:
-      - "rhel70-perf-replset"
-    depends_on: *_compile_amazon2
-    tasks:
-      - name: schedule_patch_auto_tasks
-      - name: schedule_variant_auto_tasks
-      - name: industry_benchmarks
-      - name: ycsb_60GB
-      - name: ycsb_60GB.long
-      - name: industry_benchmarks_secondary_reads
-      - name: crud_workloads
-      - name: crud_workloads_majority
-      - name: mixed_workloads
-      - name: misc_workloads
-      - name: map_reduce_workloads
-      - name: refine_shard_key_transaction_stress
-      - name: smoke_test
-      - name: retryable_writes_workloads
-      - name: industry_benchmarks_wmajority
-      - name: secondary_performance # Uses a special 2 node mongodb setup
-      - name: non_sharded_workloads
-      - name: bestbuy_agg
-      - name: bestbuy_agg_merge_different_db
-      - name: bestbuy_agg_merge_same_db
-      - name: bestbuy_agg_merge_wordcount
-      - name: bestbuy_query
-      - name: change_streams_throughput
-      - name: change_streams_latency
-      - name: change_streams_listen_throughput
-      - name: change_streams_preimage_throughput
-      - name: change_streams_preimage_latency
-      - name: snapshot_reads
-      - name: secondary_reads
-      - name: tpcc
-      - name: tpch_1_normalized
-      - name: tpch_1_denormalized
-      # TODO: Enable in SERVER-66572.
-      # - name: tpch_10_normalized
-      # - name: tpch_10_denormalized
-      - name: linkbench
-      - name: linkbench2
-      # Time-series collections are available since v5.0.
-      # - name: tsbs_load
-      # - name: tsbs_query
-      # - name: tsbs_query_manual_bucketing
-      - name: sb_large_scale
-      - name: sb_timeseries
-
   - name: linux-3-node-replSet-noflowcontrol
     display_name: Linux 3-Node ReplSet (Flow Control off)
     cron: "0 0 * * 4" # 00:00 on Thursday
@@ -2041,7 +1877,7 @@ buildvariants:
       mongodb_setup: initialsync-logkeeper
       infrastructure_provisioning: initialsync-logkeeper
       # EBS logkeeper snapshot with FCV set to 6.0
-      snapshotId: snap-0306dce35f030ebec
+      snapshotId: snap-0716ed59d18225693
       platform: linux
       authentication: disabled
       storageEngine: wiredTiger
@@ -2065,7 +1901,7 @@ buildvariants:
   #    mongodb_setup: initialsync-logkeeper-snapshot-update
   #    infrastructure_provisioning: initialsync-logkeeper-snapshot-update
   #    # Update this to latest snapshot after each LTS release.
-  #    snapshotId: snap-0306dce35f030ebec
+  #    snapshotId: snap-0716ed59d18225693
   #    platform: linux
   #    authentication: disabled
   #    storageEngine: wiredTiger
@@ -2160,12 +1996,3 @@ buildvariants:
       - name: tpcc
       - name: linkbench
       - name: linkbench2
-
-  - name: renew-ssl-cert
-    display_name: Renew SSL Cert
-    cron: "0 0 * * 4" # 00:00 on Thursday
-    modules: *modules
-    run_on:  # Certbot with route53 plugin is installed on Ubuntu 20.04
-      - "ubuntu2004-small"
-    tasks:
-      - name: renew_ssl_cert

evergreen/feature_flag_tags_check.sh

@@ -12,7 +12,7 @@ enterprise_path="src/mongo/db/modules/enterprise"
 diff_file_name="with_base_upstream.diff"
 
 # get the list of feature flags from the patched version
-$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src
+$python buildscripts/idl/gen_all_feature_flag_list.py
 mv all_feature_flags.txt patch_all_feature_flags.txt
 
 # get the list of feature flags from the base commit
@@ -29,7 +29,7 @@ if [ -s "$diff_file_name" ]; then
 fi
 popd
 
-$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src
+$python buildscripts/idl/gen_all_feature_flag_list.py
 mv all_feature_flags.txt base_all_feature_flags.txt
 
 # print out the list of tests that previously had feature flag tag, that was

evergreen/gen_feature_flags.sh

@@ -6,4 +6,4 @@ cd src
 set -o errexit
 set -o verbose
 activate_venv
-$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
+$python buildscripts/idl/gen_all_feature_flag_list.py

evergreen/generate_version.sh

@@ -6,25 +6,11 @@ cd src
 set -o errexit
 set -o verbose
 
-curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.5.3/mongo-task-generator --output mongo-task-generator
-chmod +x mongo-task-generator
-
-## Comment above and uncomment below to test unreleased mongo-task-generator changes pushed to <branch-name>
-#curl https://sh.rustup.rs -sSf | sh -s -- -y
-#source "$HOME/.cargo/env"
-#git clone git@github.com:mongodb/mongo-task-generator.git unreleased-mongo-task-generator
-#pushd unreleased-mongo-task-generator
-#git checkout <branch-name>
-#cargo build --release --locked
-#generator_path="$(pwd)/target/release/mongo-task-generator"
-#popd
-#cp "$generator_path" mongo-task-generator
-
+setup_mongo_task_generator
 activate_venv
 PATH=$PATH:$HOME:/ ./mongo-task-generator \
   --expansion-file ../expansions.yml \
   --evg-auth-file ./.evergreen.yml \
   --evg-project-file ${evergreen_config_file_path} \
   --generate-sub-tasks-config etc/generate_subtasks_config.yml \
-  --burn-in \
   $@

evergreen/generate_version_burn_in.sh

@@ -0,0 +1,17 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+setup_mongo_task_generator
+activate_venv
+PATH=$PATH:$HOME:/ ./mongo-task-generator \
+  --expansion-file ../expansions.yml \
+  --evg-auth-file ./.evergreen.yml \
+  --evg-project-file ${evergreen_config_file_path} \
+  --generate-sub-tasks-config etc/generate_subtasks_config.yml \
+  --burn-in \
+  $@

evergreen/jstestfuzz_minimize.sh

@@ -6,8 +6,6 @@ cd src/jstestfuzz
 set -o errexit
 set -o verbose
 
-add_nodejs_to_path
-
 if [ -f "../minimizer-outputs.json" ]; then
-  eval npm run ${npm_command} -- -j "../minimizer-outputs.json"
+  eval ./src/scripts/npm_run.sh ${npm_command} -- -j "../minimizer-outputs.json"
 fi

evergreen/jstestfuzz_run.sh

@@ -1,13 +1,10 @@
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$DIR/prelude.sh"
 
+set -evo pipefail
+
 cd src/jstestfuzz
 
-set -o errexit
-set -o verbose
-
-add_nodejs_to_path
-
 in_patch_build_flag=""
 if [[ "${is_patch}" = "true" ]]; then
   case "${npm_command}" in
@@ -17,4 +14,4 @@ if [[ "${is_patch}" = "true" ]]; then
   esac
 fi
 
-eval npm run "${npm_command}" -- "${jstestfuzz_vars}" "${in_patch_build_flag}" --branch "${branch_name}"
+./src/scripts/npm_run.sh ${npm_command} -- ${jstestfuzz_vars} ${in_patch_build_flag} --branch ${branch_name}

evergreen/jstestfuzz_setup.sh

@@ -6,11 +6,4 @@ cd src
 set -o errexit
 set -o verbose
 
-add_nodejs_to_path
-
 git clone git@github.com:10gen/jstestfuzz.git
-
-pushd jstestfuzz
-npm install
-npm run prepare
-popd

evergreen/lint_fuzzer_sanity_all.sh

@@ -1,17 +1,22 @@
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$DIR/prelude.sh"
 
-cd src
+cd src/jstestfuzz
 
 set -o pipefail
 set -o verbose
 
-add_nodejs_to_path
-
+# Run first with help which will do the install
+# Then we can run it in parallel
+./src/scripts/npm_run.sh --help
 # Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
-find "$PWD/jstests" "$PWD/src/mongo/db/modules/enterprise" -name "*.js" -print | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles -- | tee lint_fuzzer_sanity.log
+# Skip javascript files in third_party directory
+find "$PWD/../jstests" "$PWD/../src/mongo/db/modules/enterprise" -path "$PWD/../jstests/third_party" -prune -o -name "*.js" -print | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- | tee lint_fuzzer_sanity.log
 exit_code=$?
 
+# Exit out of the jstestfuzz directory
+cd ..
+
 activate_venv
-$python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_all --log-file lint_fuzzer_sanity.log --exit-code $exit_code
+$python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_all --log-file jstestfuzz/lint_fuzzer_sanity.log --exit-code $exit_code
 exit $exit_code

evergreen/lint_fuzzer_sanity_patch.sh

@@ -7,15 +7,17 @@ set -o pipefail
 set -o verbose
 
 activate_venv
-add_nodejs_to_path
 
 mkdir -p jstestfuzzinput jstestfuzzoutput
 
-indir="$(pwd)/jstestfuzzinput"
-outdir="$(pwd)/jstestfuzzoutput"
+# We need to be the jstestfuzz repo for node to install/run
+cd jstestfuzz
+
+indir="$(pwd)/../jstestfuzzinput"
+outdir="$(pwd)/../jstestfuzzoutput"
 
 # Grep all the js files from modified_and_created_patch_files.txt and put them into $indir.
-(grep -v "\.tpl\.js$" modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true)
+(grep -v "\.tpl\.js$" ../modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true)
 
 # Count the number of files in $indir.
 if [[ "$(ls -A $indir)" ]]; then
@@ -26,10 +28,13 @@ if [[ "$(ls -A $indir)" ]]; then
     num_files=50
   fi
 
-  npm run --prefix jstestfuzz jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50
+  ./src/scripts/npm_run.sh jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50
 
   # Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
-  ls -1 -d $outdir/* | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles -- | tee lint_fuzzer_sanity.log
+  ls -1 -d $outdir/* | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- | tee lint_fuzzer_sanity.log
   exit_code=$?
-  $python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_patch --log-file lint_fuzzer_sanity.log --exit-code $exit_code
+
+  # Exit out of the jstestfuzz directory
+  cd ..
+  $python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_patch --log-file jstestfuzz/lint_fuzzer_sanity.log --exit-code $exit_code
 fi

evergreen/ninja_compile.sh

@@ -5,6 +5,7 @@ cd src
 
 set -o errexit
 set -o verbose
+
 activate_venv
 python -m pip install ninja
 if [ "Windows_NT" = "$OS" ]; then
@@ -13,8 +14,8 @@ if [ "Windows_NT" = "$OS" ]; then
   for i in "${compile_env[@]}"; do
     echo "set $i" >> msvc.bat
   done
-  echo "ninja install-core" >> msvc.bat
+  echo "ninja -f ${ninja_file} install-core" >> msvc.bat
   cmd /C msvc.bat
 else
-  eval ${compile_env} ninja install-core
+  eval ${compile_env} ninja -f ${ninja_file} install-core
 fi

evergreen/prelude.sh

@@ -11,6 +11,7 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$evergreen_dir/prelude_python.sh"
 . "$evergreen_dir/prelude_venv.sh"
 . "$evergreen_dir/prelude_db_contrib_tool.sh"
+. "$evergreen_dir/prelude_mongo_task_generator.sh"
 
 expansions_yaml="$evergreen_dir/../../expansions.yml"
 expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
@@ -31,21 +32,6 @@ unset expansions_default_yaml
 unset script
 unset evergreen_dir
 
-function add_nodejs_to_path {
-  # Add node and npm binaries to PATH
-  if [ "Windows_NT" = "$OS" ]; then
-    # An "npm" directory might not have been created in %APPDATA% by the Windows installer.
-    # Work around the issue by specifying a different %APPDATA% path.
-    # See: https://github.com/nodejs/node-v0.x-archive/issues/8141
-    export APPDATA=${workdir}/npm-app-data
-    export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location
-    # TODO: this is to work around BUILD-8652
-    cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')"
-  else
-    export PATH="$PATH:/opt/node/bin"
-  fi
-}
-
 function posix_workdir {
   if [ "Windows_NT" = "$OS" ]; then
     echo $(cygpath -u "${workdir}")

evergreen/prelude_mongo_task_generator.sh

@@ -0,0 +1,24 @@
+function setup_mongo_task_generator {
+  if [ ! -f mongo-task-generator ]; then
+    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.6.2/mongo-task-generator --output mongo-task-generator
+    chmod +x mongo-task-generator
+  fi
+}
+
+## Comment above and uncomment below to test unreleased mongo-task-generator changes that are
+## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git`
+## repo
+#function setup_mongo_task_generator {
+#  if [ ! -f mongo-task-generator ]; then
+#
+#    curl https://sh.rustup.rs -sSf | sh -s -- -y
+#    source "$HOME/.cargo/env"
+#    git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator
+#    pushd unreleased-mongo-task-generator
+#    git checkout <branch-name>
+#    cargo build --release --locked
+#    generator_path="$(pwd)/target/release/mongo-task-generator"
+#    popd
+#    cp "$generator_path" mongo-task-generator
+#  fi
+#}

evergreen/resmoke_tests_runtime_validate.sh

@@ -9,7 +9,6 @@ set -o errexit
 activate_venv
 $python buildscripts/resmoke_tests_runtime_validate.py \
   --resmoke-report-file ./report.json \
-  --evg-api-config ./.evergreen.yml \
   --project-id ${project_id} \
   --build-variant ${build_variant} \
   --task-name ${task_name}

evergreen/scons_compile.sh

@@ -8,15 +8,6 @@ set -o verbose
 
 rm -rf ${install_directory}
 
-# Use hardlinks to reduce the disk space impact of installing
-# all of the binaries and associated debug info.
-
-# The expansion here is a workaround to let us set a different install-action
-# for tasks that don't support the one we set here. A better plan would be
-# to support install-action for Ninja builds directly.
-# TODO: https://jira.mongodb.org/browse/SERVER-48203
-extra_args="--install-action=${task_install_action}"
-
 # By default, limit link jobs to one quarter of our overall -j
 # concurrency unless locally overridden. We do this because in
 # static link environments, the memory consumption of each
@@ -58,9 +49,7 @@ else
   extra_args="$extra_args --release"
 fi
 
-# TODO: SERVER-68475
-# remove this when a better solution is found
-extra_args="$extra_args SPLIT_DWARF_DWP_FILES=1"
+extra_args="$extra_args SPLIT_DWARF=0"
 
 if [ "${generating_for_ninja}" = "true" ] && [ "Windows_NT" = "$OS" ]; then
   vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"

jstests/aggregation/expressions/switch_errors.js

@@ -60,9 +60,26 @@ pipeline = {
 };
 assertErrorCode(coll, pipeline, 40068, "$switch requires at least one branch");
 
-coll.insert({x: 1});
+assert.commandWorked(coll.insert({x: 1}));
 pipeline = {
     "$project": {"output": {"$switch": {"branches": [{"case": {"$eq": ["$x", 0]}, "then": 1}]}}}
 };
 assertErrorCode(coll, pipeline, 40066, "$switch has no default and an input matched no case");
+
+// This query was designed to reproduce SERVER-70190. The first branch of the $switch can be
+// optimized away and the $ifNull can be optimized to 2. If the field "x" exists in the input
+// document and is truthy, then the expression should return 2. Otherwise it should throw because no
+// case statement matched and there is no "default" expression.
+pipeline = [{
+    $sortByCount: {
+        $switch: {
+            branches:
+                [{case: {$literal: false}, then: 1}, {case: "$x", then: {$ifNull: [2, "$y"]}}]
+        }
+    }
+}];
+assert.eq([{"_id": 2, "count": 1}], coll.aggregate(pipeline).toArray());
+assert.commandWorked(coll.remove({x: 1}));
+assert.commandWorked(coll.insert({z: 1}));
+assertErrorCode(coll, pipeline, 40066, "$switch has no default and an input matched no case");
 }());

jstests/aggregation/sources/multiple_unpack_bucket_error.js

@@ -18,7 +18,7 @@ assert.commandFailedWithCode(db.runCommand({
     ],
     cursor: {}
 }),
-                             5348302);
+                             5348304);
 
 // $_unpackBucket is an alias of $_internalUnpackBucket, the same restriction should apply.
 assert.commandFailedWithCode(db.runCommand({
@@ -29,7 +29,7 @@ assert.commandFailedWithCode(db.runCommand({
     ],
     cursor: {}
 }),
-                             5348302);
+                             5348304);
 assert.commandFailedWithCode(db.runCommand({
     aggregate: coll.getName(),
     pipeline: [
@@ -38,11 +38,11 @@ assert.commandFailedWithCode(db.runCommand({
     ],
     cursor: {}
 }),
-                             5348302);
+                             5348304);
 assert.commandFailedWithCode(db.runCommand({
     aggregate: coll.getName(),
     pipeline: [{$_unpackBucket: {timeField: 'time'}}, {$_unpackBucket: {timeField: 'time'}}],
     cursor: {}
 }),
-                             5348302);
+                             5348304);
 })();

jstests/aggregation/sources/setWindowFields/derivative.js

@@ -192,9 +192,9 @@ function explainUnit(unit) {
     return coll.runCommand(
         {explain: {aggregate: coll.getName(), cursor: {}, pipeline: [derivativeStage(unit)]}});
 }
-assert.commandFailedWithCode(explainUnit('year'), 5490704);
-assert.commandFailedWithCode(explainUnit('quarter'), 5490704);
-assert.commandFailedWithCode(explainUnit('month'), 5490704);
+assert.commandFailedWithCode(explainUnit('year'), 5490710);
+assert.commandFailedWithCode(explainUnit('quarter'), 5490710);
+assert.commandFailedWithCode(explainUnit('month'), 5490710);
 assert.commandWorked(explainUnit('week'));
 assert.commandWorked(explainUnit('day'));
 assert.commandWorked(explainUnit('hour'));

jstests/aggregation/sources/setWindowFields/integral.js

@@ -115,9 +115,9 @@ function explainUnit(unit) {
         }
     });
 }
-assert.commandFailedWithCode(explainUnit('year'), 5490704);
-assert.commandFailedWithCode(explainUnit('quarter'), 5490704);
-assert.commandFailedWithCode(explainUnit('month'), 5490704);
+assert.commandFailedWithCode(explainUnit('year'), 5490710);
+assert.commandFailedWithCode(explainUnit('quarter'), 5490710);
+assert.commandFailedWithCode(explainUnit('month'), 5490710);
 assert.commandWorked(explainUnit('week'));
 assert.commandWorked(explainUnit('day'));
 assert.commandWorked(explainUnit('hour'));

jstests/aggregation/sources/setWindowFields/output_overwrites_existing_data.js

@@ -9,8 +9,6 @@
 (function() {
 "use strict";
 
-// TODO SERVER-63811 Ensure the database exists so we get back non-empty results even in a sharded
-// cluster.
 assert.commandWorked(db[jsTestName()].insert({dummy: 1}));
 
 let windowResults = db.aggregate([

jstests/change_streams/oplog_rewrite/change_stream_match_pushdown_namespace_rewrite.js

@@ -1084,5 +1084,29 @@ verifyOnWholeCluster(
     {change_stream_match_pushdown_and_rewrite_and_rewrite: {dropDatabase: [dbName, dbName]}},
     1 /* expectedOplogRetDocsForEachShard */);
 
+// Create two sharded collections in the main test database, then start a new change stream to get a
+// fresh resume token.
+const collWithDot =
+    createShardedCollection(st, "_id" /* shardKey */, dbName, "foo.bar", 2 /*splitAt */);
+const collWithUnderscore =
+    createShardedCollection(st, "_id" /* shardKey */, dbName, "foo_bar", 2 /*splitAt */);
+const thirdResumeAfterToken =
+    db.getSiblingDB("admin").watch([], {allChangesForCluster: true}).getResumeToken();
+
+// Insert one document per collection, per shard. The test cases below verify the behavior of regex
+// matches with escaped characters on collections with special names (e.g. containing dots). This
+// exercises the fix for SERVER-67715.
+assert.commandWorked(collWithDot.insert({_id: 1}));
+assert.commandWorked(collWithDot.insert({_id: 3}));
+assert.commandWorked(collWithUnderscore.insert({_id: 1}));
+assert.commandWorked(collWithUnderscore.insert({_id: 3}));
+
+// Ensure that a regex match properly respects escaped characters (here, testing that the escaped
+// "." character is treated as a literal dot).
+verifyOnWholeCluster(thirdResumeAfterToken,
+                     {$match: {"ns.coll": {$nin: [/^foo\./]}}},
+                     {"foo_bar": {insert: [1, 3]}},
+                     1 /* expectedOplogRetDocsForEachShard */);
+
 st.stop();
 })();

jstests/change_streams/oplog_rewrite/match_pushdown_namespace_rewrite_with_expanded_events.js

@@ -816,5 +816,48 @@ verifyOnWholeCluster(
     },
     [9, 3] /* expectedOplogRetDocsForEachShard */);
 
+// Create a new change stream and resume token for replaying the stream after this point.
+const thirdResumeAfterToken =
+    db.getSiblingDB("admin").watch([], {allChangesForCluster: true}).getResumeToken();
+
+// The test cases below verify the behavior of regex matches with escaped characters on collections
+// with special names (e.g. containing dots). This exercises the fix for SERVER-67715.
+const collWithDot =
+    createShardedCollection(st, "_id" /* shardKey */, dbName, "foo.bar", 2 /*splitAt */);
+assert.commandWorked(collWithDot.createIndex({x: 1}));
+assert.commandWorked(collWithDot.insert({_id: 1}));
+assert.commandWorked(collWithDot.insert({_id: 3}));
+assert.commandWorked(
+    collWithDot.runCommand({collMod: "foo.bar", index: {keyPattern: {x: 1}, hidden: true}}));
+assert.commandWorked(collWithDot.runCommand({dropIndexes: "foo.bar", index: {x: 1}}));
+
+const collWithUnderscore =
+    createShardedCollection(st, "_id" /* shardKey */, dbName, "foo_bar", 2 /*splitAt */);
+assert.commandWorked(collWithUnderscore.createIndex({x: 1}));
+assert.commandWorked(collWithUnderscore.insert({_id: 1}));
+assert.commandWorked(collWithUnderscore.insert({_id: 3}));
+assert.commandWorked(
+    collWithUnderscore.runCommand({collMod: "foo_bar", index: {keyPattern: {x: 1}, hidden: true}}));
+assert.commandWorked(collWithUnderscore.runCommand({dropIndexes: "foo_bar", index: {x: 1}}));
+
+// Ensure that a regex match properly respects escaped characters (here, testing that the escaped
+// "." character is treated as a literal dot). Note that we expect 5 extra oplog entries on shard0:
+//  - 1 from the "create" event (which only appears on shard0)
+//  - 4 no-op entries from sharding the two collections that are not affected by the filter pushdown
+//    (2 "shardCollection" + 2 "migrateChunkToNewShard")
+verifyOnWholeCluster(thirdResumeAfterToken,
+                     {$match: {"ns.coll": {$nin: [/^foo\./]}}},
+                     {
+                         "foo_bar": {
+                             create: ["foo_bar"],
+                             shardCollection: ["foo_bar"],
+                             createIndexes: ["foo_bar", "foo_bar"],
+                             insert: [1, 3],
+                             modify: ["foo_bar", "foo_bar"],
+                             dropIndexes: ["foo_bar", "foo_bar"]
+                         }
+                     },
+                     [9, 4] /* expectedOplogRetDocsForEachShard */);
+
 st.stop();
 })();

jstests/concurrency/fsm_workloads/collection_defragmentation.js

@@ -227,6 +227,15 @@ var $config = (function() {
 
     function teardown(db, collName, cluster) {
         const mongos = cluster.getDB('config').getMongo();
+
+        cluster.executeOnConfigNodes((db) => {
+            assert.commandWorked(db.adminCommand({
+                configureFailPoint: 'overrideBalanceRoundInterval',
+                mode: 'alwaysOn',
+                data: {intervalMs: 100}
+            }));
+        });
+
         for (let i = 0; i < dbCount; i++) {
             const dbName = dbPrefix + i;
             for (let j = 0; j < collCount; j++) {
@@ -236,11 +245,7 @@ var $config = (function() {
                 // Enable balancing and wait for balanced
                 assertAlways.commandWorked(mongos.getDB('config').collections.update(
                     {_id: fullNs}, {$set: {"noBalance": false}}));
-                assertAlways.soon(function() {
-                    let res = mongos.adminCommand({balancerCollectionStatus: fullNs});
-                    assertAlways.commandWorked(res);
-                    return res.balancerCompliant;
-                });
+                sh.awaitCollectionBalance(mongos.getCollection(fullNs));
                 // Begin defragmentation again
                 assertAlways.commandWorked(mongos.adminCommand({
                     configureCollectionBalancing: fullNs,

jstests/concurrency/fsm_workloads/insert_with_data_size_aware_balancing.js

@@ -9,8 +9,6 @@
  *   requires_sharding,
  *   assumes_balancer_on,
  *   featureFlagBalanceAccordingToDataSize,
- *   # TODO SERVER-67813 review tag when data size aware balancing lands in v6.0
- *   requires_fcv_61,
  *   antithesis_incompatible,
  *  ]
  */

jstests/core/collmod_convert_to_ttl.js

@@ -39,13 +39,6 @@ assert.commandFailedWithCode(
     db.runCommand({"collMod": collName, "index": {"keyPattern": {a: 1}, "expireAfterSeconds": -1}}),
     ErrorCodes.InvalidOptions);
 
-// Tries to modify with an 'expireAfterSeconds' value too large.
-assert.commandFailedWithCode(db.runCommand({
-    "collMod": collName,
-    "index": {"keyPattern": {a: 1}, "expireAfterSeconds": 10000000000000}
-}),
-                             ErrorCodes.InvalidOptions);
-
 // Successfully converts to a TTL index.
 assert.commandWorked(db.runCommand(
     {"collMod": collName, "index": {"keyPattern": {a: 1}, "expireAfterSeconds": 100}}));

jstests/core/cover_null_queries.js

@@ -3,7 +3,9 @@
  * @tags: [
  *   assumes_unsharded_collection,
  *   requires_non_retryable_writes,
- *   requires_fcv_52
+ *   requires_fcv_60,
+ *   # This test could produce unexpected explain output if additional indexes are created.
+ *   assumes_no_implicit_index_creation,
  * ]
  */
 (function() {
@@ -57,12 +59,22 @@ function validateStages({cmdObj, expectedStages, isAgg}) {
  */
 function validateFindCmdOutputAndPlan({filter, projection, expectedStages, expectedOutput}) {
     const cmdObj = {find: coll.getName(), filter: filter, projection: projection};
+
+    // Compare index output with expected output.
     if (expectedOutput) {
         const res = assert.commandWorked(coll.runCommand(cmdObj));
         const ouputArray = new DBCommandCursor(coll.getDB(), res).toArray();
         assert(arrayEq(expectedOutput, ouputArray), ouputArray);
     }
+
+    // Validate explain.
     validateStages({cmdObj, expectedStages});
+
+    // Verify that we get the same output as we expect without an index.
+    const noIndexCmdObj = Object.assign(cmdObj, {hint: {$natural: 1}});
+    const resNoIndex = assert.commandWorked(coll.runCommand(noIndexCmdObj));
+    const noIndexOutArr = new DBCommandCursor(coll.getDB(), resNoIndex).toArray();
+    assert(arrayEq(expectedOutput, noIndexOutArr), noIndexOutArr);
 }
 
 /**
@@ -71,10 +83,18 @@ function validateFindCmdOutputAndPlan({filter, projection, expectedStages, expec
  * are present in the plan returned.
  */
 function validateSimpleCountCmdOutputAndPlan({filter, expectedStages, expectedCount}) {
+    // Compare index output with expected output.
     const cmdObj = {count: coll.getName(), query: filter};
     const res = assert.commandWorked(coll.runCommand(cmdObj));
     assert.eq(res.n, expectedCount);
+
+    // Validate explain.
     validateStages({cmdObj, expectedStages});
+
+    // Verify that we get the same output with and without an index.
+    const noIndexCmdObj = Object.assign(cmdObj, {hint: {$natural: 1}});
+    const resNoIndex = assert.commandWorked(coll.runCommand(noIndexCmdObj));
+    assert.eq(resNoIndex.n, expectedCount);
 }
 
 /**
@@ -88,11 +108,33 @@ function validateCountAggCmdOutputAndPlan({filter, expectedStages, expectedCount
         pipeline: pipeline || [{$match: filter}, {$count: "count"}],
         cursor: {},
     };
+
+    // Compare index output with expected output.
     const cmdRes = assert.commandWorked(coll.runCommand(cmdObj));
     const countRes = cmdRes.cursor.firstBatch;
     assert.eq(countRes.length, 1, cmdRes);
     assert.eq(countRes[0].count, expectedCount, countRes);
+
+    // Validate explain.
     validateStages({cmdObj, expectedStages, isAgg: true});
+
+    // Verify that we get the same output as we expect without an index.
+    const noIndexCmdObj = Object.assign(cmdObj, {hint: {$natural: 1}});
+    const resNoIndex = assert.commandWorked(coll.runCommand(noIndexCmdObj));
+    const countResNoIndex = resNoIndex.cursor.firstBatch;
+    assert.eq(countResNoIndex.length, 1, cmdRes);
+    assert.eq(countResNoIndex[0].count, expectedCount, countRes);
+}
+
+/**
+ * Same as above, but uses a $group count.
+ */
+function validateGroupCountAggCmdOutputAndPlan({filter, expectedStages, expectedCount}) {
+    validateCountAggCmdOutputAndPlan({
+        expectedStages,
+        expectedCount,
+        pipeline: [{$match: filter}, {$group: {_id: 0, count: {$count: {}}}}]
+    });
 }
 
 assert.commandWorked(coll.createIndex({a: 1, _id: 1}));
@@ -152,7 +194,7 @@ validateFindCmdOutputAndPlan({
     expectedStages: {"IXSCAN": 1, "FETCH": 0, "PROJECTION_COVERED": 1},
 });
 
-// Same as above, but special case for null and empty array predicate.
+// We can cover a $in with null and an empty array predicate.
 validateFindCmdOutputAndPlan({
     filter: {a: {$in: [null, []]}},
     projection: {_id: 1},
@@ -165,6 +207,21 @@ validateSimpleCountCmdOutputAndPlan({
     expectedStages: {"IXSCAN": 1, "FETCH": 0},
 });
 
+// We cannot cover a $in with null and an array predicate.
+// TODO SERVER-71058: It should be possible to cover this case and the more general case of matching
+// an array on a non-multikey index.
+validateFindCmdOutputAndPlan({
+    filter: {a: {$in: [null, ["a"]]}},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 3}, {_id: 4}, {_id: 6}, {_id: 7}],
+    expectedStages: {"IXSCAN": 1, "FETCH": 1, "PROJECTION_SIMPLE": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {a: {$in: [null, ["a"]]}},
+    expectedCount: 4,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1, "COUNT": 1},
+});
+
 // Verify that a more complex projection that only relies on the _id field does not need a FETCH.
 validateFindCmdOutputAndPlan({
     filter: {a: null},
@@ -691,4 +748,265 @@ validateCountAggCmdOutputAndPlan({
     expectedCount: 4,
     expectedStages: {"OR": 1, "COUNT_SCAN": 2, "IXSCAN": 0, "FETCH": 0},
 });
+
+// Validate that we can use the optimization when we have regex without array elements in a $in or
+// $or. See SERVER-70436 for more details.
+coll.drop();
+
+assert.commandWorked(coll.insertMany([
+    {_id: 1, a: '123456'},
+    {_id: 2, a: '1234567'},
+    {_id: 3, a: ' 12345678'},
+    {_id: 4, a: '444456'},
+    {_id: 5, a: ''},
+    {_id: 6, a: null},
+    {_id: 7},
+]));
+
+assert.commandWorked(coll.createIndex({a: 1, _id: 1}));
+
+// TODO SERVER-70998: Can apply optimization in case without regex; however, we still can't use a
+// COUNT_SCAN in this case.
+validateFindCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: ""}]},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 5}, {_id: 6}, {_id: 7}],
+    expectedStages: {"IXSCAN": 1, "FETCH": 0, "PROJECTION_COVERED": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: ""}]},
+    expectedCount: 3,
+    expectedStages: {"COUNT": 1, "IXSCAN": 1, "FETCH": 0},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: ""}]},
+    expectedCount: 3,
+    expectedStages: {"IXSCAN": 1, "FETCH": 0},
+});
+
+// Can still apply optimization when we have regex.
+validateFindCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: {$regex: "^$"}}]},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 5}, {_id: 6}, {_id: 7}],
+    expectedStages: {"IXSCAN": 1, "FETCH": 0, "PROJECTION_COVERED": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: {$regex: "^$"}}]},
+    expectedCount: 3,
+    expectedStages: {"IXSCAN": 1, "FETCH": 0, "COUNT": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: {$regex: "^$"}}]},
+    expectedCount: 3,
+    expectedStages: {"IXSCAN": 1, "FETCH": 0},
+});
+
+// Now test case with a multikey index. We can't leverage the optimization here.
+assert.commandWorked(coll.insert({_id: 8, a: [1, 2, 3]}));
+assert.commandWorked(coll.insert({_id: 9, a: []}));
+
+validateFindCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: []}, {a: {$regex: "^$"}}]},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 5}, {_id: 6}, {_id: 7}, {_id: 9}],
+    expectedStages: {"IXSCAN": 1, "FETCH": 1, "PROJECTION_SIMPLE": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: []}, {a: {$regex: "^$"}}]},
+    expectedCount: 4,
+    expectedStages: {"COUNT": 1, "IXSCAN": 1, "FETCH": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: []}, {a: {$regex: "^$"}}]},
+    expectedCount: 4,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+
+// We also shouldn't cover queries on multikey indexes where $in includes an array, as we will still
+// need a filter after the IXSCAN to correctly return
+validateFindCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: []}, {a: [2]}]},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 6}, {_id: 7}, {_id: 9}],
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: []}, {a: [2]}]},
+    expectedCount: 3,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {$or: [{a: null}, {a: []}, {a: [2]}]},
+    expectedCount: 3,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+
+// Validate that when we have a dotted path, we return the correct results for null queries.
+coll.drop();
+assert.commandWorked(coll.insertMany([
+    {_id: 1, a: 1},
+    {_id: 2, a: null},
+    {_id: 3},
+    {_id: 4, a: {b: 1}},
+    {_id: 5, a: {b: null}},
+    {_id: 6, a: {c: 1}},
+]));
+assert.commandWorked(coll.createIndex({"a.b": 1, _id: 1}));
+
+validateFindCmdOutputAndPlan({
+    filter: {"a.b": null},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 5}, {_id: 6}],
+    expectedStages: {"IXSCAN": 1, "PROJECTION_COVERED": 1, "FETCH": 0},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {"a.b": null},
+    expectedCount: 5,
+    expectedStages: {"OR": 1, "COUNT_SCAN": 2, "IXSCAN": 0, "FETCH": 0},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {"a.b": null},
+    expectedCount: 5,
+    expectedStages: {"OR": 1, "COUNT_SCAN": 2, "IXSCAN": 0, "FETCH": 0},
+});
+
+validateFindCmdOutputAndPlan({
+    filter: {a: {b: null}},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 5}],
+    expectedStages: {"COLLSCAN": 1, "PROJECTION_SIMPLE": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {a: {b: null}},
+    expectedCount: 1,
+    expectedStages: {"COLLSCAN": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {a: {b: null}},
+    expectedCount: 1,
+    expectedStages: {"COLLSCAN": 1},
+});
+
+// Still need fetch if we don't have a sufficiently restrictive projection.
+validateFindCmdOutputAndPlan({
+    filter: {"a.b": null},
+    projection: {_id: 1, a: 1},
+    expectedOutput: [
+        {_id: 1, a: 1},
+        {_id: 2, a: null},
+        {_id: 3},
+        {_id: 5, a: {b: null}},
+        {_id: 6, a: {c: 1}},
+    ],
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+
+// Make index multikey, and test case where field b is nested in an array.
+assert.commandWorked(coll.insertMany([
+    {_id: 7, a: [{b: null}]},
+    {_id: 8, a: [{b: []}]},
+    {_id: 9, a: [{b: [1, 2, 3]}]},
+    {_id: 10, a: [{b: 123}]},
+    {_id: 11, a: [{c: 123}]},
+    {_id: 12, a: []},
+    {_id: 13, a: [{}]},
+    {_id: 14, a: [1, 2, 3]},
+    {_id: 15, a: [{b: 1}, {c: 2}, {b: 3}]},
+    {_id: 16, a: [null]},
+]));
+
+validateFindCmdOutputAndPlan({
+    filter: {"a.b": null},
+    projection: {_id: 1},
+    expectedOutput: [
+        {_id: 1},
+        {_id: 2},
+        {_id: 3},
+        {_id: 5},
+        {_id: 6},
+        {_id: 7},
+        {_id: 11},
+        {_id: 13},
+        {_id: 15}
+    ],
+    expectedStages: {"IXSCAN": 1, "PROJECTION_SIMPLE": 1, "FETCH": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {"a.b": null},
+    expectedCount: 9,
+    expectedStages: {"COUNT": 1, "IXSCAN": 1, "FETCH": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {"a.b": null},
+    expectedCount: 9,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+
+validateFindCmdOutputAndPlan({
+    filter: {a: {b: null}},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 5}, {_id: 7}],
+    expectedStages: {
+        "COLLSCAN": 1,
+        "PROJECTION_SIMPLE": 1,
+    },
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {a: {b: null}},
+    expectedCount: 2,
+    expectedStages: {"COLLSCAN": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {a: {b: null}},
+    expectedCount: 2,
+    expectedStages: {"COLLSCAN": 1},
+});
+
+validateFindCmdOutputAndPlan({
+    filter: {a: [{b: null}]},
+    projection: {_id: 1},
+    expectedOutput: [{_id: 7}],
+    expectedStages: {"COLLSCAN": 1, "PROJECTION_SIMPLE": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {a: [{b: null}]},
+    expectedCount: 1,
+    expectedStages: {"COLLSCAN": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {a: [{b: null}]},
+    expectedCount: 1,
+    expectedStages: {"COLLSCAN": 1},
+});
+
+// We still need a FETCH for composite paths, because both {a: [1,2,3]} and {"a.b": null} generate
+// null index keys, but the former should not match the predicate below.
+validateFindCmdOutputAndPlan({
+    filter: {"a.b": {$in: [null, []]}},
+    projection: {_id: 1},
+    expectedOutput: [
+        {_id: 1},
+        {_id: 2},
+        {_id: 3},
+        {_id: 5},
+        {_id: 6},
+        {_id: 7},
+        {_id: 8},
+        {_id: 11},
+        {_id: 13},
+        {_id: 15}
+    ],
+    expectedStages: {"IXSCAN": 1, "PROJECTION_SIMPLE": 1, "FETCH": 1},
+});
+validateSimpleCountCmdOutputAndPlan({
+    filter: {"a.b": {$in: [null, []]}},
+    expectedCount: 10,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
+validateGroupCountAggCmdOutputAndPlan({
+    filter: {"a.b": {$in: [null, []]}},
+    expectedCount: 10,
+    expectedStages: {"IXSCAN": 1, "FETCH": 1},
+});
 })();

jstests/core/elemmatch_index.js

@@ -2,6 +2,7 @@
  * Test that queries containing $elemMatch correctly use an index if each child expression is
  * compatible with the index.
  * @tags: [
+ *   assumes_balancer_off,
  *   assumes_read_concern_local,
  * ]
  */
@@ -64,4 +65,51 @@ assertIndexResults(coll, {a: {$elemMatch: {$not: {$in: [/^a/]}}}}, false, 3);
 coll.dropIndexes();
 assert.commandWorked(coll.createIndex({a: 1}));
 assertIndexResults(coll, {a: {$elemMatch: {$not: {$in: [/^a/]}}}}, false, 3);
+
+(function() {
+assert(coll.drop());
+assert.commandWorked(coll.insert({a: [{b: {c: "x"}}]}));
+assert.commandWorked(coll.createIndex({"a.b.c": 1}));
+
+// Tests $elemMatch with path components that are empty strings. The system should not attempt to
+// use the index for these queries.
+assertIndexResults(coll, {"": {$elemMatch: {"a.b.c": "x"}}}, false, 0);
+assertIndexResults(coll, {"": {$all: [{$elemMatch: {"a.b.c": "x"}}]}}, false, 0);
+assertIndexResults(coll, {a: {$elemMatch: {"": {$elemMatch: {"b.c": "x"}}}}}, false, 0);
+
+// Tests $elemMatch with supporting index and no path components that are empty strings.
+assertIndexResults(coll, {a: {$elemMatch: {"b.c": "x"}}}, true, 1);
+assertIndexResults(coll, {a: {$all: [{$elemMatch: {"b.c": "x"}}]}}, true, 1);
+})();
+
+(function() {
+const coll = db.index_elemmatch1;
+coll.drop();
+
+let x = 0;
+let y = 0;
+const bulk = coll.initializeUnorderedBulkOp();
+for (let a = 0; a < 10; a++) {
+    for (let b = 0; b < 10; b++) {
+        bulk.insert({a: a, b: b % 10, arr: [{x: x++ % 10, y: y++ % 10}]});
+    }
+}
+assert.commandWorked(bulk.execute());
+
+assert.commandWorked(coll.createIndex({a: 1, b: 1}));
+assert.commandWorked(coll.createIndex({"arr.x": 1, a: 1}));
+
+const query = {
+    a: 5,
+    b: {$in: [1, 3, 5]},
+    arr: {$elemMatch: {x: 5, y: 5}}
+};
+
+const count = coll.find(query).itcount();
+assert.eq(count, 1);
+
+const explain = coll.find(query).hint({"arr.x": 1, a: 1}).explain("executionStats");
+assert.commandWorked(explain);
+assert.eq(count, explain.executionStats.totalKeysExamined, explain);
+})();
 })();

jstests/core/explain_missing_collection.js

@@ -5,6 +5,7 @@
  * ]
  */
 (function() {
+db.filler_collection.drop();
 assert.commandWorked(db.createCollection("filler_collection"));
 
 var missingColl = db.explain_null_collection;

jstests/core/index_elemmatch1.js

@@ -1,39 +0,0 @@
-/**
- * Tests find with $elemMatch when supporting indexes are in place.
- * @tags: [
- *   assumes_balancer_off,
- *   assumes_read_concern_local,
- * ]
- */
-(function() {
-"use strict";
-
-const coll = db.index_elemmatch1;
-coll.drop();
-
-let x = 0;
-let y = 0;
-const bulk = coll.initializeUnorderedBulkOp();
-for (let a = 0; a < 10; a++) {
-    for (let b = 0; b < 10; b++) {
-        bulk.insert({a: a, b: b % 10, arr: [{x: x++ % 10, y: y++ % 10}]});
-    }
-}
-assert.commandWorked(bulk.execute());
-
-assert.commandWorked(coll.createIndex({a: 1, b: 1}));
-assert.commandWorked(coll.createIndex({"arr.x": 1, a: 1}));
-
-const query = {
-    a: 5,
-    b: {$in: [1, 3, 5]},
-    arr: {$elemMatch: {x: 5, y: 5}}
-};
-
-const count = coll.find(query).itcount();
-assert.eq(count, 1);
-
-const explain = coll.find(query).hint({"arr.x": 1, a: 1}).explain("executionStats");
-assert.commandWorked(explain);
-assert.eq(count, explain.executionStats.totalKeysExamined, explain);
-})();

jstests/core/mr_single_reduce.js

@@ -0,0 +1,18 @@
+(function() {
+"use strict";
+const coll = db.bar;
+
+assert.commandWorked(coll.insert({x: 1}));
+
+const map = function() {
+    emit(0, "mapped value");
+};
+
+const reduce = function(key, values) {
+    return "reduced value";
+};
+
+const res = assert.commandWorked(
+    db.runCommand({mapReduce: 'bar', map: map, reduce: reduce, out: {inline: 1}}));
+assert.eq(res.results[0], {_id: 0, value: "reduced value"});
+}());

jstests/core/null_query_semantics.js

@@ -41,6 +41,9 @@ function testNullSemantics(coll) {
             [{_id: "a_null", a: null}, {_id: "a_undefined", a: undefined}, {_id: "no_a"}];
         assert(resultsEq(expected, noProjectResults), tojson(noProjectResults));
 
+        const count = coll.count({a: {$eq: null}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find({a: {$eq: null}}, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), tojson(projectResults));
     }());
@@ -57,6 +60,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
 
+        const count = coll.count({a: {$ne: null}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find({a: {$ne: null}}, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), tojson(projectResults));
     }());
@@ -72,6 +78,9 @@ function testNullSemantics(coll) {
             {_id: "no_a"},
         ];
 
+        const count = coll.count(query);
+        assert.eq(count, expected.length);
+
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
         const projectResults = coll.find(query, projectToOnlyA).toArray();
@@ -89,6 +98,9 @@ function testNullSemantics(coll) {
             {_id: "no_a"},
         ];
 
+        const count = coll.count(query);
+        assert.eq(count, expected.length);
+
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
         const projectResults = coll.find(query, projectToOnlyA).toArray();
@@ -106,6 +118,9 @@ function testNullSemantics(coll) {
             {_id: "a_subobject_b_undefined", a: {b: undefined}},
         ];
 
+        const count = coll.count(query);
+        assert.eq(count, expected.length);
+
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
         const projectResults = coll.find(query, projectToOnlyA).toArray();
@@ -126,6 +141,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
 
+        const count = coll.count(query);
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -137,6 +155,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
 
+        const count = coll.count({a: {$exists: false}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find({a: {$exists: false}}, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), tojson(projectResults));
     }());
@@ -144,17 +165,19 @@ function testNullSemantics(coll) {
     // Test the semantics of the query {"a.b": {$eq: null}}.
     (function testDottedEqualsNull() {
         const noProjectResults = coll.find({"a.b": {$eq: null}}).toArray();
-        assert(resultsEq(noProjectResults,
-                         [
-                             {_id: "a_empty_subobject", a: {}},
-                             {_id: "a_null", a: null},
-                             {_id: "a_number", a: 4},
-                             {_id: "a_subobject_b_null", a: {b: null}},
-                             {_id: "a_subobject_b_undefined", a: {b: undefined}},
-                             {_id: "a_undefined", a: undefined},
-                             {_id: "no_a"}
-                         ]),
-               tojson(noProjectResults));
+        const expected = [
+            {_id: "a_empty_subobject", a: {}},
+            {_id: "a_null", a: null},
+            {_id: "a_number", a: 4},
+            {_id: "a_subobject_b_null", a: {b: null}},
+            {_id: "a_subobject_b_undefined", a: {b: undefined}},
+            {_id: "a_undefined", a: undefined},
+            {_id: "no_a"}
+        ];
+        assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
+
+        const count = coll.count({"a.b": {$eq: null}});
+        assert.eq(count, expected.length);
 
         const projectResults = coll.find({"a.b": {$eq: null}}, projectToOnlyADotB).toArray();
         assert(resultsEq(projectResults,
@@ -165,8 +188,11 @@ function testNullSemantics(coll) {
     // Test the semantics of the query {"a.b": {$ne: null}}.
     (function testDottedNotEqualsNull() {
         const noProjectResults = coll.find({"a.b": {$ne: null}}).toArray();
-        assert(resultsEq(noProjectResults, [{_id: "a_subobject_b_not_null", a: {b: "hi"}}]),
-               tojson(noProjectResults));
+        const expected = [{_id: "a_subobject_b_not_null", a: {b: "hi"}}];
+        assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
+
+        const count = coll.count({"a.b": {$ne: null}});
+        assert.eq(count, expected.length);
 
         const projectResults = coll.find({"a.b": {$ne: null}}, projectToOnlyADotB).toArray();
         assert(resultsEq(projectResults, [{a: {b: "hi"}}]), tojson(projectResults));
@@ -183,6 +209,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
 
+        const count = coll.count({"a.b": {$exists: false}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find({"a.b": {$exists: false}}, projectToOnlyADotB).toArray();
         assert(resultsEq(projectResults, [{}, {a: {}}, {}, {}, {}]), tojson(projectResults));
     }());
@@ -202,6 +231,9 @@ function testNullSemantics(coll) {
                    `Expected no results for query ${tojson(elemMatchQuery)}, got ` +
                        tojson(noProjectResults));
 
+            const count = coll.count(elemMatchQuery);
+            assert.eq(count, 0);
+
             let projectResults = coll.find(elemMatchQuery, projectToOnlyA).toArray();
             assert(resultsEq(projectResults, []),
                    `Expected no results for query ${tojson(elemMatchQuery)}, got ` +
@@ -250,6 +282,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
 
+        const count = coll.count({a: {$eq: null}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find({a: {$eq: null}}, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), tojson(projectResults));
     }());
@@ -274,6 +309,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
 
+        const count = coll.count({a: {$ne: null}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find({a: {$ne: null}}, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), tojson(projectResults));
     }());
@@ -297,6 +335,8 @@ function testNullSemantics(coll) {
             {_id: "a_value_array_no_nulls", a: [1, "string", 4]},
         ];
         assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
+        const count = coll.count({a: {$not: {$gte: null}}});
+        assert.eq(count, expected.length);
     }());
 
     // Test the semantics of the query {a: {$in: [null, <number>]}}.
@@ -314,6 +354,9 @@ function testNullSemantics(coll) {
 
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
+        const count = coll.count({a: {$in: [null, 75]}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -333,6 +376,9 @@ function testNullSemantics(coll) {
 
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
+        const count = coll.count({$or: [{a: null}, {a: 75}]});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -360,6 +406,9 @@ function testNullSemantics(coll) {
 
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
+        const count = coll.count({a: {$nin: [null, 75]}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -395,6 +444,9 @@ function testNullSemantics(coll) {
 
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
+        const count = coll.count({a: {$nin: [null, []]}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -420,6 +472,9 @@ function testNullSemantics(coll) {
 
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
+        const count = coll.count(query);
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -435,6 +490,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expectedEqualToNull), tojson(noProjectResults));
 
+        const count = coll.count({a: {$elemMatch: {$eq: null}}});
+        assert.eq(count, expectedEqualToNull.length);
+
         let projectResults = coll.find({a: {$elemMatch: {$eq: null}}}, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expectedEqualToNull)),
                tojson(projectResults));
@@ -467,25 +525,23 @@ function testNullSemantics(coll) {
     //    value for "b".
     (function testDottedEqualsNull() {
         const noProjectResults = coll.find({"a.b": {$eq: null}}).toArray();
-        assert(
-            resultsEq(noProjectResults,
-                      [
-                          {_id: "a_empty_subobject", a: {}},
-                          {_id: "a_null", a: null},
-                          {_id: "a_number", a: 4},
-                          {_id: "a_subobject_b_null", a: {b: null}},
-                          {_id: "a_subobject_b_undefined", a: {b: undefined}},
-                          {_id: "a_undefined", a: undefined},
-                          {_id: "no_a"},
-                          {
-                              _id: "a_object_array_all_b_nulls",
-                              a: [{b: null}, {b: undefined}, {b: null}, {}]
-                          },
-                          {_id: "a_object_array_some_b_nulls", a: [{b: null}, {b: 3}, {b: null}]},
-                          {_id: "a_object_array_some_b_undefined", a: [{b: undefined}, {b: 3}]},
-                          {_id: "a_object_array_some_b_missing", a: [{b: 3}, {}]},
-                      ]),
-            tojson(noProjectResults));
+        const expected = [
+            {_id: "a_empty_subobject", a: {}},
+            {_id: "a_null", a: null},
+            {_id: "a_number", a: 4},
+            {_id: "a_subobject_b_null", a: {b: null}},
+            {_id: "a_subobject_b_undefined", a: {b: undefined}},
+            {_id: "a_undefined", a: undefined},
+            {_id: "no_a"},
+            {_id: "a_object_array_all_b_nulls", a: [{b: null}, {b: undefined}, {b: null}, {}]},
+            {_id: "a_object_array_some_b_nulls", a: [{b: null}, {b: 3}, {b: null}]},
+            {_id: "a_object_array_some_b_undefined", a: [{b: undefined}, {b: 3}]},
+            {_id: "a_object_array_some_b_missing", a: [{b: 3}, {}]},
+        ];
+        assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
+
+        const count = coll.count({"a.b": {$eq: null}});
+        assert.eq(count, expected.length);
 
         const projectResults = coll.find({"a.b": {$eq: null}}, projectToOnlyADotB).toArray();
         assert(resultsEq(projectResults,
@@ -508,18 +564,20 @@ function testNullSemantics(coll) {
     // Test the semantics of the query {"a.b": {$ne: null}}.
     (function testDottedNotEqualsNull() {
         const noProjectResults = coll.find({"a.b": {$ne: null}}).toArray();
-        assert(resultsEq(noProjectResults,
-                         [
-                             {_id: "a_subobject_b_not_null", a: {b: "hi"}},
-                             {_id: "a_double_array", a: [[]]},
-                             {_id: "a_empty_array", a: []},
-                             {_id: "a_object_array_no_b_nulls", a: [{b: 1}, {b: 3}, {b: "string"}]},
-                             {_id: "a_value_array_all_nulls", a: [null, null]},
-                             {_id: "a_value_array_no_nulls", a: [1, "string", 4]},
-                             {_id: "a_value_array_with_null", a: [1, "string", null, 4]},
-                             {_id: "a_value_array_with_undefined", a: [1, "string", undefined, 4]}
-                         ]),
-               tojson(noProjectResults));
+        const expected = [
+            {_id: "a_subobject_b_not_null", a: {b: "hi"}},
+            {_id: "a_double_array", a: [[]]},
+            {_id: "a_empty_array", a: []},
+            {_id: "a_object_array_no_b_nulls", a: [{b: 1}, {b: 3}, {b: "string"}]},
+            {_id: "a_value_array_all_nulls", a: [null, null]},
+            {_id: "a_value_array_no_nulls", a: [1, "string", 4]},
+            {_id: "a_value_array_with_null", a: [1, "string", null, 4]},
+            {_id: "a_value_array_with_undefined", a: [1, "string", undefined, 4]}
+        ];
+        assert(resultsEq(noProjectResults, expected), tojson(noProjectResults));
+
+        const count = coll.count({"a.b": {$ne: null}});
+        assert.eq(count, expected.length);
 
         const projectResults = coll.find({"a.b": {$ne: null}}, projectToOnlyADotB).toArray();
         assert(resultsEq(projectResults,
@@ -554,6 +612,9 @@ function testNullSemantics(coll) {
             {_id: "a_object_array_some_b_missing", a: [{b: 3}, {}]},
         ];
 
+        const count = coll.count({"a.b": {$in: [null, 75]}});
+        assert.eq(count, expected.length);
+
         assert(resultsEq(noProjectResults, expected), noProjectResults);
     }());
 
@@ -575,6 +636,9 @@ function testNullSemantics(coll) {
             {_id: "a_object_array_some_b_missing", a: [{b: 3}, {}]},
         ];
 
+        const count = coll.count({$or: [{"a.b": null}, {"a.b": 75}]});
+        assert.eq(count, expected.length);
+
         assert(resultsEq(noProjectResults, expected), noProjectResults);
     }());
 
@@ -595,6 +659,9 @@ function testNullSemantics(coll) {
 
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
+        const count = coll.count({"a.b": {$nin: [null, 75]}});
+        assert.eq(count, expected.length);
+
         const projectResults = coll.find(query, projectToOnlyA).toArray();
         assert(resultsEq(projectResults, extractAValues(expected)), projectResults);
     }());
@@ -613,6 +680,9 @@ function testNullSemantics(coll) {
             {_id: "a_value_array_with_undefined", a: [1, "string", undefined, 4]},
         ];
 
+        const count = coll.count({"a.b": {$nin: [null, /^str.*/]}});
+        assert.eq(count, expected.length);
+
         assert(resultsEq(noProjectResults, expected), noProjectResults);
 
         const projectResults = coll.find(query, projectToOnlyA).toArray();
@@ -625,6 +695,9 @@ function testNullSemantics(coll) {
         let results = coll.find({"a.b": {$elemMatch: {$eq: null}}}).toArray();
         assert(resultsEq(results, []), tojson(results));
 
+        const count = coll.count({"a.b": {$elemMatch: {$eq: null}}});
+        assert.eq(count, 0);
+
         results = coll.find({"a.b": {$elemMatch: {$ne: null}}}).toArray();
         assert(resultsEq(results, []), tojson(results));
     }());
@@ -642,6 +715,9 @@ function testNullSemantics(coll) {
         ];
         assert(resultsEq(noProjectResults, expectedEqualToNull), tojson(noProjectResults));
 
+        const count = coll.count({a: {$elemMatch: {b: {$eq: null}}}});
+        assert.eq(count, expectedEqualToNull.length);
+
         let projectResults =
             coll.find({a: {$elemMatch: {b: {$eq: null}}}}, projectToOnlyADotB).toArray();
         assert(resultsEq(projectResults,

jstests/core/timeseries/timeseries_collmod.js

@@ -6,7 +6,7 @@
  *  # expected.
  *  assumes_no_implicit_collection_creation_after_drop,
  *  requires_non_retryable_commands,
- *  requires_fcv_51,
+ *  requires_fcv_61,
  * ]
  */
 

jstests/core/ttl_index_options.js

@@ -1,7 +1,10 @@
 /**
  * Ensures that the options passed in for TTL indexes are validated during index creation.
  *
- * @tags: [requires_ttl_index]
+ * @tags: [
+ *     requires_fcv_61,
+ *     requires_ttl_index,
+ * ]
  */
 (function() {
 'use strict';
@@ -16,11 +19,10 @@ assert.commandFailedWithCode(
 assert.commandFailedWithCode(coll.createIndexes([{x: 1}], {expireAfterSeconds: 9999999999999999}),
                              ErrorCodes.CannotCreateIndex);
 
-// Ensure that we cannot provide a time that is larger than the current epoch time.
+// Ensure that we can provide a time that is larger than the current epoch time.
 let secondsSinceEpoch = Date.now() / 1000;
-assert.commandFailedWithCode(
-    coll.createIndexes([{x: 1}], {expireAfterSeconds: secondsSinceEpoch + 1000}),
-    ErrorCodes.CannotCreateIndex);
+assert.commandWorked(
+    coll.createIndexes([{x_before_epoch: 1}], {expireAfterSeconds: secondsSinceEpoch + 1000}));
 
 // 'expireAfterSeconds' cannot be less than 0.
 assert.commandFailedWithCode(coll.createIndexes([{x: 1}], {expireAfterSeconds: -1}),

jstests/core/views/views_all_commands.js

@@ -550,7 +550,6 @@ let viewsCommandTests = {
             skipSharded: true,
         }
     ],
-    repairDatabase: {skip: isUnrelated},
     repairShardedCollectionChunksHistory: {
         command: {repairShardedCollectionChunksHistory: "test.view"},
         skipStandalone: true,

jstests/disk/wt_table_checks.js

@@ -8,6 +8,43 @@
 
 load('jstests/disk/libs/wt_file_helper.js');
 
+function checkTableLogSettings(conn, enabled) {
+    conn.getDBNames().forEach(function(d) {
+        let collNames =
+            conn.getDB(d)
+                .runCommand({listCollections: 1, nameOnly: true, filter: {type: "collection"}})
+                .cursor.firstBatch;
+
+        collNames.forEach(function(c) {
+            let stats = conn.getDB(d).runCommand({collStats: c.name});
+
+            let logStr = "log=(enabled=" + (enabled ? "true" : "false") + ")";
+            if (d == "local") {
+                if (c.name == "replset.minvalid" && !enabled) {
+                    // This collection is never logged in a replica set.
+                    logStr = "log=(enabled=false)";
+                } else {
+                    // All other collections and indexes in the 'local' database have table
+                    // logging enabled always.
+                    logStr = "log=(enabled=true)";
+                }
+            }
+
+            assert.eq(true, stats.wiredTiger.creationString.includes(logStr));
+            Object.keys(stats.indexDetails).forEach(function(i) {
+                assert.eq(true, stats.indexDetails[i].creationString.includes(logStr));
+            });
+        });
+    });
+}
+
+function checkTableChecksFileRemoved(dbpath) {
+    let files = listFiles(dbpath);
+    for (file of files) {
+        assert.eq(false, file.name.includes("_wt_table_checks"));
+    }
+}
+
 // Create a bunch of collections under various database names.
 let conn = MongoRunner.runMongod({});
 const dbpath = conn.dbpath;
@@ -16,112 +53,75 @@ for (let i = 0; i < 10; i++) {
     assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString()));
 }
 
+checkTableLogSettings(conn, /*enabled=*/true);
 MongoRunner.stopMongod(conn);
 
 /**
- * Test 1. The regular case, where no table logging setting modifications are needed.
+ * Test 1. Change into a single node replica set, which requires all of the table logging settings
+ * to be updated. Write the '_wt_table_checks' file and check that it gets removed.
  */
 jsTest.log("Test 1.");
 
-conn = startMongodOnExistingPath(dbpath, {});
-checkLog.containsJson(conn, 4366408, {loggingEnabled: true});
+writeFile(dbpath + "/_wt_table_checks", "");
+conn = startMongodOnExistingPath(
+    dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}});
+checkTableChecksFileRemoved(dbpath);
+
+// Changing table logging settings.
+checkLog.containsJson(conn, 22432);
 MongoRunner.stopMongod(conn);
 
 /**
- * Test 2. Repair checks all of the table logging settings.
+ * Test 2. Restart in standalone mode with wiredTigerSkipTableLoggingChecksOnStartup. No table log
+ * settings are updated.  Write the '_wt_table_checks' file and check that it gets removed.
  */
 jsTest.log("Test 2.");
-
-assertRepairSucceeds(dbpath, conn.port, {});
-
-// Cannot use checkLog here as the server is no longer running.
-let logContents = rawMongoProgramOutput();
-assert(logContents.indexOf(
-           "Modifying the table logging settings for all existing WiredTiger tables") > 0);
-
-/**
- * Test 3. Explicitly create the '_wt_table_checks' file to force all of the table logging setting
- * modifications to be made.
- */
-jsTest.log("Test 3.");
-
-let files = listFiles(dbpath);
-for (f in files) {
-    assert(!files[f].name.includes("_wt_table_checks"));
-}
-
 writeFile(dbpath + "/_wt_table_checks", "");
-
-// Cannot skip table logging checks on startup when there are previously incomplete table checks.
-assert.throws(() => startMongodOnExistingPath(
-                  dbpath, {setParameter: "wiredTigerSkipTableLoggingChecksOnStartup=true"}));
-
-conn = startMongodOnExistingPath(dbpath, {});
-checkLog.containsJson(
-    conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true});
-MongoRunner.stopMongod(conn);
-
-/**
- * Test 4. Change into a single replica set, which requires all of the table logging settings to be
- * updated. But simulate an interruption/crash while starting up during the table logging check
- * phase.
- *
- * The next start up will detect an unclean shutdown causing all of the table logging settings to be
- * updated.
- */
-jsTest.log("Test 4.");
-
-assert.throws(() => startMongodOnExistingPath(dbpath, {
-                  replSet: "mySet",
-                  setParameter: "failpoint.crashAfterUpdatingFirstTableLoggingSettings=" +
-                      tojson({"mode": "alwaysOn"})
-              }));
-
-// Cannot use checkLog here as the server is no longer running.
-logContents = rawMongoProgramOutput();
-assert(logContents.indexOf(
-           "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point") > 0);
-
-// The '_wt_table_checks' still exists, so all table logging settings should be modified.
-conn = startMongodOnExistingPath(dbpath, {});
-checkLog.containsJson(
-    conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true});
-MongoRunner.stopMongod(conn);
-
-/**
- * Test 5. Change into a single node replica set, which requires all of the table logging settings
- * to be updated as the node was successfully started up as a standalone the last time.
- */
-jsTest.log("Test 5.");
-
-conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"});
-checkLog.containsJson(conn, 4366406, {loggingEnabled: false});
-MongoRunner.stopMongod(conn);
-
-/**
- * Test 6. Restart as a standalone and skip table logging checks on startup. Verify that restarting
- * as a replica set again does not require any table logging modifications.
- */
-jsTest.log("Test 6.");
-
 conn = startMongodOnExistingPath(dbpath, {
     setParameter: {
         wiredTigerSkipTableLoggingChecksOnStartup: true,
         logComponentVerbosity: tojson({verbosity: 1})
     }
 });
+checkTableChecksFileRemoved(dbpath);
 
-// Skipping table logging checks for all existing tables.
-checkLog.containsJson(conn, 5548301, {wiredTigerSkipTableLoggingChecksOnStartup: true});
-
-// Log level 1 prints each individual table it skips table logging checks for.
+// Skipping table logging checks.
 checkLog.containsJson(conn, 5548302);
 
+// Changing table logging settings.
+assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0));
+checkTableLogSettings(conn, /*enabled=*/false);
 MongoRunner.stopMongod(conn);
 
-conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"});
+/**
+ * Test 3. Change into a single node replica set again. Table log settings are checked but none are
+ * changed.  Write the '_wt_table_checks' file and check that it gets removed.
+ */
+jsTestLog("Test 3.");
+writeFile(dbpath + "/_wt_table_checks", "");
+conn = startMongodOnExistingPath(
+    dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}});
+checkTableChecksFileRemoved(dbpath);
 
-// No table logging settings modifications are required.
-checkLog.containsJson(conn, 4366408);
+// Changing table logging settings.
+assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0));
+MongoRunner.stopMongod(conn);
+
+/**
+ * Test 4. Back to standalone. Check that the table log settings are enabled. Write the
+ * '_wt_table_checks' file and check that it gets removed.
+ */
+jsTest.log("Test 4.");
+writeFile(dbpath + "/_wt_table_checks", "");
+conn = startMongodOnExistingPath(dbpath,
+                                 {setParameter: {logComponentVerbosity: tojson({verbosity: 1})}});
+checkTableChecksFileRemoved(dbpath);
+
+// Changing table logging settings.
+checkLog.containsJson(conn, 22432);
+
+// Skipping table logging checks.
+assert(checkLog.checkContainsWithCountJson(conn, 5548302, undefined, 0));
+checkTableLogSettings(conn, /*enabled=*/true);
 MongoRunner.stopMongod(conn);
 }());

jstests/libs/change_stream_rewrite_util.js

@@ -138,7 +138,10 @@ function assertNumMatchingOplogEventsForShard(stats, shardName, expectedTotalRet
     assert(stats.shards.hasOwnProperty(shardName), stats);
     assert.eq(Object.keys(stats.shards[shardName].stages[0])[0], "$cursor", stats);
     const executionStats = stats.shards[shardName].stages[0].$cursor.executionStats;
-    assert.eq(executionStats.nReturned, expectedTotalReturned, executionStats);
+    assert.eq(executionStats.nReturned,
+              expectedTotalReturned,
+              () => `Expected ${expectedTotalReturned} events on shard ${shardName} but got ` +
+                  `${executionStats.nReturned}. Execution stats:\n${tojson(executionStats)}`);
 }
 
 // Returns a newly created sharded collection sharded by caller provided shard key.
@@ -180,7 +183,10 @@ function verifyChangeStreamOnWholeCluster(
             eventIdentifierList.forEach(eventIdentifier => {
                 assert.soon(() => cursor.hasNext(), {op: op, eventIdentifier: eventIdentifier});
                 const event = cursor.next();
-                assert.eq(event.operationType, op, event);
+                assert.eq(event.operationType,
+                          op,
+                          () => `Expected "${op}" but got "${event.operationType}". Full event: ` +
+                              `${tojson(event)}`);
 
                 if (op == "dropDatabase") {
                     assert.eq(event.ns.db, eventIdentifier, event);

jstests/noPassthrough/change_streams_per_shard_cursor.js

@@ -98,7 +98,7 @@ let c = pscWatch(sdb, "coll", shardId);
 for (let i = 1; i <= 4; i++) {
     sdb.coll.insertOne({location: 2, i});
     assert(!c.isExhausted());
-    assert(c.hasNext());
+    assert.soon(() => c.hasNext());
     c.next();
 }
 assert(!c.hasNext());
@@ -108,12 +108,12 @@ c = pscWatch(sdb, 1, shardId);
 
 sdb.coll.insertOne({location: 3});
 assert(!c.isExhausted());
-assert(c.hasNext());
+assert.soon(() => c.hasNext());
 c.next();
 
 sdb.coll2.insertOne({location: 4});
 assert(!c.isExhausted());
-assert(c.hasNext());
+assert.soon(() => c.hasNext());
 c.next();
 
 assert(!c.hasNext());
@@ -129,7 +129,7 @@ assert(!c.hasNext());
 
 sdb.toBeCreated.insertOne({location: 8});
 assert(!c.isExhausted());
-assert(c.hasNext());
+assert.soon(() => c.hasNext());
 c.next();
 
 assert(!c.hasNext());
@@ -164,7 +164,7 @@ c = pscWatch(sdb, "coll", shardId);
 
 sdb.coll.insertOne({location: 5, _id: -2});
 assert(!c.isExhausted());
-assert(c.hasNext());
+assert.soon(() => c.hasNext());
 c.next();
 
 sdb.coll.insertOne({location: 6, _id: 2});
@@ -176,12 +176,12 @@ c = pscWatch(sdb.getSiblingDB("admin"), 1, shardId, {}, {allChangesForCluster: t
 
 sdb.coll.insertOne({location: 7, _id: -3});
 assert(!c.isExhausted());
-assert(c.hasNext());
+assert.soon(() => c.hasNext());
 c.next();
 
 sdb.coll2.insertOne({location: 8, _id: -4});
 assert(!c.isExhausted());
-assert(c.hasNext());
+assert.soon(() => c.hasNext());
 c.next();
 
 sdb.coll.insertOne({location: 9, _id: 3});

jstests/noPassthrough/collmod_convert_to_unique_sharded.js

@@ -0,0 +1,91 @@
+/**
+ * Tests collMod unique conversion ensures consistent index specs across all shards.
+ *
+ * @tags: [
+ *   # TODO(SERVER-61182): Fix WiredTigerKVEngine::alterIdentMetadata() under inMemory.
+ *   requires_persistence,
+ *   requires_sharding,
+ * ]
+ */
+(function() {
+'use strict';
+
+function countUniqueIndexes(coll, key) {
+    const all = coll.getIndexes().filter(function(z) {
+        return z.unique && friendlyEqual(z.key, key);
+    });
+    return all.length;
+}
+
+function countPrepareUniqueIndexes(coll, key) {
+    const all = coll.getIndexes().filter(function(z) {
+        return z.prepareUnique && friendlyEqual(z.key, key);
+    });
+    return all.length;
+}
+
+const st = new ShardingTest({shards: 2});
+const mongos = st.s;
+
+const db = mongos.getDB(jsTestName());
+assert.commandWorked(
+    mongos.adminCommand({enableSharding: db.getName(), primaryShard: st.shard0.name}));
+
+const shardedColl = db.sharded;
+
+assert.commandWorked(
+    mongos.adminCommand({shardCollection: shardedColl.getFullName(), key: {a: 1}}));
+
+// Move {a: 1} to shard0 and {a: 2} to shard1.
+assert.commandWorked(st.splitAt(shardedColl.getFullName(), {a: 2}));
+assert.commandWorked(mongos.adminCommand(
+    {moveChunk: shardedColl.getFullName(), find: {a: 1}, to: st.shard0.shardName}));
+assert.commandWorked(mongos.adminCommand(
+    {moveChunk: shardedColl.getFullName(), find: {a: 2}, to: st.shard1.shardName}));
+
+assert.commandWorked(shardedColl.createIndex({a: 1}));
+
+assert.commandWorked(shardedColl.insert({_id: 0, a: 1}));
+assert.commandWorked(shardedColl.insert({_id: 1, a: 2}));
+assert.commandWorked(shardedColl.insert({_id: 2, a: 2}));
+
+// Setting the indexes to 'prepareUnique' ensures no new duplicates will be inserted.
+assert.commandWorked(db.runCommand(
+    {collMod: shardedColl.getName(), index: {keyPattern: {a: 1}, prepareUnique: true}}));
+assert.commandFailedWithCode(shardedColl.insert({_id: 3, a: 1}), ErrorCodes.DuplicateKey);
+assert.commandFailedWithCode(shardedColl.insert({_id: 4, a: 2}), ErrorCodes.DuplicateKey);
+
+// Try converting the index to unique and make sure no indexes are converted on any shards.
+assert.commandFailedWithCode(
+    db.runCommand(
+        {collMod: shardedColl.getName(), index: {keyPattern: {a: 1}, hidden: true, unique: true}}),
+    ErrorCodes.CannotConvertIndexToUnique);
+
+const s0Coll = st.shard0.getDB(jsTestName()).getCollection("sharded");
+const s1Coll = st.shard1.getDB(jsTestName()).getCollection("sharded");
+assert.eq(countUniqueIndexes(s0Coll, {a: 1}),
+          0,
+          'index should not be unique: ' + tojson(s0Coll.getIndexes()));
+assert.eq(countUniqueIndexes(s1Coll, {a: 1}),
+          0,
+          'index should not be unique: ' + tojson(s1Coll.getIndexes()));
+assert.eq(countPrepareUniqueIndexes(s0Coll, {a: 1}),
+          1,
+          'index should be prepareUnique: ' + tojson(s0Coll.getIndexes()));
+assert.eq(countPrepareUniqueIndexes(s1Coll, {a: 1}),
+          1,
+          'index should be prepareUnique: ' + tojson(s1Coll.getIndexes()));
+
+// Remove the duplicate and confirm the indexes are converted.
+assert.commandWorked(shardedColl.deleteOne({_id: 2}));
+assert.commandWorked(db.runCommand(
+    {collMod: shardedColl.getName(), index: {keyPattern: {a: 1}, hidden: true, unique: true}}));
+assert.eq(countUniqueIndexes(s0Coll, {a: 1}),
+          1,
+          'index should be unique: ' + tojson(s0Coll.getIndexes()));
+assert.eq(countUniqueIndexes(s1Coll, {a: 1}),
+          1,
+          'index should be unique: ' + tojson(s1Coll.getIndexes()));
+
+st.stop();
+})();

jstests/noPassthrough/delete_incomplete_index_data_on_startup_recovery.js

@@ -1,92 +0,0 @@
-/**
- * Test that the data tables for an incomplete index on unclean shutdown is dropped immediately as
- * opposed to deferred per the usual two-phase index drop logic.
- *
- * @tags: [
- *   requires_replication,
- *   # The primary is restarted and must retain its data.
- *   requires_persistence,
- * ]
- */
-
-(function() {
-"use strict";
-
-load("jstests/libs/fail_point_util.js");
-
-const rst = new ReplSetTest({name: jsTest.name(), nodes: 1});
-rst.startSet();
-rst.initiate();
-
-const dbName = "testDB";
-const collName = "fantasticalCollName";
-const primary = rst.getPrimary();
-const primaryDB = primary.getDB(dbName);
-const primaryColl = primaryDB[collName];
-
-assert.commandWorked(primaryDB.createCollection(collName));
-assert.commandWorked(primaryColl.insert({x: 1}));
-assert.commandWorked(primaryColl.insert({x: 2}));
-assert.commandWorked(primaryColl.insert({x: 3}));
-
-const failPoint = configureFailPoint(primaryDB, "hangIndexBuildBeforeCommit");
-let ps = undefined;
-try {
-    TestData.dbName = dbName;
-    TestData.collName = collName;
-    TestData.indexSpec = {x: 1};
-    TestData.indexName = {name: "myFantasticalIndex"};
-
-    ps = startParallelShell(() => {
-        jsTestLog("Starting an index build that will hang, establishing an unfinished index " +
-                  "to be found on startup.");
-        // Crashing the server while this command is running may cause the parallel shell code to
-        // error and stop executing. We will therefore ignore the result of this command and
-        // parallel shell. Test correctness is guaranteed by waiting for the failpoint this command
-        // hits.
-        db.getSiblingDB(TestData.dbName)[TestData.collName].createIndex(TestData.indexSpec,
-                                                                        TestData.indexName);
-    }, primary.port);
-
-    jsTest.log("Waiting for the async index build to hit the failpoint.");
-    failPoint.wait();
-
-    jsTest.log(
-        "Force a checkpoint, now that the index is present in the catalog. This ensures that on " +
-        "startup the index will not be an 'unknown ident' discarded immediately because it does " +
-        "not have an associcated catalog entry. Or some other unexpected checkpoint situation.");
-    assert.commandWorked(primary.adminCommand({fsync: 1}));
-
-    // Crash and restart the primary, which should cause the old index data to be deleted before
-    // rebuilding the index from scratch.
-    rst.stop(primary, 9, {allowedExitCode: MongoRunner.EXIT_SIGKILL});
-} catch (error) {
-    // Turn off the failpoint before allowing the test to end, so nothing hangs while the server
-    // shuts down or in post-test hooks.
-    failPoint.off();
-    throw error;
-} finally {
-    if (ps) {
-        ps({checkExitSuccess: false});
-    }
-}
-
-// Restart the node and set 'noCleanData' so that the node's data doesn't get wiped out.
-rst.start(primary, {noCleanData: true});
-
-// Wait for the node to become available.
-rst.getPrimary();
-
-// For debugging purposes, fetch and log the test collection's indexes after restart.
-const indexes =
-    assert.commandWorked(rst.getPrimary().getDB(dbName).runCommand({listIndexes: collName}))
-        .cursor.firstBatch;
-jsTestLog("The node restarted with the following indexes on coll '" + collName +
-          "': " + tojson(indexes));
-
-// Now that the node is up and running, check that the expected code path was hit, to immediately
-// drop the index data table on startup.
-checkLog.containsJson(rst.getPrimary(), 6361201, {index: "myFantasticalIndex"});
-
-rst.stopSet();
-})();

jstests/noPassthrough/drop_connections_replSet.js

@@ -52,5 +52,8 @@ cfg.members.push(removedMember[0]);
 cfg.version++;
 assert.commandWorked(primary.adminCommand({replSetReconfig: cfg}));
 
+// Make sure all nodes, including the once-removed node, have the final config.
+rst.waitForConfigReplication(primary);
+
 rst.stopSet();
 })();

jstests/noPassthrough/durable_history_index_usage.js

@@ -109,13 +109,15 @@ replTest.start(
     true /* restart */);
 
 const checkLogs = function() {
-    // The index build was not yet completed at the recovery timestamp, it will be dropped and
-    // rebuilt.
-    checkLog.containsJson(primary(), 6361201, {
+    // Found index from unfinished build.
+    checkLog.containsJson(primary(), 22253, {
         index: "a_1",
         namespace: coll().getFullName(),
     });
 
+    // Resetting unfinished index.
+    checkLog.containsJson(primary(), 6987700, {namespace: coll().getFullName(), index: "a_1"});
+
     // Index build restarting.
     checkLog.containsJson(primary(), 20660);
 };

jstests/noPassthrough/floating_point_exp_contraction_off.js

@@ -0,0 +1,47 @@
+/**
+ * Validates that the server doesn't use fused multiply-add instructions (-ffp-contract=off).
+ *
+ * @tags: [
+ * multiversion_incompatible,
+ * ]
+ */
+
+(function() {
+'use strict';
+
+const conn = MongoRunner.runMongod();
+
+const coll = conn.getDB('test').getCollection('c');
+
+assert.commandWorked(coll.createIndex({loc: "2dsphere"}));
+assert.commandWorked(coll.insertOne({
+    "loc": {
+        "type": "Polygon",
+        "coordinates": [[
+            [-85.0329458713531, 41.3677690255613],
+            [-85.0296092033386, 41.3677690255613],
+            [-85.0296092033386, 41.360594065847],
+            [-85.0329458713531, 41.360594065847],
+            [-85.0329458713531, 41.3677690255613]
+        ]]
+    }
+}));
+
+// Assert that the query returns the document. If the query does not return any result, then
+// this is likely because of different rounding due to fused multiply-add instructions on platforms
+// that have native support, like arm64.
+assert.eq(
+    1,
+    coll.find({
+            "loc": {
+                "$near": {
+                    "$geometry":
+                        {"type": "Point", "coordinates": [-85.031218528747559, 41.364586470348961]},
+                    "$maxDistance": 0
+                }
+            }
+        })
+        .itcount());
+
+MongoRunner.stopMongod(conn);
+}());

jstests/noPassthrough/libs/missing_index_ident.js

@@ -76,8 +76,8 @@ const MissingIndexIdent = class {
         });
 
         // Since the index build was not yet completed at the recovery timestamp, its ident will
-        // be dropped.
-        checkLog.containsJson(conn, 6361201, {
+        // be reset.
+        checkLog.containsJson(conn, 6987700, {
             index: 'a_1',
             namespace: coll.getFullName(),
             ident: ident,

jstests/noPassthrough/lookup_metrics.js

@@ -56,19 +56,17 @@ function generateExpectedCounters(joinStrategy = lookupStrategy.nonSbe, spillToD
     let counters = db.serverStatus().metrics.query.lookup;
     assert(counters, "counters did not exist");
     let expected = Object.assign(counters);
-    expected.pipelineTotalCount = NumberLong(expected.pipelineTotalCount + 1);
-    let sbeCounters = expected.slotBasedExecutionCounters;
     switch (joinStrategy) {
         case lookupStrategy.nestedLoopJoin:
-            sbeCounters.nestedLoopJoin = NumberLong(sbeCounters.nestedLoopJoin + 1);
+            expected.nestedLoopJoin = NumberLong(expected.nestedLoopJoin + 1);
             break;
         case lookupStrategy.indexedLoopJoin:
-            sbeCounters.indexedLoopJoin = NumberLong(sbeCounters.indexedLoopJoin + 1);
+            expected.indexedLoopJoin = NumberLong(expected.indexedLoopJoin + 1);
             break;
         case lookupStrategy.hashLookup:
-            sbeCounters.hashLookup = NumberLong(sbeCounters.hashLookup + 1);
-            sbeCounters.hashLookupSpillToDisk =
-                NumberLong(sbeCounters.hashLookupSpillToDisk + spillToDisk);
+            expected.hashLookup = NumberLong(expected.hashLookup + 1);
+            expected.hashLookupSpillToDisk =
+                NumberLong(expected.hashLookupSpillToDisk + spillToDisk);
             break;
     }
     return expected;

jstests/noPassthrough/missing_index_ident_standalone_drop.js

@@ -24,6 +24,12 @@ IndexBuildTest.assertIndexes(coll, 2, ['_id_', 'a_1']);
 assert.commandWorked(standalone.getDB('test')[jsTestName()].dropIndex('a_1'));
 IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
 
+// Completing drop for index table immediately.
+checkLog.containsJson(standalone, 6361201, {
+    index: 'a_1',
+    namespace: coll.getFullName(),
+});
+
 MongoRunner.stopMongod(standalone);
 replTest.start(0, undefined, true /* restart */);
 IndexBuildTest.assertIndexes(replTest.getPrimary().getDB('test')[jsTestName()], 1, ['_id_']);

jstests/noPassthrough/mr_single_reduce_optimization.js

@@ -0,0 +1,29 @@
+(function() {
+"use strict";
+// See SERVER-68766. Verify that the reduce function is not run on a single value if the relevant
+// flag is enabled.
+
+const conn = MongoRunner.runMongod({setParameter: {mrEnableSingleReduceOptimization: true}});
+const testDB = conn.getDB('foo');
+const coll = testDB.bar;
+
+assert.commandWorked(coll.insert({x: 1}));
+
+const map = function() {
+    emit(0, "mapped value");
+};
+
+const reduce = function(key, values) {
+    return "reduced value";
+};
+
+let res = assert.commandWorked(
+    testDB.runCommand({mapReduce: 'bar', map: map, reduce: reduce, out: {inline: 1}}));
+assert.eq(res.results[0], {_id: 0, value: "mapped value"});
+assert.commandWorked(coll.insert({x: 2}));
+res = assert.commandWorked(
+    testDB.runCommand({mapReduce: 'bar', map: map, reduce: reduce, out: {inline: 1}}));
+assert.eq(res.results[0], {_id: 0, value: "reduced value"});
+
+MongoRunner.stopMongod(conn);
+}());

jstests/noPassthrough/retryable_writes_operation_metrics.js

@@ -0,0 +1,146 @@
+/**
+ * Tests resource consumption metrics for retryable writes.
+ * Retryable writes persist transaction documents in "config.transaction" and all reads and writes
+ * to "config.transaction" should not trigger any operation metrics.
+ *
+ * @tags: [
+ *   requires_replication,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/retryable_writes_util.js");
+
+function setupReplicaSet() {
+    var rst = new ReplSetTest({
+        nodes: 2,
+        nodeOptions: {setParameter: {"aggregateOperationResourceConsumptionMetrics": true}}
+    });
+
+    rst.startSet();
+    rst.initiate();
+    return rst;
+}
+
+function clearMetrics(conn) {
+    conn.getDB('admin').aggregate([{$operationMetrics: {clearMetrics: true}}]);
+}
+
+function getMetrics(conn) {
+    const cursor = conn.getDB('admin').aggregate([{$operationMetrics: {}}]);
+
+    let allMetrics = {};
+    while (cursor.hasNext()) {
+        let doc = cursor.next();
+        allMetrics[doc.db] = doc;
+    }
+    return allMetrics;
+}
+
+function assertMetricsZeroRead(metrics, dbName) {
+    assert(metrics[dbName]);
+    assert.eq(metrics[dbName].primaryMetrics.docBytesRead, 0);
+    assert.eq(metrics[dbName].primaryMetrics.docUnitsRead, 0);
+    assert.eq(metrics[dbName].primaryMetrics.idxEntryBytesRead, 0);
+    assert.eq(metrics[dbName].primaryMetrics.idxEntryUnitsRead, 0);
+}
+
+function assertMetricsWritten(metrics, dbName, expectedWritten) {
+    assert(metrics[dbName]);
+    assert.eq(metrics[dbName].docBytesWritten, expectedWritten.docBytesWritten);
+    assert.eq(metrics[dbName].docUnitsWritten, expectedWritten.docUnitsWritten);
+    assert.eq(metrics[dbName].idxEntryBytesWritten, expectedWritten.idxEntryBytesWritten);
+    assert.eq(metrics[dbName].idxEntryUnitsWritten, expectedWritten.idxEntryUnitsWritten);
+    assert.eq(metrics[dbName].totalUnitsWritten, expectedWritten.totalUnitsWritten);
+}
+
+function runRetryableTransaction(sessionDb, txnNumber, cmdObj) {
+    let cmd = Object.assign(
+        {}, cmdObj, {txnNumber: NumberLong(txnNumber), startTransaction: true, autocommit: false});
+
+    assert.commandWorked(sessionDb.runCommand(cmd));
+    assert.commandWorked(sessionDb.adminCommand({
+        commitTransaction: 1,
+        txnNumber: NumberLong(txnNumber),
+        autocommit: false,
+        writeConcern: {w: "majority"}
+    }));
+}
+
+function runRetryableWriteCmd(sessionDb, txnNumber, cmdObj) {
+    let cmd = Object.assign({}, cmdObj, {txnNumber: NumberLong(txnNumber)});
+    jsTest.log("run retryable write with command :" + tojson(cmd));
+    assert.commandWorked(sessionDb.runCommand(cmd));
+}
+
+const rst = setupReplicaSet();
+const primary = rst.getPrimary();
+const kDbName = "testDb";
+const kCollName = "testColl";
+const testDb = primary.getDB(kDbName);
+
+assert.commandWorked(testDb.createCollection(kCollName));
+
+const session = testDb.getMongo().startSession();
+let sessionDB = session.getDatabase(kDbName);
+let txnNumber = 0;
+
+let makeInsertCmdObj = (docs) => {
+    return {insert: kCollName, documents: docs, ordered: false};
+};
+
+let makeDocs = (fromId, toId) => {
+    let docs = [];
+    for (let i = fromId; i <= toId; i++) {
+        docs.push({_id: i, number: i});
+    }
+    return docs;
+};
+
+let expectedWritten;
+
+jsTest.log("Tests non-retryable insert comamnd which has no transaction number.");
+{
+    clearMetrics(primary);
+    let cmdObj = makeInsertCmdObj(makeDocs(1, 3));
+    assert.commandWorked(testDb.runCommand(cmdObj));
+    let metrics = getMetrics(primary);
+
+    assertMetricsZeroRead(metrics, kDbName);
+
+    // Init the expected doc, index and total data written. The following retryable writes
+    // should have the same output as the size of inserted user data is same.
+    expectedWritten = {
+        docBytesWritten: metrics[kDbName].docBytesWritten,
+        docUnitsWritten: metrics[kDbName].docUnitsWritten,
+        idxEntryBytesWritten: metrics[kDbName].idxEntryBytesWritten,
+        idxEntryUnitsWritten: metrics[kDbName].idxEntryUnitsWritten,
+        totalUnitsWritten: metrics[kDbName].totalUnitsWritten
+    };
+}
+
+jsTest.log("Tests retryable commitTransaction command which inserts documents.");
+{
+    clearMetrics(primary);
+    let cmdObj = makeInsertCmdObj(makeDocs(4, 6));
+    runRetryableTransaction(sessionDB, txnNumber, cmdObj);
+    let metrics = getMetrics(primary);
+    assertMetricsZeroRead(metrics, kDbName);
+    assertMetricsWritten(metrics, kDbName, expectedWritten);
+}
+
+txnNumber++;
+jsTest.log("Tests retryable insert command.");
+{
+    clearMetrics(primary);
+    let cmdObj = makeInsertCmdObj(makeDocs(7, 9));
+    runRetryableWriteCmd(sessionDB, txnNumber, cmdObj);
+    let metrics = getMetrics(primary);
+    assertMetricsZeroRead(metrics, kDbName);
+    assertMetricsWritten(metrics, kDbName, expectedWritten);
+}
+
+session.endSession();
+rst.stopSet();
+}());

jstests/noPassthrough/sharding_index_uniqueness.js

@@ -0,0 +1,64 @@
+/**
+ * Tests that an index with unique or prepareUnique option cannot be built on a sharded collection
+ * with an incompatible shard key, or the collection cannot be sharded with an incompatible
+ * unique/prepareUnique index.
+ *
+ * @tags: [
+ *   # TODO(SERVER-61182): Fix WiredTigerKVEngine::alterIdentMetadata() under inMemory.
+ *   requires_persistence,
+ * ]
+ */
+
+(function() {
+"use strict";
+
+const st = new ShardingTest({shards: 1, mongos: 1});
+const dbName = "db";
+const db = st.getDB(dbName);
+const collNamePrefix = jsTestName();
+let count = 0;
+
+assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+
+// Cannot shard the collection with a conflicting prepareUnique index.
+let coll = db[collNamePrefix + count++];
+assert.commandWorked(coll.createIndex({a: 1}, {prepareUnique: 1}));
+assert.commandFailedWithCode(st.s.adminCommand({shardCollection: coll.getFullName(), key: {b: 1}}),
+                             ErrorCodes.InvalidOptions);
+// Can shard the collection with a compatible prepareUnique index.
+assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {a: 1}}));
+
+// Cannot shard the collection with a conflicting unique index.
+coll = db[collNamePrefix + count++];
+assert.commandWorked(coll.createIndex({a: 1}, {unique: 1}));
+assert.commandFailedWithCode(st.s.adminCommand({shardCollection: coll.getFullName(), key: {b: 1}}),
+                             ErrorCodes.InvalidOptions);
+// Can shard the collection with a compatible unique index.
+assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {a: 1}}));
+
+// Cannot create the index with a conflicting shard key.
+coll = db[collNamePrefix + count++];
+assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {b: 1}}));
+assert.commandFailedWithCode(coll.createIndex({a: 1}, {prepareUnique: 1}),
+                             ErrorCodes.CannotCreateIndex);
+assert.commandFailedWithCode(coll.createIndex({a: 1}, {unique: 1}), ErrorCodes.CannotCreateIndex);
+// Can create the index with a compatible shard key.
+assert.commandWorked(coll.createIndex({b: 1, a: 1}, {prepareUnique: 1}));
+assert.commandWorked(coll.createIndex({b: 1, c: 1}, {unique: 1}));
+
+// Cannot convert an index to prepareUnique with a conflicting shard key.
+coll = db[collNamePrefix + count++];
+assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {b: 1}}));
+assert.commandWorked(coll.createIndex({a: 1}));
+assert.commandFailedWithCode(
+    db.runCommand({collMod: coll.getName(), index: {keyPattern: {a: 1}, prepareUnique: true}}),
+    ErrorCodes.InvalidOptions);
+// Can convert an index to prepareUnique and unique with a compatible shard key.
+assert.commandWorked(coll.createIndex({b: 1, d: 1}));
+assert.commandWorked(db.runCommand(
+    {collMod: coll.getName(), index: {keyPattern: {b: 1, d: 1}, prepareUnique: true}}));
+assert.commandWorked(
+    db.runCommand({collMod: coll.getName(), index: {keyPattern: {b: 1, d: 1}, unique: true}}));
+
+st.stop();
+})();

jstests/noPassthrough/timeseries_extended_range_rollback.js

@@ -65,11 +65,6 @@ rollbackTest.transitionToSyncSourceOperationsDuringRollback();
 rollbackTest.transitionToSteadyStateOperations();
 
 // Make sure the collections get flagged properly again during rollback.
-assert(checkLog.checkContainsWithCountJson(
-    rollbackNode, 6679402, {"nss": "test.standard", "timeField": "time"}, 0));
-assert(checkLog.checkContainsWithCountJson(
-    rollbackNode, 6679402, {"nss": "test.extended", "timeField": "time"}, 2));
-
 assert.eq(1, getExtendedRangeCount(rollbackNode));
 
 rollbackTest.stop();

jstests/noPassthrough/timeseries_extended_range_startup.js

@@ -54,11 +54,6 @@ assert.eq(1, primaryDB.standard.count());
 assert.eq(1, primaryDB.extended.count());
 
 // Make sure the collections get flagged properly again after startup.
-assert(checkLog.checkContainsWithCountJson(
-    primary, 6679402, {"nss": "testDB.standard", "timeField": "time"}, 0));
-assert(checkLog.checkContainsWithCountJson(
-    primary, 6679402, {"nss": "testDB.extended", "timeField": "time"}, 1));
-
 assert.eq(1, getExtendedRangeCount(primary));
 
 rst.stopSet();