SERVER-72276 Removing outdated entries for multiversion tests

(cherry picked from commit 6c1b682951)
(cherry picked from commit bc4e993a11)

.pydocstyle

@@ -9,6 +9,10 @@ inherit = false
 # D212 - Multi-line docstring summary should start at the first line
 # D213 - Multi-line docstring summary should start at the second line
 # D301 - Use r""" if any backslashes in a docstring
-ignore = D105,D202,D203,D212,D213,D301
+# D401 - First line should be in imperative mood
+# D407 - Missing dashed underline after section
+# D413 - Missing blank line after last section
+# D415 - First line should end with a period, question mark, or exclamation point
+ignore = D105,D202,D203,D212,D213,D301,D401,D407,D413,D415
 # Do not run on buildscripts/tests/
 match = ^((?!buildscripts[\\\/]tests[\\\/]).)*$

README.md

@@ -90,3 +90,4 @@ Welcome to MongoDB!
   October 16, 2018, including patch fixes for prior versions, are published
   under the [Server Side Public License (SSPL) v1](LICENSE-Community.txt).
   See individual files for details.
+

SConstruct

@@ -2,8 +2,8 @@
 
 import atexit
 import copy
-import datetime
 import errno
+import functools
 import json
 import os
 import re
@@ -31,6 +31,7 @@ import mongo.platform as mongo_platform
 import mongo.toolchain as mongo_toolchain
 import mongo.generators as mongo_generators
 import mongo.install_actions as install_actions
+from mongo.build_profiles import BUILD_PROFILES
 
 EnsurePythonVersion(3, 6)
 EnsureSConsVersion(3, 1, 1)
@@ -105,9 +106,23 @@ SetOption('random', 1)
 #   using the nargs='const' mechanism.
 #
 
-add_option('ninja',
+add_option(
+    'build-profile',
+    choices=list(BUILD_PROFILES.keys()),
+    default='default',
+    type='choice',
+    help='''Short hand for common build options. These profiles are well supported by SDP and are
+    kept up to date. Unless you need something specific, it is recommended that you only build with
+    these. san is the recommeneded profile since it exposes bugs before they are found in patch
+    builds. Check out site_scons/mongo/build_profiles.py to see each profile.''',
+)
+
+build_profile = BUILD_PROFILES[get_option('build-profile')]
+
+add_option(
+    'ninja',
     choices=['enabled', 'disabled'],
-    default='disabled',
+    default=build_profile.ninja,
     nargs='?',
     const='enabled',
     type='choice',
@@ -152,8 +167,9 @@ add_option('install-mode',
 
 add_option('install-action',
     choices=([*install_actions.available_actions] + ['default']),
-    default='default',
-    help='select mechanism to use to install files (advanced option to reduce disk IO and utilization)',
+    default='hardlink',
+    help=
+    'select mechanism to use to install files (advanced option to reduce disk IO and utilization)',
     nargs=1,
     type='choice',
 )
@@ -245,7 +261,7 @@ add_option('noshell',
 add_option('dbg',
     choices=['on', 'off'],
     const='on',
-    default='off',
+    default=build_profile.dbg,
     help='Enable runtime debugging checks',
     nargs='?',
     type='choice',
@@ -270,8 +286,9 @@ add_option('spider-monkey-dbg',
 )
 
 add_option('opt',
-    choices=['on', 'size', 'off'],
+    choices=['on', 'size', 'off', 'auto'],
     const='on',
+    default=build_profile.opt,
     help='Enable compile-time optimization',
     nargs='?',
     type='choice',
@@ -312,6 +329,7 @@ add_option('debug-compress',
 add_option('sanitize',
     help='enable selected sanitizers',
     metavar='san1,san2,...sanN',
+    default=build_profile.sanitize,
 )
 
 add_option('sanitize-coverage',
@@ -321,7 +339,7 @@ add_option('sanitize-coverage',
 
 add_option('allocator',
     choices=["auto", "system", "tcmalloc", "tcmalloc-experimental"],
-    default="auto",
+    default=build_profile.allocator,
     help='allocator to use (use "auto" for best choice for current platform)',
     type='choice',
 )
@@ -426,8 +444,9 @@ add_option("disable-warnings-as-errors",
     action="append",
     choices=["configure", "source"],
     const="source",
-    default=[],
-    help="Don't add a warnings-as-errors flag to compiler command lines in selected contexts; defaults to 'source' if no argument is provided",
+    default=build_profile.disable_warnings_as_errors,
+    help=
+    "Don't add a warnings-as-errors flag to compiler command lines in selected contexts; defaults to 'source' if no argument is provided",
     nargs="?",
     type="choice",
 )
@@ -494,8 +513,9 @@ def find_mongo_custom_variables():
             files.append(probe)
     return files
 
-add_option('variables-files',
-    default=[],
+add_option(
+    'variables-files',
+    default=build_profile.variables_files,
     action="append",
     help="Specify variables files to load.",
 )
@@ -503,7 +523,7 @@ add_option('variables-files',
 link_model_choices = ['auto', 'object', 'static', 'dynamic', 'dynamic-strict', 'dynamic-sdk']
 add_option('link-model',
     choices=link_model_choices,
-    default='auto',
+    default=build_profile.link_model,
     help='Select the linking model for the project',
     type='choice'
 )
@@ -777,8 +797,11 @@ env_vars.Add('ARFLAGS',
     help='Sets flags for the archiver',
     converter=variable_shlex_converter)
 
-env_vars.Add('CCACHE',
-    help='Tells SCons where the ccache binary is')
+env_vars.Add(
+    'CCACHE',
+    help='Tells SCons where the ccache binary is',
+    default=build_profile.CCACHE,
+)
 
 env_vars.Add(
     'CACHE_SIZE',
@@ -825,16 +848,11 @@ env_vars.Add('CXXFLAGS',
     help='Sets flags for the C++ compiler',
     converter=variable_shlex_converter)
 
-default_destdir = '$BUILD_ROOT/install'
-if get_option('ninja') != 'disabled':
-    # Workaround for SERVER-53952 where issues wih different
-    # ninja files building to the same install dir. Different
-    # ninja files need to build to different install dirs.
-    default_destdir = '$BUILD_DIR/install'
-
-env_vars.Add('DESTDIR',
+env_vars.Add(
+    'DESTDIR',
     help='Where builds will install files',
-    default=default_destdir)
+    default='$BUILD_ROOT/install',
+)
 
 env_vars.Add('DSYMUTIL',
     help='Path to the dsymutil utility',
@@ -870,8 +888,11 @@ env_vars.Add('HOST_ARCH',
     converter=variable_arch_converter,
     default=None)
 
-env_vars.Add('ICECC',
-    help='Tells SCons where icecream icecc tool is')
+env_vars.Add(
+    'ICECC',
+    help='Tells SCons where icecream icecc tool is',
+    default=build_profile.ICECC,
+)
 
 env_vars.Add('ICERUN',
     help='Tells SCons where icecream icerun tool is')
@@ -962,8 +983,14 @@ env_vars.Add('MSVC_VERSION',
     help='Sets the version of Visual C++ to use (e.g. 14.2 for VS2019, 14.3 for VS2022)',
     default="14.3")
 
-env_vars.Add('NINJA_PREFIX',
-    default="build",
+env_vars.Add('NINJA_BUILDDIR',
+    help="Location for shared Ninja state",
+    default="$BUILD_ROOT/ninja",
+)
+
+env_vars.Add(
+    'NINJA_PREFIX',
+    default=build_profile.NINJA_PREFIX,
     help="""A prefix to add to the beginning of generated ninja
 files. Useful for when compiling multiple build ninja files for
 different configurations, for instance:
@@ -976,7 +1003,7 @@ Will generate the files (respectively):
     asan.ninja
     tsan.ninja
 
-Defaults to build. Best used with the generate-ninja alias so you don't have to
+Defaults to build. Best used with the --ninja flag so you don't have to
 reiterate the prefix in the target name and variable.
 """)
 
@@ -1067,7 +1094,7 @@ env_vars.Add('TOOLS',
 
 env_vars.Add('VARIANT_DIR',
     help='Sets the name (or generator function) for the variant directory',
-    default=mongo_generators.default_variant_dir_generator,
+    default=build_profile.VARIANT_DIR,
 )
 
 env_vars.Add('VERBOSE',
@@ -1156,6 +1183,8 @@ dbg_opt_mapping = {
     ( "off", "off" ) : ( False, False ),
     ( "on",  "size"  ) : ( True,  True ),
     ( "off", "size"  ) : ( False, True ),
+    ( "on",  "auto"  ) : ( True,  False ),
+    ( "off", "auto"  ) : ( False, True ),
 }
 debugBuild, optBuild = dbg_opt_mapping[(get_option('dbg'), get_option('opt'))]
 optBuildForSize = True if optBuild and get_option('opt') == "size" else False
@@ -1207,7 +1236,6 @@ envDict = dict(BUILD_ROOT=buildDir,
                LIBDEPS_TAG_EXPANSIONS=[],
                )
 
-
 # By default, we will get the normal SCons tool search. But if the
 # user has opted into the next gen tools, add our experimental tool
 # directory into the default toolpath, ahead of whatever is already in
@@ -1319,8 +1347,6 @@ unknown_vars = env_vars.UnknownVariables()
 if unknown_vars:
     env.FatalError("Unknown variables specified: {0}", ", ".join(list(unknown_vars.keys())))
 
-if get_option('install-action') != 'default' and get_option('ninja') != "disabled":
-    env.FatalError("Cannot use non-default install actions when generating Ninja.")
 install_actions.setup(env, get_option('install-action'))
 
 def set_config_header_define(env, varname, varval = 1):
@@ -2089,6 +2115,15 @@ for suboption in get_option('experimental-runtime-hardening'):
     elif suboption.startswith('+'):
         selected_experimental_runtime_hardenings.add(suboption[1:])
 
+# Disable floating-point contractions such as forming of fused multiply-add operations.
+if env.ToolchainIs('clang', 'gcc'):
+    env.Append(CCFLAGS=["-ffp-contract=off"])
+else:
+    # msvc defaults to /fp:precise. Visual Studio 2022 does not emit floating-point contractions
+    # with /fp:precise, but previous versions can. Disable contractions altogether by using
+    # /fp:strict.
+    env.Append(CCFLAGS=["/fp:strict"])
+
 if env.TargetOSIs('linux'):
     env.Append( LIBS=["m"] )
     if not env.TargetOSIs('android'):
@@ -2619,13 +2654,25 @@ if get_option("system-boost-lib-search-suffixes") is not None:
 # discover modules, and load the (python) module for each module's build.py
 mongo_modules = moduleconfig.discover_modules('src/mongo/db/modules', get_option('modules'))
 
-if get_option('ninja') != 'disabled':
-    for module in mongo_modules:
-        if hasattr(module, 'NinjaFile'):
-            env.FatalError(textwrap.dedent("""\
-                ERROR: Ninja tool option '--ninja' should not be used with the ninja module.
-                    Remove the ninja module directory or use '--modules= ' to select no modules.
-                    If using enterprise module, explicitly set '--modules=<name-of-enterprise-module>' to exclude the ninja module."""))
+has_ninja_module = False
+for module in mongo_modules:
+    if hasattr(module, 'NinjaFile'):
+        has_ninja_module = True
+        break
+
+if get_option('ninja') != 'disabled' and has_ninja_module:
+    env.FatalError(
+        textwrap.dedent("""\
+        ERROR: Ninja tool option '--ninja' should not be used with the ninja module.
+            Using both options simultaneously may clobber build.ninja files.
+            Remove the ninja module directory or use '--modules= ' to select no modules.
+            If using enterprise module, explicitly set '--modules=<name-of-enterprise-module>' to exclude the ninja module."""
+                        ))
+
+if has_ninja_module:
+    print(
+        "WARNING: You are attempting to use the unsupported/legacy ninja module, instead of the integrated ninja generator. You are strongly encouraged to remove the ninja module from your module list and invoke scons with --ninja generate-ninja"
+    )
 
 # --- check system ---
 ssl_provider = None
@@ -2821,7 +2868,7 @@ def doConfigure(myenv):
 
         conf.Finish()
 
-    def AddFlagIfSupported(env, tool, extension, flag, link, **mutation):
+    def CheckFlag(env, flag, tool, extension, link, **mutation):
         def CheckFlagTest(context, tool, extension, flag):
             if link:
                 if tool == 'C':
@@ -2893,24 +2940,33 @@ def doConfigure(myenv):
         })
         available = conf.CheckFlag()
         conf.Finish()
+        return available
+
+    def AddFlagIfSupported(env, flag, tool, extension, link, **mutation):
+
+        available = CheckFlag(env, flag, tool, extension, link, **mutation)
+
         if available:
             env.Append(**mutation)
         return available
 
-    def AddToCFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, False, CFLAGS=[flag])
+    conf_check_vars = {
+        'CFLAGS': {'tool': 'C', 'extension': '.c', 'link': False},
+        'CCFLAGS': {'tool': 'C', 'extension': '.c', 'link': False},
+        'CXXFLAGS': {'tool': 'C++', 'extension': '.cpp', 'link': False},
+        'LINKFLAGS': {'tool': 'C', 'extension': '.c', 'link': True},
+        'SHLINKFLAGS': {'tool': 'C', 'extension': '.c', 'link': True},
+    }
 
-    def AddToCCFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, False, CCFLAGS=[flag])
+    def var_func(env, flag, var, func):
+        kwargs = dict({var: [flag]}, **conf_check_vars[var])
+        return func(env, flag, **kwargs)
 
-    def AddToCXXFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C++', '.cpp', flag, False, CXXFLAGS=[flag])
-
-    def AddToLINKFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, True, LINKFLAGS=[flag])
-
-    def AddToSHLINKFLAGSIfSupported(env, flag):
-        return AddFlagIfSupported(env, 'C', '.c', flag, True, SHLINKFLAGS=[flag])
+    for var in conf_check_vars:
+        myenv.AddMethod(
+            functools.partial(var_func, var=var, func=AddFlagIfSupported), f"AddTo{var}IfSupported")
+        myenv.AddMethod(
+            functools.partial(var_func, var=var, func=CheckFlag), f"Check{var}Supported")
 
     if myenv.ToolchainIs('gcc', 'clang'):
         # This tells clang/gcc to use the gold linker if it is available - we prefer the gold linker
@@ -2933,20 +2989,20 @@ def doConfigure(myenv):
             #
             # We should revisit all of these issues the next time we upgrade our clang minimum.
             if get_option('separate-debug') == 'off' and get_option('link-model') != 'dynamic':
-                if not AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=lld'):
-                    AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=gold')
+                if not myenv.AddToLINKFLAGSIfSupported('-fuse-ld=lld'):
+                    myenv.AddToLINKFLAGSIfSupported('-fuse-ld=gold')
             else:
-                AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=gold')
+                myenv.AddToLINKFLAGSIfSupported('-fuse-ld=gold')
         elif link_model.startswith("dynamic") and linker_ld == 'bfd':
             # BFD is not supported due to issues with it causing warnings from some of
             # the third party libraries that mongodb is linked with:
             # https://jira.mongodb.org/browse/SERVER-49465
             myenv.FatalError(f"Linker {linker_ld} is not supported with dynamic link model builds.")
         else:
-            if not AddToLINKFLAGSIfSupported(myenv, f'-fuse-ld={linker_ld}'):
+            if not myenv.AddToLINKFLAGSIfSupported(f'-fuse-ld={linker_ld}'):
                 myenv.FatalError(f"Linker {linker_ld} could not be configured.")
 
-        if has_option('gcov') and AddToCCFLAGSIfSupported(myenv, '-fprofile-update=single'):
+        if has_option('gcov') and myenv.AddToCCFLAGSIfSupported('-fprofile-update=single'):
             myenv.AppendUnique(LINKFLAGS=['-fprofile-update=single'])
 
     detectCompiler = Configure(myenv, help=False, custom_tests = {
@@ -2973,11 +3029,11 @@ def doConfigure(myenv):
 
     if myenv.ToolchainIs('clang', 'gcc'):
         # This warning was added in g++-4.8.
-        AddToCCFLAGSIfSupported(myenv, '-Wno-unused-local-typedefs')
+        myenv.AddToCCFLAGSIfSupported('-Wno-unused-local-typedefs')
 
         # Clang likes to warn about unused functions, which seems a tad aggressive and breaks
         # -Werror, which we want to be able to use.
-        AddToCCFLAGSIfSupported(myenv, '-Wno-unused-function')
+        myenv.AddToCCFLAGSIfSupported('-Wno-unused-function')
 
         # TODO: Note that the following two flags are added to CCFLAGS even though they are
         # really C++ specific. We need to do this because SCons passes CXXFLAGS *before*
@@ -2987,79 +3043,79 @@ def doConfigure(myenv):
         #
         # Clang likes to warn about unused private fields, but some of our third_party
         # libraries have such things.
-        AddToCCFLAGSIfSupported(myenv, '-Wno-unused-private-field')
+        myenv.AddToCCFLAGSIfSupported('-Wno-unused-private-field')
 
         # Prevents warning about using deprecated features (such as auto_ptr in c++11)
         # Using -Wno-error=deprecated-declarations does not seem to work on some compilers,
         # including at least g++-4.6.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-deprecated-declarations")
+        myenv.AddToCCFLAGSIfSupported("-Wno-deprecated-declarations")
 
         # As of clang-3.4, this warning appears in v8, and gets escalated to an error.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-constant-out-of-range-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-constant-out-of-range-compare")
 
         # As of clang in Android NDK 17, these warnings appears in boost and/or ICU, and get escalated to errors
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-constant-compare")
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-unsigned-zero-compare")
-        AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-unsigned-enum-zero-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-constant-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-unsigned-zero-compare")
+        myenv.AddToCCFLAGSIfSupported("-Wno-tautological-unsigned-enum-zero-compare")
 
         # New in clang-3.4, trips up things mostly in third_party, but in a few places in the
         # primary mongo sources as well.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-unused-const-variable")
+        myenv.AddToCCFLAGSIfSupported("-Wno-unused-const-variable")
 
         # Prevents warning about unused but set variables found in boost version 1.49
         # in boost/date_time/format_date_parser.hpp which does not work for compilers
         # GCC >= 4.6. Error explained in https://svn.boost.org/trac/boost/ticket/6136 .
-        AddToCCFLAGSIfSupported(myenv, "-Wno-unused-but-set-variable")
+        myenv.AddToCCFLAGSIfSupported("-Wno-unused-but-set-variable")
 
         # This has been suppressed in gcc 4.8, due to false positives, but not in clang.  So
         # we explicitly disable it here.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-missing-braces")
+        myenv.AddToCCFLAGSIfSupported("-Wno-missing-braces")
 
         # Suppress warnings about not consistently using override everywhere in a class. It seems
         # very pedantic, and we have a fair number of instances.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-inconsistent-missing-override")
+        myenv.AddToCCFLAGSIfSupported("-Wno-inconsistent-missing-override")
 
         # Don't issue warnings about potentially evaluated expressions
-        AddToCCFLAGSIfSupported(myenv, "-Wno-potentially-evaluated-expression")
+        myenv.AddToCCFLAGSIfSupported("-Wno-potentially-evaluated-expression")
 
         # Warn about moves of prvalues, which can inhibit copy elision.
-        AddToCXXFLAGSIfSupported(myenv, "-Wpessimizing-move")
+        myenv.AddToCXXFLAGSIfSupported("-Wpessimizing-move")
 
         # Disable warning about variables that may not be initialized
         # Failures are triggered in the case of boost::optional in GCC 4.8.x
         # TODO: re-evaluate when we move to GCC 5.3
         # see: http://stackoverflow.com/questions/21755206/how-to-get-around-gcc-void-b-4-may-be-used-uninitialized-in-this-funct
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-maybe-uninitialized")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-maybe-uninitialized")
 
         # Disable warning about templates that can't be implicitly instantiated. It is an attempt to
         # make a link error into an easier-to-debug compiler failure, but it triggers false
         # positives if explicit instantiation is used in a TU that can see the full definition. This
         # is a problem at least for the S2 headers.
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-undefined-var-template")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-undefined-var-template")
 
         # This warning was added in clang-4.0, but it warns about code that is required on some
         # platforms. Since the warning just states that 'explicit instantiation of [a template] that
         # occurs after an explicit specialization has no effect', it is harmless on platforms where
         # it isn't required
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-instantiation-after-specialization")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-instantiation-after-specialization")
 
         # This warning was added in clang-5 and flags many of our lambdas. Since it isn't actively
         # harmful to capture unused variables we are suppressing for now with a plan to fix later.
-        AddToCCFLAGSIfSupported(myenv, "-Wno-unused-lambda-capture")
+        myenv.AddToCCFLAGSIfSupported("-Wno-unused-lambda-capture")
 
         # Enable sized deallocation support.
-        AddToCXXFLAGSIfSupported(myenv, '-fsized-deallocation')
+        myenv.AddToCXXFLAGSIfSupported('-fsized-deallocation')
 
         # This warning was added in Apple clang version 11 and flags many explicitly defaulted move
         # constructors and assignment operators for being implicitly deleted, which is not useful.
-        AddToCXXFLAGSIfSupported(myenv, "-Wno-defaulted-function-deleted")
+        myenv.AddToCXXFLAGSIfSupported("-Wno-defaulted-function-deleted")
 
         # SERVER-44856: Our windows builds complain about unused
         # exception parameters, but GCC and clang don't seem to do
         # that for us automatically. In the interest of making it more
         # likely to catch these errors early, add the (currently clang
         # only) flag that turns it on.
-        AddToCXXFLAGSIfSupported(myenv, "-Wunused-exception-parameter")
+        myenv.AddToCXXFLAGSIfSupported("-Wunused-exception-parameter")
 
         # TODO(SERVER-60151): Avoid the dilemma identified in
         # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100493. Unfortunately,
@@ -3071,7 +3127,7 @@ def doConfigure(myenv):
         # TODO(SERVER-60175): In fact we will want to explicitly opt
         # in to -Wdeprecated, since clang doesn't include it in -Wall.
         if get_option('cxx-std') == "20":
-            AddToCXXFLAGSIfSupported(myenv, '-Wno-deprecated')
+            myenv.AddToCXXFLAGSIfSupported('-Wno-deprecated')
 
         # Check if we can set "-Wnon-virtual-dtor" when "-Werror" is set. The only time we can't set it is on
         # clang 3.4, where a class with virtual function(s) and a non-virtual destructor throws a warning when
@@ -3115,25 +3171,21 @@ def doConfigure(myenv):
         # by -Wall), in order to enforce that -mXXX-version-min=YYY
         # will enforce that you don't use APIs from ZZZ.
         if env.TargetOSIs('darwin'):
-            AddToCCFLAGSIfSupported(env, '-Wunguarded-availability')
+            env.AddToCCFLAGSIfSupported('-Wunguarded-availability')
 
     if get_option('runtime-hardening') == "on":
         # Enable 'strong' stack protection preferentially, but fall back to 'all' if it is not
         # available. Note that we need to add these to the LINKFLAGS as well, since otherwise we
         # might not link libssp when we need to (see SERVER-12456).
         if myenv.ToolchainIs('gcc', 'clang'):
-            if AddToCCFLAGSIfSupported(myenv, '-fstack-protector-strong'):
-                myenv.Append(
-                    LINKFLAGS=[
-                        '-fstack-protector-strong',
-                    ]
-                )
-            elif AddToCCFLAGSIfSupported(myenv, '-fstack-protector-all'):
-                myenv.Append(
-                    LINKFLAGS=[
-                        '-fstack-protector-all',
-                    ]
-                )
+            if myenv.AddToCCFLAGSIfSupported('-fstack-protector-strong'):
+                myenv.Append(LINKFLAGS=[
+                    '-fstack-protector-strong',
+                ], )
+            elif myenv.AddToCCFLAGSIfSupported('-fstack-protector-all'):
+                myenv.Append(LINKFLAGS=[
+                    '-fstack-protector-all',
+                ], )
 
             if 'cfex' in selected_experimental_runtime_hardenings:
                 myenv.Append(
@@ -3141,10 +3193,10 @@ def doConfigure(myenv):
                 )
 
             if 'stackclash' in selected_experimental_runtime_hardenings:
-                AddToCCFLAGSIfSupported(myenv, "-fstack-clash-protection")
+                myenv.AddToCCFLAGSIfSupported("-fstack-clash-protection")
 
             if 'controlflow' in selected_experimental_runtime_hardenings:
-                AddToCCFLAGSIfSupported(myenv, "-fcf-protection=full")
+                myenv.AddToCCFLAGSIfSupported("-fcf-protection=full")
 
         if myenv.ToolchainIs('clang'):
             # TODO: There are several interesting things to try here, but they each have
@@ -3187,7 +3239,7 @@ def doConfigure(myenv):
     if has_option('libc++'):
         if not myenv.ToolchainIs('clang'):
             myenv.FatalError('libc++ is currently only supported for clang')
-        if AddToCXXFLAGSIfSupported(myenv, '-stdlib=libc++'):
+        if myenv.AddToCXXFLAGSIfSupported('-stdlib=libc++'):
             myenv.Append(LINKFLAGS=['-stdlib=libc++'])
         else:
             myenv.ConfError('libc++ requested, but compiler does not support -stdlib=libc++' )
@@ -3218,14 +3270,13 @@ def doConfigure(myenv):
             myenv.AppendUnique(CCFLAGS=['/std:c++20'])
     else:
         if get_option('cxx-std') == "17":
-            if not AddToCXXFLAGSIfSupported(myenv, '-std=c++17'):
+            if not myenv.AddToCXXFLAGSIfSupported('-std=c++17'):
                 myenv.ConfError('Compiler does not honor -std=c++17')
         elif get_option('cxx-std') == "20":
-            if not AddToCXXFLAGSIfSupported(myenv, '-std=c++20'):
+            if not myenv.AddToCXXFLAGSIfSupported('-std=c++20'):
                 myenv.ConfError('Compiler does not honor -std=c++20')
 
-
-        if not AddToCFLAGSIfSupported(myenv, '-std=c11'):
+        if not myenv.AddToCFLAGSIfSupported('-std=c11'):
             myenv.ConfError("C++17 mode selected for C++ files, but can't enable C11 for C files")
 
     if using_system_version_of_cxx_libraries():
@@ -3498,7 +3549,7 @@ def doConfigure(myenv):
 
         sanitizer_option = '-fsanitize=' + ','.join(sanitizer_list)
 
-        if AddToCCFLAGSIfSupported(myenv, sanitizer_option):
+        if myenv.AddToCCFLAGSIfSupported(sanitizer_option):
             myenv.Append(LINKFLAGS=[sanitizer_option])
             myenv.Append(CCFLAGS=['-fno-omit-frame-pointer'])
         else:
@@ -3509,7 +3560,7 @@ def doConfigure(myenv):
         if has_option('sanitize-coverage') and using_fsan:
             sanitize_coverage_list = get_option('sanitize-coverage')
             sanitize_coverage_option = '-fsanitize-coverage=' + sanitize_coverage_list
-            if AddToCCFLAGSIfSupported(myenv,sanitize_coverage_option):
+            if myenv.AddToCCFLAGSIfSupported(sanitize_coverage_option):
                 myenv.Append(LINKFLAGS=[sanitize_coverage_option])
             else:
                 myenv.ConfError('Failed to enable -fsanitize-coverage with flag: {0}', sanitize_coverage_option )
@@ -3531,7 +3582,7 @@ def doConfigure(myenv):
         supportedDenyfiles = []
         denyfilesTestEnv = myenv.Clone()
         for denyfile in denyfiles:
-            if AddToCCFLAGSIfSupported(denyfilesTestEnv, f"-fsanitize-blacklist={denyfile}"):
+            if denyfilesTestEnv.AddToCCFLAGSIfSupported(f"-fsanitize-blacklist={denyfile}"):
                 supportedDenyfiles.append(denyfile)
         denyfilesTestEnv = None
         supportedDenyfiles = sorted(supportedDenyfiles)
@@ -3666,8 +3717,8 @@ def doConfigure(myenv):
             # have renamed the flag.
             # However, this flag cannot be included when using the fuzzer sanitizer
             # if we want to suppress errors to uncover new ones.
-            if not using_fsan and not AddToCCFLAGSIfSupported(myenv, "-fno-sanitize-recover"):
-                AddToCCFLAGSIfSupported(myenv, "-fno-sanitize-recover=undefined")
+            if not using_fsan and not myenv.AddToCCFLAGSIfSupported("-fno-sanitize-recover"):
+                myenv.AddToCCFLAGSIfSupported("-fno-sanitize-recover=undefined")
             myenv.AppendUnique(CPPDEFINES=['UNDEFINED_BEHAVIOR_SANITIZER'])
 
             # If anything is changed, added, or removed in ubsan_options, be
@@ -3689,7 +3740,7 @@ def doConfigure(myenv):
             # same as the correct link graph for a regular dynamic
             # build.
             if link_model == "dynamic":
-                if AddToCCFLAGSIfSupported(myenv, "-fno-sanitize=vptr"):
+                if myenv.AddToCCFLAGSIfSupported("-fno-sanitize=vptr"):
                     myenv.AppendUnique(LINKFLAGS=["-fno-sanitize=vptr"])
 
     if myenv.ToolchainIs('msvc') and optBuild:
@@ -3708,21 +3759,28 @@ def doConfigure(myenv):
         # Usually, --gdb-index is too expensive in big static binaries, but for dynamic
         # builds it works well.
         if link_model.startswith("dynamic"):
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--gdb-index')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--gdb-index')
+
+        if link_model != 'dynamic':
+            # This will create an extra section where debug types can be referred from,
+            # reducing other section sizes. This helps most with big static links as there
+            # will be lots of duplicate debug type info.
+            myenv.AddToCCFLAGSIfSupported('-fdebug-types-section')
+            myenv.AddToLINKFLAGSIfSupported('-fdebug-types-section')
 
         # Our build is already parallel.
-        AddToLINKFLAGSIfSupported(myenv, '-Wl,--no-threads')
+        myenv.AddToLINKFLAGSIfSupported('-Wl,--no-threads')
 
         # Explicitly enable GNU build id's if the linker supports it.
-        AddToLINKFLAGSIfSupported(myenv, '-Wl,--build-id')
+        myenv.AddToLINKFLAGSIfSupported('-Wl,--build-id')
 
         # Explicitly use the new gnu hash section if the linker offers
         # it, except on android since older runtimes seem to not
         # support it. For that platform, use 'both'.
         if env.TargetOSIs('android'):
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--hash-style=both')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--hash-style=both')
         else:
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--hash-style=gnu')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--hash-style=gnu')
 
         # Try to have the linker tell us about ODR violations. Don't
         # use it when using clang with libstdc++, as libstdc++ was
@@ -3734,17 +3792,17 @@ def doConfigure(myenv):
                 env.FatalError('The --detect-odr-violations flag does not work with clang and libstdc++')
             if optBuild:
                 env.FatalError('The --detect-odr-violations flag is expected to only be reliable with --opt=off')
-            AddToLINKFLAGSIfSupported(myenv, '-Wl,--detect-odr-violations')
+            myenv.AddToLINKFLAGSIfSupported('-Wl,--detect-odr-violations')
 
         # Disallow an executable stack. Also, issue a warning if any files are found that would
         # cause the stack to become executable if the noexecstack flag was not in play, so that we
         # can find them and fix them. We do this here after we check for ld.gold because the
         # --warn-execstack is currently only offered with gold.
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,-z,noexecstack")
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,--warn-execstack")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,-z,noexecstack")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,--warn-execstack")
 
         # If possible with the current linker, mark relocations as read-only.
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,-z,relro")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,-z,relro")
 
         # As far as we know these flags only apply on posix-y systems,
         # and not on Darwin.
@@ -3777,9 +3835,9 @@ def doConfigure(myenv):
             compress_type="zlib-gabi"
             compress_flag="compress-debug-sections"
 
-            AddToCCFLAGSIfSupported(
-                myenv,
-                f"-Wa,--{compress_flag}={compress_type}" if "as" in debug_compress else f"-Wa,--no{compress_flag}")
+            myenv.AddToCCFLAGSIfSupported(
+                f"-Wa,--{compress_flag}={compress_type}"
+                if "as" in debug_compress else f"-Wa,--no{compress_flag}", )
 
             # We shouldn't enable debug compression in the linker
             # (meaning our final binaries contain compressed debug
@@ -3811,20 +3869,16 @@ def doConfigure(myenv):
             conf.Finish()
 
             if have_shf_compressed and 'ld' in debug_compress:
-                AddToLINKFLAGSIfSupported(
-                    myenv,
-                    f"-Wl,--{compress_flag}={compress_type}")
+                myenv.AddToLINKFLAGSIfSupported(f"-Wl,--{compress_flag}={compress_type}", )
             else:
-                AddToLINKFLAGSIfSupported(
-                    myenv,
-                    f"-Wl,--{compress_flag}=none")
+                myenv.AddToLINKFLAGSIfSupported(f"-Wl,--{compress_flag}=none", )
 
         if "fnsi" in selected_experimental_optimizations:
-            AddToCCFLAGSIfSupported(myenv, "-fno-semantic-interposition")
+            myenv.AddToCCFLAGSIfSupported("-fno-semantic-interposition")
 
     # Avoid deduping symbols on OS X debug builds, as it takes a long time.
     if not optBuild and myenv.ToolchainIs('clang') and env.TargetOSIs('darwin'):
-        AddToLINKFLAGSIfSupported(myenv, "-Wl,-no_deduplicate")
+        myenv.AddToLINKFLAGSIfSupported("-Wl,-no_deduplicate")
 
     # Apply any link time optimization settings as selected by the 'lto' option.
     if has_option('lto'):
@@ -3841,13 +3895,13 @@ def doConfigure(myenv):
         elif myenv.ToolchainIs('gcc', 'clang'):
             # For GCC and clang, the flag is -flto, and we need to pass it both on the compile
             # and link lines.
-            if not AddToCCFLAGSIfSupported(myenv, '-flto') or \
-                    not AddToLINKFLAGSIfSupported(myenv, '-flto'):
+            if not myenv.AddToCCFLAGSIfSupported('-flto') or \
+                    not myenv.AddToLINKFLAGSIfSupported('-flto'):
                 myenv.ConfError("Link time optimization requested, "
                     "but selected compiler does not honor -flto" )
 
             if myenv.TargetOSIs('darwin'):
-                AddToLINKFLAGSIfSupported(myenv, '-Wl,-object_path_lto,${TARGET}.lto')
+                myenv.AddToLINKFLAGSIfSupported('-Wl,-object_path_lto,${TARGET}.lto')
 
         else:
             myenv.ConfError("Don't know how to enable --lto on current toolchain")
@@ -3889,11 +3943,76 @@ def doConfigure(myenv):
 
         myenv = conf.Finish()
 
+    # Our build generally assumes that we have C11-compliant libc headers for
+    # C++ source. On most systems, that will be the case. However, on systems
+    # using glibc older than 2.18 (or other libc implementations that have
+    # stubbornly refused to update), we need to add some preprocessor defines.
+    #
+    # See: https://sourceware.org/bugzilla/show_bug.cgi?id=15366
+    #
+    # These headers are only fully standards-compliant on POSIX platforms. Windows
+    # in particular doesn't implement inttypes.h
+    if env.TargetOSIs('posix'):
+        def NeedStdCLimitMacros(context):
+            test_body="""
+            #undef __STDC_LIMIT_MACROS
+            #include <stdint.h>
+            #if defined(INT64_MAX)
+            #  error
+            #endif
+            """
+            context.Message('Checking whether to define __STDC_LIMIT_MACROS... ')
+            ret = context.TryCompile(textwrap.dedent(test_body), '.cpp')
+            context.Result(ret)
+            return ret
+
+        def NeedStdCConstantMacros(context):
+            test_body="""
+            #undef __STDC_CONSTANT_MACROS
+            #include <stdint.h>
+            #if defined(INTMAX_C)
+            #  error
+            #endif
+            """
+            context.Message('Checking whether to define __STDC_CONSTANT_MACROS... ')
+            ret = context.TryCompile(textwrap.dedent(test_body), '.cpp')
+            context.Result(ret)
+            return ret
+
+        def NeedStdCFormatMacros(context):
+            test_body="""
+            #undef __STDC_FORMAT_MACROS
+            #include <inttypes.h>
+            #if defined(PRIx64)
+            #  error
+            #endif
+            """
+            context.Message('Checking whether to define __STDC_FORMAT_MACROS... ')
+            ret = context.TryCompile(textwrap.dedent(test_body), '.cpp')
+            context.Result(ret)
+            return ret
+
+        conf = Configure(myenv, help=False, custom_tests = {
+            'NeedStdCLimitMacros': NeedStdCLimitMacros,
+            'NeedStdCConstantMacros': NeedStdCConstantMacros,
+            'NeedStdCFormatMacros': NeedStdCFormatMacros,
+        })
+
+        conf.env.AppendUnique(
+            CPPDEFINES=[
+                '__STDC_LIMIT_MACROS' if conf.NeedStdCLimitMacros() else '',
+                '__STDC_CONSTANT_MACROS' if conf.NeedStdCConstantMacros() else '',
+                '__STDC_FORMAT_MACROS' if conf.NeedStdCFormatMacros() else '',
+            ]
+        )
+
+        myenv = conf.Finish()
+
     # We set this with GCC on x86 platforms to work around
     # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052
     if myenv.ToolchainIs('gcc') and (env['TARGET_ARCH'] in ['i386', 'x86_64']):
         if not 'builtin-memcmp' in selected_experimental_optimizations:
-            AddToCCFLAGSIfSupported(myenv, '-fno-builtin-memcmp')
+            myenv.AddToCCFLAGSIfSupported('-fno-builtin-memcmp')
 
     # pthread_setname_np was added in GLIBC 2.12, and Solaris 11.3
     if posix_system:
@@ -4522,7 +4641,7 @@ def doConfigure(myenv):
     myenv = conf.Finish()
 
     if env['TARGET_ARCH'] == "aarch64":
-        AddToCCFLAGSIfSupported(myenv, "-moutline-atomics")
+        myenv.AddToCCFLAGSIfSupported("-moutline-atomics")
 
     conf = Configure(myenv)
     usdt_enabled = get_option('enable-usdt-probes')
@@ -4647,7 +4766,7 @@ if get_option('ninja') != 'disabled':
             env.FatalError("Use of ccache is mandatory with --ninja and icecream older than 1.2. You are running {}.".format(env['ICECREAM_VERSION']))
 
     ninja_builder = Tool("ninja")
-    env["NINJA_BUILDDIR"] = env.Dir("$BUILD_DIR/ninja")
+    env["NINJA_BUILDDIR"] = env.Dir("$NINJA_BUILDDIR")
     ninja_builder.generate(env)
 
     ninjaConf = Configure(env, help=False, custom_tests = {
@@ -4673,6 +4792,54 @@ if get_option('ninja') != 'disabled':
 
     env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
 
+    if env.TargetOSIs('windows'):
+        # The /b option here will make sure that windows updates the mtime
+        # when copying the file. This allows to not need to use restat for windows
+        # copy commands.
+        copy_install_cmd = "cmd.exe /c copy /b $in $out 1>NUL"
+    else:
+        copy_install_cmd = "install $in $out"
+
+    if env.GetOption('install-action') == 'hardlink':
+        if env.TargetOSIs('windows'):
+            install_cmd = f"cmd.exe /c mklink /h $out $in 1>nul || {copy_install_cmd}"
+        else:
+            install_cmd = f"ln $in $out || {copy_install_cmd}"
+
+    elif env.GetOption('install-action') == 'symlink':
+
+        # macOS's ln and Windows mklink command do not support relpaths
+        # out of the box so we will  precompute during generation in a
+        # custom handler.
+        def symlink_install_action_function(_env, node):
+            # should only be one output and input for this case
+            output_file = _env.NinjaGetOutputs(node)[0]
+            input_file = _env.NinjaGetDependencies(node)[0]
+            try:
+                relpath = os.path.relpath(input_file, os.path.dirname(output_file))
+            except ValueError:
+                relpath = os.path.abspath(input_file)
+
+            return {
+                "outputs": [output_file],
+                "rule": "INSTALL",
+                "inputs": [input_file],
+                "implicit": _env.NinjaGetDependencies(node),
+                "variables": {"precious": node.precious, "relpath": relpath},
+            }
+
+        env.NinjaRegisterFunctionHandler("installFunc", symlink_install_action_function)
+
+        if env.TargetOSIs('windows'):
+            install_cmd = "cmd.exe /c mklink $out $relpath 1>nul"
+        else:
+            install_cmd = "ln -s $relpath $out"
+
+    else:
+        install_cmd = copy_install_cmd
+
+    env.NinjaRule("INSTALL", install_cmd, description="Installed $out", pool="install_pool")
+
     if env.TargetOSIs("windows"):
         # This is a workaround on windows for SERVER-48691 where the line length
         # in response files is too long:
@@ -4951,15 +5118,15 @@ env.AddPackageNameAlias(
 )
 
 env.AddPackageNameAlias(
-    component="shell",
+    component="jstestshell",
     role="runtime",
-    name="mongodb-shell",
+    name="mongodb-jstestshell",
 )
 
 env.AddPackageNameAlias(
-    component="shell",
+    component="jstestshell",
     role="debug",
-    name="mongodb-shell-debugsymbols",
+    name="mongodb-jstestshell-debugsymbols",
 )
 
 env.AddPackageNameAlias(
@@ -4992,7 +5159,10 @@ env.AddPackageNameAlias(
     name="mh-debugsymbols",
 )
 
-def rpath_generator(env, source, target, for_signature):
+env['RPATH_ESCAPED_DOLLAR_ORIGIN'] = '\\$$$$ORIGIN'
+
+
+def prefix_libdir_rpath_generator(env, source, target, for_signature):
     # If the PREFIX_LIBDIR has an absolute path, we will use that directly as
     # RPATH because that indicates the final install destination of the libraries.
     prefix_libdir = env.subst('$PREFIX_LIBDIR')
@@ -5004,16 +5174,18 @@ def rpath_generator(env, source, target, for_signature):
     lib_rel = os.path.relpath(prefix_libdir, env.subst('$PREFIX_BINDIR'))
 
     if env['PLATFORM'] == 'posix':\
-        return [env.Literal(f"\\$$ORIGIN/{lib_rel}")]
+        return f"$RPATH_ESCAPED_DOLLAR_ORIGIN/{lib_rel}"
 
     if env['PLATFORM'] == 'darwin':
-        return [f"@loader_path/{lib_rel}",]
+        return f"@loader_path/{lib_rel}"
 
-env['RPATH_GENERATOR'] = rpath_generator
+
+if get_option('link-model').startswith('dynamic'):
+    env['PREFIX_LIBDIR_RPATH_GENERATOR'] = prefix_libdir_rpath_generator
 
 if env['PLATFORM'] == 'posix':
     env.AppendUnique(
-        RPATH='$RPATH_GENERATOR',
+        RPATH=['$PREFIX_LIBDIR_RPATH_GENERATOR'],
         LINKFLAGS=[
             # Most systems *require* -z,origin to make origin work, but android
             # blows up at runtime if it finds DF_ORIGIN_1 in DT_FLAGS_1.
@@ -5034,12 +5206,12 @@ elif env['PLATFORM'] == 'darwin':
     # so we setup RPATH and LINKFLAGS ourselves.
     env['RPATHPREFIX'] = '-Wl,-rpath,'
     env['RPATHSUFFIX'] = ''
-    env['RPATH'] = '$RPATH_GENERATOR'
     env.AppendUnique(
         LINKFLAGS="${_concat(RPATHPREFIX, RPATH, RPATHSUFFIX, __env__)}",
         SHLINKFLAGS=[
             "-Wl,-install_name,@rpath/${TARGET.file}",
         ],
+        RPATH=['$PREFIX_LIBDIR_RPATH_GENERATOR'],
     )
 
 env.Default(env.Alias("install-default"))

buildscripts/blackduck_hub.py

@@ -70,7 +70,7 @@ BUILD_LOGGER_APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s"
 BUILD_LOGGER_CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test"
 BUILD_LOGGER_APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s"
 
-BUILD_LOGGER_DEFAULT_URL = "https://logkeeper.mongodb.org"
+BUILD_LOGGER_DEFAULT_URL = "https://logkeeper2.build.10gen.cc"
 BUILD_LOGGER_TIMEOUT_SECS = 65
 
 LOCAL_REPORTS_DIR = "bd_reports"
@@ -86,7 +86,7 @@ THIRD_PARTY_COMPONENTS_FILE = "etc/third_party_components.yml"
 
 ############################################################################
 
-RE_LETTERS = re.compile("[A-Za-z]{2,}")
+RE_LETTER = re.compile("[A-Za-z]")
 
 
 def default_if_none(value, default):
@@ -287,7 +287,9 @@ class VersionInfo:
             self.production_version = True
 
             # Abseil has an empty string for one version
-            if self.ver_str == "":
+            # Abseil thinks "~" is character to use in versions
+            # Geary thinks "+" is character to use in versions
+            if self.ver_str == "" or "~" in self.ver_str or "+" in self.ver_str:
                 self.production_version = False
                 return
 
@@ -329,8 +331,9 @@ class VersionInfo:
             if self.ver_str.endswith('-'):
                 self.ver_str = self.ver_str[0:-1]
 
-            # Boost keeps varying the version strings so filter for anything with 2 or more ascii charaters
-            if RE_LETTERS.search(self.ver_str):
+            # Boost keeps varying the version strings so filter for anything with a letter
+            # Safeint has "3.0.26c" as a version number
+            if RE_LETTER.search(self.ver_str):
                 self.production_version = False
                 return
 
@@ -570,7 +573,7 @@ curl --retry 5 -s -L https://detect.synopsys.com/detect.sh  | bash -s -- --black
 """.encode())
         fp.flush()
 
-        subprocess.call(["/bin/sh", fp.name])
+        subprocess.check_call(["/bin/sh", fp.name])
 
 
 def _scan_cmd_args(args):

buildscripts/build_system_options.py

@@ -9,7 +9,7 @@ class PathOptions:
     shared_library_folder_name = "lib"
 
     # extend the below list if there are new types of shared libraries
-    _shared_library_file_patterns = [r".+\.so(\.\d{1,})?$", r".+\.dylib$"]
+    _shared_library_file_patterns = [r".+\.so(\.(\d{1,}|debug))?$", r".+\.dylib$"]
     _compiled_shared_library_file_patterns = None
 
     @property

buildscripts/burn_in_tags.py

@@ -160,7 +160,7 @@ def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject
                                          repeat_tests_max=config_options.repeat_tests_max,
                                          repeat_tests_secs=config_options.repeat_tests_secs)
 
-            burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config, evergreen_api)
+            burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config)
             burn_in_generator.generate_tasks_for_variant(tests_by_task, shrub_build_variant)
             shrub_project.add_build_variant(shrub_build_variant)
 

buildscripts/burn_in_tests.py

@@ -13,6 +13,7 @@ from typing import Optional, Set, Tuple, List, Dict, NamedTuple
 import click
 import yaml
 from git import Repo
+from pydantic import BaseModel
 import structlog
 from structlog.stdlib import LoggerFactory
 
@@ -352,7 +353,8 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str,
 
 
 def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig,
-                         changed_tests: Set[str], install_dir: str) -> Dict[str, TaskInfo]:
+                         changed_tests: Set[str],
+                         install_dir: Optional[str]) -> Dict[str, TaskInfo]:
     """
     Create a list of tests by task.
 
@@ -367,7 +369,11 @@ def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig,
         exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*")
     changed_tests = filter_tests(changed_tests, exclude_tests)
 
-    buildscripts.resmokelib.parser.set_run_options(f"--installDir={shlex.quote(install_dir)}")
+    run_options = ""
+    if install_dir is not None:
+        run_options = f"--installDir={shlex.quote(install_dir)}"
+    buildscripts.resmokelib.parser.set_run_options(run_options)
+
     if changed_tests:
         return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites,
                                           exclude_tasks)
@@ -408,7 +414,7 @@ def _configure_logging(verbose: bool):
     logging.basicConfig(
         format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s",
         level=level,
-        stream=sys.stdout,
+        stream=sys.stderr,
     )
     for log_name in EXTERNAL_LOGGERS:
         logging.getLogger(log_name).setLevel(logging.WARNING)
@@ -537,11 +543,45 @@ class LocalBurnInExecutor(BurnInExecutor):
         run_tests(tests_by_task, resmoke_cmd)
 
 
+class DiscoveredTask(BaseModel):
+    """
+    Model for a discovered task to run.
+
+    * task_name: Name of discovered task.
+    * test_list: List of tests to run under discovered task.
+    """
+
+    task_name: str
+    test_list: List[str]
+
+
+class DiscoveredTaskList(BaseModel):
+    """Model for a list of discovered tasks."""
+
+    discovered_tasks: List[DiscoveredTask]
+
+
+class YamlBurnInExecutor(BurnInExecutor):
+    """A burn-in executor that outputs discovered tasks as YAML."""
+
+    def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None:
+        """
+        Report the given tasks and their tests to stdout.
+
+        :param tests_by_task: Dictionary of tasks to run with tests to run in each.
+        """
+        discovered_tasks = DiscoveredTaskList(discovered_tasks=[
+            DiscoveredTask(task_name=task_name, test_list=task_info.tests)
+            for task_name, task_info in tests_by_task.items()
+        ])
+        print(yaml.safe_dump(discovered_tasks.dict()))
+
+
 class BurnInOrchestrator:
     """Orchestrate the execution of burn_in_tests."""
 
     def __init__(self, change_detector: FileChangeDetector, burn_in_executor: BurnInExecutor,
-                 evg_conf: EvergreenProjectConfig) -> None:
+                 evg_conf: EvergreenProjectConfig, install_dir: Optional[str]) -> None:
         """
         Create a new orchestrator.
 
@@ -552,8 +592,9 @@ class BurnInOrchestrator:
         self.change_detector = change_detector
         self.burn_in_executor = burn_in_executor
         self.evg_conf = evg_conf
+        self.install_dir = install_dir
 
-    def burn_in(self, repos: List[Repo], build_variant: str, install_dir: str) -> None:
+    def burn_in(self, repos: List[Repo], build_variant: str) -> None:
         """
         Execute burn in tests for the given git repositories.
 
@@ -564,7 +605,7 @@ class BurnInOrchestrator:
         LOGGER.info("Found changed tests", files=changed_tests)
 
         tests_by_task = create_tests_by_task(build_variant, self.evg_conf, changed_tests,
-                                             install_dir)
+                                             self.install_dir)
         LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
 
         self.burn_in_executor.execute(tests_by_task)
@@ -584,18 +625,20 @@ class BurnInOrchestrator:
               help="The maximum number of times to repeat tests if time option is specified.")
 @click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS",
               help="Repeat tests for the given time (in secs).")
+@click.option("--yaml", "use_yaml", is_flag=True, default=False,
+              help="Output discovered tasks in YAML. Tests will not be run.")
 @click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.")
 @click.option(
     "--origin-rev", "origin_rev", default=None,
     help="The revision in the mongo repo that changes will be compared against if specified.")
-@click.option("--install-dir", "install_dir", required=True, type=str,
+@click.option("--install-dir", "install_dir", type=str,
               help="Path to bin directory of a testable installation")
 @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED)
 # pylint: disable=too-many-arguments,too-many-locals
 def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
          repeat_tests_min: Optional[int], repeat_tests_max: Optional[int],
          repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool,
-         origin_rev: Optional[str], install_dir: str) -> None:
+         origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool) -> None:
     """
     Run new or changed tests in repeated mode to validate their stability.
 
@@ -639,11 +682,13 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
 
     change_detector = LocalFileChangeDetector(origin_rev)
     executor = LocalBurnInExecutor(resmoke_args, repeat_config)
-    if no_exec:
+    if use_yaml:
+        executor = YamlBurnInExecutor()
+    elif no_exec:
         executor = NopBurnInExecutor()
 
-    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf)
-    burn_in_orchestrator.burn_in(repos, build_variant, install_dir)
+    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir)
+    burn_in_orchestrator.burn_in(repos, build_variant)
 
 
 if __name__ == "__main__":

buildscripts/collect_resource_info.py

@@ -18,7 +18,7 @@ from buildscripts.resmokelib import utils  # pylint: disable=wrong-import-positi
 
 
 def main():
-    """Main."""
+    """Collect system resources."""
     usage = "usage: %prog [options]"
     parser = optparse.OptionParser(description=__doc__, usage=usage)
     parser.add_option(

buildscripts/debugsymb_mapper.py

@@ -4,11 +4,13 @@ import json
 import logging
 import os
 import pathlib
+import re
 import shutil
 import subprocess
 import sys
 import time
-import typing
+from json import JSONDecoder
+from typing import Optional, Tuple, Generator, Dict, List, NamedTuple
 
 import requests
 
@@ -20,30 +22,95 @@ from buildscripts.util.oauth import get_client_cred_oauth_credentials, Configs
 from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, download
 from buildscripts.build_system_options import PathOptions
 
+BUILD_INFO_RE = re.compile(r"Build Info: ({(\n.*)*})")
+MONGOD = "mongod"
 
-class LinuxBuildIDExtractor:
-    """Parse readlef command output & extract Build ID."""
 
-    default_executable_path = "readelf"
-
-    def __init__(self, executable_path: str = None):
-        """Initialize instance."""
-
-        self.executable_path = executable_path or self.default_executable_path
-
-    def callreadelf(self, binary_path: str) -> str:
-        """Call readelf command for given binary & return string output."""
-
-        args = [self.executable_path, "-n", binary_path]
-        process = subprocess.Popen(args=args, close_fds=True, stdin=subprocess.PIPE,
-                                   stdout=subprocess.PIPE)
-        process.wait()
-        return process.stdout.read().decode()
+class CmdClient:
+    """Client to run commands."""
 
     @staticmethod
-    def extractbuildid(out: str) -> typing.Optional[str]:
-        """Parse readelf output and extract Build ID from it."""
+    def run(args: List[str]) -> str:
+        """
+        Run command with args.
 
+        :param args: Argument list.
+        :return: Command output.
+        """
+
+        out = subprocess.run(args, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                             check=False)
+        return out.stdout.strip().decode()
+
+
+class BuildIdOutput(NamedTuple):
+    """
+    Build ID and command output.
+
+    * build_id: Build ID or None.
+    * cmd_output: Command output.
+    """
+
+    build_id: Optional[str]
+    cmd_output: str
+
+
+class BinVersionOutput(NamedTuple):
+    """
+    Mongodb bin version and command output.
+
+    * mongodb_version: Bin version.
+    * cmd_output: Command output.
+    """
+
+    mongodb_version: Optional[str]
+    cmd_output: str
+
+
+class CmdOutputExtractor:
+    """Data extractor from command output."""
+
+    def __init__(self, cmd_client: Optional[CmdClient] = None,
+                 json_decoder: Optional[JSONDecoder] = None) -> None:
+        """
+        Initialize.
+
+        :param cmd_client: Client to run commands.
+        :param json_decoder: JSONDecoder object.
+        """
+        self.cmd_client = cmd_client if cmd_client is not None else CmdClient()
+        self.json_decoder = json_decoder if json_decoder is not None else JSONDecoder()
+
+    def get_build_id(self, bin_path: str) -> BuildIdOutput:
+        """
+        Get build ID from readelf command.
+
+        :param bin_path: Path to binary of the build.
+        :return: Build ID or None and command output.
+        """
+        out = self.cmd_client.run(["readelf", "-n", bin_path])
+        build_id = self._extract_build_id(out)
+        return BuildIdOutput(build_id, out)
+
+    def get_bin_version(self, bin_path: str) -> BinVersionOutput:
+        """
+        Get mongodb bin version from `{bin} --version` command.
+
+        :param bin_path: Path to mongodb binary.
+        :return: Bin version or None and command output.
+        """
+        out = self.cmd_client.run([os.path.abspath(bin_path), "--version"])
+        mongodb_version = self._get_mongodb_version(out)
+        return BinVersionOutput(mongodb_version, out)
+
+    @staticmethod
+    def _extract_build_id(out: str) -> Optional[str]:
+        """
+        Parse readelf output and extract Build ID from it.
+
+        :param out: readelf command output.
+        :return: Build ID on None.
+        """
         build_id = None
         for line in out.splitlines():
             line = line.strip()
@@ -53,13 +120,21 @@ class LinuxBuildIDExtractor:
                 build_id = line.split(': ')[1]
         return build_id
 
-    def run(self, binary_path: str) -> typing.Tuple[str, str]:
-        """Perform all necessary actions to get Build ID."""
+    def _get_mongodb_version(self, out: str) -> Optional[str]:
+        """
+        Parse version command output and extract mongodb version.
 
-        readelfout = self.callreadelf(binary_path)
-        buildid = self.extractbuildid(readelfout)
+        :param out: Version command output.
+        :return: Version or None.
+        """
+        mongodb_version = None
 
-        return buildid, readelfout
+        search = BUILD_INFO_RE.search(out)
+        if search:
+            build_info = self.json_decoder.decode(search.group(1))
+            mongodb_version = build_info.get("version")
+
+        return mongodb_version
 
 
 class DownloadOptions(object):
@@ -88,19 +163,22 @@ class Mapper:
     default_client_credentials_user_name = "client-user"
     default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json')
 
-    def __init__(self, version: str, client_id: str, client_secret: str, variant: str,
+    def __init__(self, evg_version: str, evg_variant: str, client_id: str, client_secret: str,
                  cache_dir: str = None, web_service_base_url: str = None,
                  logger: logging.Logger = None):
         """
         Initialize instance.
 
-        :param version: version string
-        :param variant: build variant string
-        :param cache_dir: full path to cache directory as a string
-        :param web_service_base_url: URL of symbolizer web service
+        :param evg_version: Evergreen version ID.
+        :param evg_variant: Evergreen build variant name.
+        :param client_id: Client id for Okta Oauth.
+        :param client_secret: Secret key for Okta Oauth.
+        :param cache_dir: Full path to cache directory as a string.
+        :param web_service_base_url: URL of symbolizer web service.
+        :param logger: Debug symbols mapper logger.
         """
-        self.version = version
-        self.variant = variant
+        self.evg_version = evg_version
+        self.evg_variant = evg_variant
         self.cache_dir = cache_dir or self.default_cache_dir
         self.web_service_base_url = web_service_base_url or self.default_web_service_base_url
 
@@ -113,8 +191,8 @@ class Mapper:
         self.http_client = requests.Session()
 
         self.multiversion_setup = SetupMultiversion(
-            DownloadOptions(download_symbols=True, download_binaries=True), variant=self.variant,
-            ignore_failed_push=True)
+            DownloadOptions(download_symbols=True, download_binaries=True),
+            variant=self.evg_variant, ignore_failed_push=True)
         self.debug_symbols_url = None
         self.url = None
         self.configs = Configs(
@@ -139,7 +217,7 @@ class Mapper:
                 data = json.loads(cfile.read())
                 access_token, expire_time = data.get("access_token"), data.get("expire_time")
                 if time.time() < expire_time:
-                    # credentials hasn't expired yet
+                    # credentials haven't expired yet
                     self.http_client.headers.update({"Authorization": f"Bearer {access_token}"})
                     return
 
@@ -147,7 +225,7 @@ class Mapper:
                                                         configs=self.configs)
         self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"})
 
-        # write credentials to local file for further useage
+        # write credentials to local file for further usage
         with open(self.default_creds_file_path, "w") as cfile:
             cfile.write(
                 json.dumps({
@@ -184,7 +262,7 @@ class Mapper:
     def setup_urls(self):
         """Set up URLs using multiversion."""
 
-        urlinfo = self.multiversion_setup.get_urls(self.version, self.variant)
+        urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant)
 
         download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None)
         binaries_url = urlinfo.urls.get("Binaries", "")
@@ -194,7 +272,7 @@ class Mapper:
 
         if not download_symbols_url:
             self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s",
-                              self.version, urlinfo.urls)
+                              self.evg_version, urlinfo.urls)
             raise ValueError(f"Debug symbols URL not found. URLs dict: {urlinfo.urls}")
 
         self.debug_symbols_url = download_symbols_url
@@ -233,14 +311,14 @@ class Mapper:
         tarball_full_path = download.download_from_s3(url)
         return tarball_full_path
 
-    def generate_build_id_mapping(self) -> typing.Generator[typing.Dict[str, str], None, None]:
+    def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]:
         """
         Extract build id from binaries and creates new dict using them.
 
         :return: mapped data as dict
         """
 
-        readelf_extractor = LinuxBuildIDExtractor()
+        extractor = CmdOutputExtractor()
 
         debug_symbols_path = self.download(self.debug_symbols_url)
         debug_symbols_unpacked_path = self.unpack(debug_symbols_path)
@@ -262,6 +340,17 @@ class Mapper:
         self.logger.info("INSIDE unpacked binaries/dist-test: %s",
                          os.listdir(binaries_unpacked_path))
 
+        mongod_bin = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name,
+                                  MONGOD)
+        bin_version_output = extractor.get_bin_version(mongod_bin)
+
+        if bin_version_output.mongodb_version is None:
+            self.logger.error("mongodb version could not be extracted. \n`%s --version` output: %s",
+                              mongod_bin, bin_version_output.cmd_output)
+            return
+        else:
+            self.logger.info("Extracted mongodb version: %s", bin_version_output.mongodb_version)
+
         # start with main binary folder
         for binary in self.selected_binaries:
             full_bin_path = os.path.join(debug_symbols_unpacked_path,
@@ -271,16 +360,21 @@ class Mapper:
                 self.logger.error("Could not find binary at %s", full_bin_path)
                 return
 
-            build_id, readelf_out = readelf_extractor.run(full_bin_path)
+            build_id_output = extractor.get_build_id(full_bin_path)
 
-            if not build_id:
+            if not build_id_output.build_id:
                 self.logger.error("Build ID couldn't be extracted. \nReadELF output %s",
-                                  readelf_out)
+                                  build_id_output.cmd_output)
                 return
+            else:
+                self.logger.info("Extracted build ID: %s", build_id_output.build_id)
 
             yield {
-                'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id,
-                'file_name': binary, 'version': self.version
+                'url': self.url,
+                'debug_symbols_url': self.debug_symbols_url,
+                'build_id': build_id_output.build_id,
+                'file_name': binary,
+                'version': bin_version_output.mongodb_version,
             }
 
         # move to shared libraries folder.
@@ -306,18 +400,21 @@ class Mapper:
                 self.logger.error("Could not find binary at %s", sofile_path)
                 return
 
-            build_id, readelf_out = readelf_extractor.run(sofile_path)
+            build_id_output = extractor.get_build_id(sofile_path)
 
-            if not build_id:
-                self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", readelf_out)
+            if not build_id_output.build_id:
+                self.logger.error("Build ID couldn't be extracted. \nReadELF out %s",
+                                  build_id_output.cmd_output)
                 return
+            else:
+                self.logger.info("Extracted build ID: %s", build_id_output.build_id)
 
             yield {
                 'url': self.url,
                 'debug_symbols_url': self.debug_symbols_url,
-                'build_id': build_id,
+                'build_id': build_id_output.build_id,
                 'file_name': sofile,
-                'version': self.version,
+                'version': bin_version_output.mongodb_version,
             }
 
     def run(self):
@@ -355,8 +452,8 @@ def make_argument_parser(parser=None, **kwargs):
 def main(options):
     """Execute mapper here. Main entry point."""
 
-    mapper = Mapper(version=options.version, variant=options.variant, client_id=options.client_id,
-                    client_secret=options.client_secret,
+    mapper = Mapper(evg_version=options.version, evg_variant=options.variant,
+                    client_id=options.client_id, client_secret=options.client_secret,
                     web_service_base_url=options.web_service_base_url)
 
     # when used as a context manager, mapper instance automatically cleans files/folders after finishing its job.

buildscripts/errorcodes.py

@@ -46,6 +46,7 @@ _CODE_PATTERNS = [
         r"(?:StatusOK)?"
         r"(?:WithContext)?"
         r"\s*\(",
+        r"MONGO_UNREACHABLE_TASSERT\(",
         # DBException and AssertionException constructors
         r"(?:DB|Assertion)Exception\s*[({]",
         # Calls to all LOGV2* variants
@@ -239,14 +240,15 @@ def read_error_codes(src_root='src/mongo'):
 
 
 def replace_bad_codes(errors, next_code_generator):  # pylint: disable=too-many-locals
-    """Modify C++ source files to replace invalid assertion codes.
+    """
+    Modify C++ source files to replace invalid assertion codes.
 
     For now, we only modify zero codes.
 
-    Args:
-        errors: list of AssertLocation
-        next_code_generator: generator -> int, next non-conflicting assertion code
+    :param errors: list of AssertLocation
+    :param next_code_generator: generator -> int, next non-conflicting assertion code
     """
+
     zero_errors = [e for e in errors if int(e.code) == 0]
     skip_errors = [e for e in errors if int(e.code) != 0]
 
@@ -296,7 +298,7 @@ def coerce_to_number(ticket_value):
 
 
 def main():
-    """Main."""
+    """Validate error codes."""
     parser = OptionParser(description=__doc__.strip())
     parser.add_option("--fix", dest="replace", action="store_true", default=False,
                       help="Fix zero codes in source files [default: %default]")

buildscripts/evergreen_burn_in_tests.py

@@ -34,7 +34,6 @@ DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required"
 BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen"
 BURN_IN_TESTS_TASK = "burn_in_tests"
 BURN_IN_ENV_VAR = "BURN_IN_TESTS"
-AVG_TEST_RUNTIME_ANALYSIS_DAYS = 14
 AVG_TEST_SETUP_SEC = 4 * 60
 AVG_TEST_TIME_MULTIPLIER = 3
 MIN_AVG_TEST_OVERFLOW_SEC = float(60)
@@ -328,23 +327,17 @@ class GenerateBurnInExecutor(BurnInExecutor):
 
     # pylint: disable=too-many-arguments
     def __init__(self, generate_config: GenerateConfig, repeat_config: RepeatConfig,
-                 evg_api: EvergreenApi, generate_tasks_file: Optional[str] = None,
-                 history_end_date: Optional[datetime] = None) -> None:
+                 generate_tasks_file: Optional[str] = None) -> None:
         """
         Create a new generate burn-in executor.
 
         :param generate_config: Configuration for how to generate tasks.
         :param repeat_config: Configuration for how tests should be repeated.
-        :param evg_api: Evergreen API client.
         :param generate_tasks_file: File to write generated task configuration to.
-        :param history_end_date: End date of range to query for historic test data.
         """
         self.generate_config = generate_config
         self.repeat_config = repeat_config
-        self.evg_api = evg_api
         self.generate_tasks_file = generate_tasks_file
-        self.history_end_date = history_end_date if history_end_date else datetime.utcnow()\
-            .replace(microsecond=0)
 
     def get_task_runtime_history(self, task: str) -> List[TestRuntime]:
         """
@@ -353,21 +346,10 @@ class GenerateBurnInExecutor(BurnInExecutor):
         :param task: Task to query.
         :return: List of runtime histories for all tests in specified task.
         """
-        try:
-            project = self.generate_config.project
-            variant = self.generate_config.build_variant
-            end_date = self.history_end_date
-            start_date = end_date - timedelta(days=AVG_TEST_RUNTIME_ANALYSIS_DAYS)
-            test_stats = HistoricTaskData.from_evg(self.evg_api, project, start_date=start_date,
-                                                   end_date=end_date, task=task, variant=variant)
-            return test_stats.get_tests_runtimes()
-        except requests.HTTPError as err:
-            if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE:
-                # Evergreen may return a 503 when the service is degraded.
-                # We fall back to returning no test history
-                return []
-            else:
-                raise
+        project = self.generate_config.project
+        variant = self.generate_config.build_variant
+        test_stats = HistoricTaskData.from_s3(project, task, variant)
+        return test_stats.get_tests_runtimes()
 
     def _get_existing_tasks(self) -> Optional[Set[ExistingTask]]:
         """Get any existing tasks that should be included in the generated display task."""
@@ -434,10 +416,10 @@ def burn_in(task_id: str, build_variant: str, generate_config: GenerateConfig,
     :param install_dir: Path to bin directory of a testable installation
     """
     change_detector = EvergreenFileChangeDetector(task_id, evg_api, os.environ)
-    executor = GenerateBurnInExecutor(generate_config, repeat_config, evg_api, generate_tasks_file)
+    executor = GenerateBurnInExecutor(generate_config, repeat_config, generate_tasks_file)
 
-    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf)
-    burn_in_orchestrator.burn_in(repos, build_variant, install_dir)
+    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir)
+    burn_in_orchestrator.burn_in(repos, build_variant)
 
 
 @click.command()

buildscripts/evergreen_task_timeout.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 """Determine the timeout value a task should use in evergreen."""
+from __future__ import annotations
 
 import argparse
 import math
 import os
+import shlex
 import sys
-from datetime import datetime, timedelta
+from datetime import timedelta
 from pathlib import Path
 from typing import Dict, List, Optional
-import shlex
 
 import inject
 import structlog
@@ -16,9 +17,9 @@ import yaml
 from pydantic import BaseModel
 from evergreen import EvergreenApi, RetryingEvergreenApi
 
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, parse_evergreen_file)
-from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService, TimeoutSettings)
+from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
+from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService)
 from buildscripts.util.cmdutils import enable_logging
 from buildscripts.util.taskname import determine_task_base_name
 
@@ -56,6 +57,18 @@ class TimeoutOverride(BaseModel):
     exec_timeout: Optional[int] = None
     idle_timeout: Optional[int] = None
 
+    @classmethod
+    def from_seconds(cls, task: str, exec_timeout_secs: Optional[float],
+                     idle_timeout_secs: Optional[float]) -> TimeoutOverride:
+        """Create an instance of an override from seconds."""
+        exec_timeout = exec_timeout_secs / 60 if exec_timeout_secs else None
+        idle_timeout = idle_timeout_secs / 60 if idle_timeout_secs else None
+        return cls(
+            task=task,
+            exec_timeout=exec_timeout,
+            idle_timeout=idle_timeout,
+        )
+
     def get_exec_timeout(self) -> Optional[timedelta]:
         """Get a timedelta of the exec timeout to use."""
         if self.exec_timeout is not None:
@@ -173,14 +186,16 @@ class TaskTimeoutOrchestrator:
 
         :param timeout_service: Service for calculating historic timeouts.
         :param timeout_overrides: Timeout overrides for specific tasks.
+        :param evg_project_config: Evergreen project configuration.
         """
         self.timeout_service = timeout_service
         self.timeout_overrides = timeout_overrides
         self.evg_project_config = evg_project_config
 
-    def determine_exec_timeout(
-            self, task_name: str, variant: str, idle_timeout: Optional[timedelta] = None,
-            exec_timeout: Optional[timedelta] = None, evg_alias: str = "") -> timedelta:
+    def determine_exec_timeout(self, task_name: str, variant: str,
+                               idle_timeout: Optional[timedelta] = None,
+                               exec_timeout: Optional[timedelta] = None, evg_alias: str = "",
+                               historic_timeout: Optional[timedelta] = None) -> timedelta:
         """
         Determine what exec timeout should be used.
 
@@ -189,9 +204,12 @@ class TaskTimeoutOrchestrator:
         :param idle_timeout: Idle timeout if specified.
         :param exec_timeout: Override to use for exec_timeout or 0 if no override.
         :param evg_alias: Evergreen alias running the task.
+        :param historic_timeout: Timeout determined by looking at previous task executions.
         :return: Exec timeout to use for running task.
         """
         determined_timeout = DEFAULT_NON_REQUIRED_BUILD_TIMEOUT
+        if historic_timeout is not None:
+            determined_timeout = historic_timeout
 
         override = self.timeout_overrides.lookup_exec_override(variant, task_name)
 
@@ -200,25 +218,26 @@ class TaskTimeoutOrchestrator:
                         exec_timeout_secs=exec_timeout.total_seconds())
             determined_timeout = exec_timeout
 
+        elif override is not None:
+            LOGGER.info("Overriding configured timeout", exec_timeout_secs=override.total_seconds())
+            determined_timeout = override
+
         elif task_name == UNITTEST_TASK and override is None:
             LOGGER.info("Overriding unittest timeout",
                         exec_timeout_secs=UNITTESTS_TIMEOUT.total_seconds())
             determined_timeout = UNITTESTS_TIMEOUT
 
+        elif _is_required_build_variant(
+                variant) and determined_timeout > DEFAULT_REQUIRED_BUILD_TIMEOUT:
+            LOGGER.info("Overriding required-builder timeout",
+                        exec_timeout_secs=DEFAULT_REQUIRED_BUILD_TIMEOUT.total_seconds())
+            determined_timeout = DEFAULT_REQUIRED_BUILD_TIMEOUT
+
         elif evg_alias == COMMIT_QUEUE_ALIAS:
             LOGGER.info("Overriding commit-queue timeout",
                         exec_timeout_secs=COMMIT_QUEUE_TIMEOUT.total_seconds())
             determined_timeout = COMMIT_QUEUE_TIMEOUT
 
-        elif override is not None:
-            LOGGER.info("Overriding configured timeout", exec_timeout_secs=override.total_seconds())
-            determined_timeout = override
-
-        elif _is_required_build_variant(variant):
-            LOGGER.info("Overriding required-builder timeout",
-                        exec_timeout_secs=DEFAULT_REQUIRED_BUILD_TIMEOUT.total_seconds())
-            determined_timeout = DEFAULT_REQUIRED_BUILD_TIMEOUT
-
         # The timeout needs to be at least as large as the idle timeout.
         if idle_timeout and determined_timeout.total_seconds() < idle_timeout.total_seconds():
             LOGGER.info("Making exec timeout as large as idle timeout",
@@ -228,16 +247,19 @@ class TaskTimeoutOrchestrator:
         return determined_timeout
 
     def determine_idle_timeout(self, task_name: str, variant: str,
-                               idle_timeout: Optional[timedelta] = None) -> Optional[timedelta]:
+                               idle_timeout: Optional[timedelta] = None,
+                               historic_timeout: Optional[timedelta] = None) -> Optional[timedelta]:
         """
         Determine what idle timeout should be used.
 
         :param task_name: Name of task being run.
         :param variant: Name of build variant being run.
         :param idle_timeout: Override to use for idle_timeout.
+        :param historic_timeout: Timeout determined by looking at previous task executions.
         :return: Idle timeout to use for running task.
         """
-        determined_timeout = None
+        determined_timeout = historic_timeout
+
         override = self.timeout_overrides.lookup_idle_override(variant, task_name)
 
         if idle_timeout and idle_timeout.total_seconds() != 0:
@@ -252,7 +274,7 @@ class TaskTimeoutOrchestrator:
         return determined_timeout
 
     def determine_historic_timeout(self, task: str, variant: str, suite_name: str,
-                                   exec_timeout_factor: Optional[float]) -> Optional[timedelta]:
+                                   exec_timeout_factor: Optional[float]) -> TimeoutOverride:
         """
         Calculate the timeout based on historic test results.
 
@@ -262,7 +284,7 @@ class TaskTimeoutOrchestrator:
         :param exec_timeout_factor: Scaling factor to use when determining timeout.
         """
         if suite_name in IGNORED_SUITES:
-            return None
+            return TimeoutOverride(task=task, exec_timeout=None, idle_timeout=None)
 
         timeout_params = TimeoutParams(
             evg_project="mongodb-mongo-master",
@@ -275,10 +297,12 @@ class TaskTimeoutOrchestrator:
         if timeout_estimate and timeout_estimate.is_specified():
             exec_timeout = timeout_estimate.calculate_task_timeout(
                 repeat_factor=1, scaling_factor=exec_timeout_factor)
-            if exec_timeout is not None:
-                LOGGER.info("Using historic based timeout", exec_timeout_secs=exec_timeout)
-                return timedelta(seconds=exec_timeout)
-        return None
+            idle_timeout = timeout_estimate.calculate_test_timeout(repeat_factor=1)
+            if exec_timeout is not None or idle_timeout is not None:
+                LOGGER.info("Getting historic based timeout", exec_timeout_secs=exec_timeout,
+                            idle_timeout_secs=idle_timeout)
+                return TimeoutOverride.from_seconds(task, exec_timeout, idle_timeout)
+        return TimeoutOverride(task=task, exec_timeout=None, idle_timeout=None)
 
     def is_build_variant_asan(self, build_variant: str) -> bool:
         """
@@ -305,14 +329,13 @@ class TaskTimeoutOrchestrator:
         :param suite_name: Name of evergreen suite being run.
         :param exec_timeout_factor: Scaling factor to use when determining timeout.
         """
-        idle_timeout = self.determine_idle_timeout(task, variant, cli_idle_timeout)
-        exec_timeout = self.determine_exec_timeout(task, variant, idle_timeout, cli_exec_timeout,
-                                                   evg_alias)
-
         historic_timeout = self.determine_historic_timeout(task, variant, suite_name,
                                                            exec_timeout_factor)
-        if historic_timeout:
-            exec_timeout = historic_timeout
+
+        idle_timeout = self.determine_idle_timeout(task, variant, cli_idle_timeout,
+                                                   historic_timeout.get_idle_timeout())
+        exec_timeout = self.determine_exec_timeout(task, variant, idle_timeout, cli_exec_timeout,
+                                                   evg_alias, historic_timeout.get_exec_timeout())
 
         output_timeout(exec_timeout, idle_timeout, outfile)
 
@@ -346,9 +369,6 @@ def main():
 
     options = parser.parse_args()
 
-    end_date = datetime.now()
-    start_date = end_date - HISTORY_LOOKBACK
-
     timeout_override = timedelta(seconds=options.timeout) if options.timeout else None
     exec_timeout_override = timedelta(
         seconds=options.exec_timeout) if options.exec_timeout else None
@@ -363,7 +383,6 @@ def main():
         binder.bind(
             EvergreenApi,
             RetryingEvergreenApi.get_api(config_file=os.path.expanduser(options.evg_api_config)))
-        binder.bind(TimeoutSettings, TimeoutSettings(start_date=start_date, end_date=end_date))
         binder.bind(TimeoutOverrides, timeout_overrides)
         binder.bind(EvergreenProjectConfig,
                     parse_evergreen_file(os.path.expanduser(options.evg_project_config)))

buildscripts/gdb/mongo.py

@@ -69,7 +69,7 @@ def get_current_thread_name():
     fallback_name = '"%s"' % (gdb.selected_thread().name or '')
     try:
         # This goes through the pretty printer for StringData which adds "" around the name.
-        name = str(gdb.parse_and_eval("mongo::ThreadName::getStaticString()"))
+        name = str(gdb.parse_and_eval("mongo::getThreadName()"))
         if name == '""':
             return fallback_name
         return name
@@ -318,75 +318,100 @@ class DumpMongoDSessionCatalog(gdb.Command):
         else:
             lsids_to_print = [str(s['first']['_id']) for s in session_kv_pairs]
 
-        for sess_kv in session_kv_pairs:
-            # The Session is stored inside the SessionRuntimeInfo object.
-            session_runtime_info = get_unique_ptr(sess_kv['second']).dereference()  # pylint: disable=undefined-variable
-            session = session_runtime_info['session']
-            # TODO: Add a custom pretty printer for LogicalSessionId.
-            lsid_str = str(session['_sessionId']['_id'])
+        for session_kv in session_kv_pairs:
+            # The Session objects are stored inside the SessionRuntimeInfo object.
+            session_runtime_info = get_unique_ptr(session_kv['second']).dereference()  # pylint: disable=undefined-variable
+            parent_session = session_runtime_info['parentSession']
+            child_sessions = absl_get_nodes(session_runtime_info['childSessions'])  # pylint: disable=undefined-variable
+            lsid = str(parent_session['_sessionId']['_id'])
 
             # If we are only interested in a specific session, then we print out the entire Session
-            # object, to aid more detailed debugging.
-            if lsid_str == lsid_to_find:
-                print("SessionId", "=", lsid_str)
-                print(session)
+            # objects, to aid more detailed debugging.
+            if lsid == lsid_to_find:
+                self.dump_session_runtime_info(session_runtime_info)
+                print(parent_session)
+                for child_session_kv in child_sessions:
+                    child_session = child_session_kv['second']
+                    print(child_session)
                 # Terminate if this is the only session we care about.
                 break
 
             # Only print info for the necessary sessions.
-            if lsid_str not in lsids_to_print:
+            if lsid not in lsids_to_print:
                 continue
 
-            # If we are printing multiple sessions, we only print the most interesting fields from
-            # the Session object for the sake of efficiency. We print the session id string first so
-            # the session is easily identifiable.
-            print("Session (" + str(session.address) + "):")
-            print("SessionId", "=", lsid_str)
-            session_fields_to_print = ['_checkoutOpCtx', '_killsRequested']
-            for field in session_fields_to_print:
+            # If we are printing multiple sessions, we only print the most interesting fields for
+            # each Session object for the sake of efficiency.
+            self.dump_session_runtime_info(session_runtime_info)
+            self.dump_session(parent_session)
+            for child_session_kv in child_sessions:
+                child_session = child_session_kv['second']
+                self.dump_session(child_session)
+
+    @staticmethod
+    def dump_session_runtime_info(session_runtime_info):
+        """Dump the session runtime info."""
+
+        parent_session = session_runtime_info['parentSession']
+        # TODO: Add a custom pretty printer for LogicalSessionId.
+        lsid = str(parent_session['_sessionId']['_id'])[1:-1]
+        print("SessionId =", lsid)
+        fields_to_print = ['checkoutOpCtx', 'killsRequested']
+        for field in fields_to_print:
+            # Skip fields that aren't found on the object.
+            if field in get_field_names(session_runtime_info):
+                print(field, "=", session_runtime_info[field])
+            else:
+                print("Could not find field '%s' on the SessionRuntimeInfo object." % field)
+        print("")
+
+    @staticmethod
+    def dump_session(session):
+        """Dump the session."""
+
+        print("Session (" + str(session.address) + "):")
+        fields_to_print = ['_sessionId', '_numWaitingToCheckOut']
+        for field in fields_to_print:
+            # Skip fields that aren't found on the object.
+            if field in get_field_names(session):
+                print(field, "=", session[field])
+            else:
+                print("Could not find field '%s' on the SessionRuntimeInfo object." % field)
+
+        # Print the information from a TransactionParticipant if a session has one.
+        txn_part_dec = get_decoration(session, "TransactionParticipant")
+        if txn_part_dec:
+            # Only print the most interesting fields for debugging transactions issues. The
+            # TransactionParticipant class encapsulates internal state in two distinct
+            # structures: a 'PrivateState' type (stored in private field '_p') and an
+            # 'ObservableState' type (stored in private field '_o'). The information we care
+            # about here is all contained inside the 'ObservableState', so we extract fields
+            # from that object. If, in the future, we want to print fields from the
+            # 'PrivateState' object, we can inspect the TransactionParticipant's '_p' field.
+            txn_part = txn_part_dec[1]
+            txn_part_observable_state = txn_part['_o']
+            fields_to_print = ['txnState', 'activeTxnNumberAndRetryCounter']
+            print("TransactionParticipant (" + str(txn_part.address) + "):")
+            for field in fields_to_print:
                 # Skip fields that aren't found on the object.
-                if field in get_field_names(session):
-                    print(field, "=", session[field])
+                if field in get_field_names(txn_part_observable_state):
+                    print(field, "=", txn_part_observable_state[field])
                 else:
-                    print("Could not find field '%s' on the Session object." % field)
+                    print("Could not find field '%s' on the TransactionParticipant" % field)
 
-            # Print the information from a TransactionParticipant if a session has one. Otherwise
-            # we just print the session's id and nothing else.
-            txn_part_dec = get_decoration(session, "TransactionParticipant")
-            if txn_part_dec:
-                # Only print the most interesting fields for debugging transactions issues. The
-                # TransactionParticipant class encapsulates internal state in two distinct
-                # structures: a 'PrivateState' type (stored in private field '_p') and an
-                # 'ObservableState' type (stored in private field '_o'). The information we care
-                # about here is all contained inside the 'ObservableState', so we extract fields
-                # from that object. If, in the future, we want to print fields from the
-                # 'PrivateState' object, we can inspect the TransactionParticipant's '_p' field.
-                txn_part = txn_part_dec[1]
-                txn_part_observable_state = txn_part['_o']
-                fields_to_print = ['txnState', 'activeTxnNumber']
-                print("TransactionParticipant (" + str(txn_part.address) + "):")
-                for field in fields_to_print:
-                    # Skip fields that aren't found on the object.
-                    if field in get_field_names(txn_part_observable_state):
-                        print(field, "=", txn_part_observable_state[field])
-                    else:
-                        print("Could not find field '%s' on the TransactionParticipant" % field)
-
-                # The 'txnResourceStash' field is a boost::optional so we unpack it manually if it
-                # is non-empty. We are only interested in its Locker object for now. TODO: Load the
-                # boost pretty printers so the object will be printed clearly by default, without
-                # the need for special unpacking.
-                val = get_boost_optional(txn_part_observable_state['txnResourceStash'])
-                if val:
-                    locker_addr = get_unique_ptr(val["_locker"])  # pylint: disable=undefined-variable
-                    locker_obj = locker_addr.dereference().cast(
-                        gdb.lookup_type("mongo::LockerImpl"))
-                    print('txnResourceStash._locker', "@", locker_addr)
-                    print("txnResourceStash._locker._id", "=", locker_obj["_id"])
-                else:
-                    print('txnResourceStash', "=", None)
-            # Separate sessions by a newline.
-            print("")
+            # The 'txnResourceStash' field is a boost::optional so we unpack it manually if it
+            # is non-empty. We are only interested in its Locker object for now. TODO: Load the
+            # boost pretty printers so the object will be printed clearly by default, without
+            # the need for special unpacking.
+            val = get_boost_optional(txn_part_observable_state['txnResourceStash'])
+            if val:
+                locker_addr = get_unique_ptr(val["_locker"])  # pylint: disable=undefined-variable
+                locker_obj = locker_addr.dereference().cast(gdb.lookup_type("mongo::LockerImpl"))
+                print('txnResourceStash._locker', "@", locker_addr)
+                print("txnResourceStash._locker._id", "=", locker_obj["_id"])
+            else:
+                print('txnResourceStash', "=", None)
+        print("")
 
 
 # Register command
@@ -536,39 +561,51 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
 
         # Dump stashed recovery unit info for each session in a mongod process
         for session_kv in get_session_kv_pairs():
-            session_runtime_info = get_unique_ptr(session_kv["second"]).dereference()  # pylint: disable=undefined-variable
-            session = session_runtime_info["session"]
+            # The Session objects are stored inside the SessionRuntimeInfo object.
+            session_runtime_info = get_unique_ptr(session_kv['second']).dereference()  # pylint: disable=undefined-variable
+            parent_session = session_runtime_info['parentSession']
+            child_sessions = absl_get_nodes(session_runtime_info['childSessions'])  # pylint: disable=undefined-variable
 
-            # Prepare structured output doc
-            session_lsid = str(session["_sessionId"]["_id"])[1:-1]
-            txn_participant_dec = get_decoration(session, "TransactionParticipant")
-            output_doc = {"session": session_lsid, "txnResourceStash": "0x0"}
-
-            recovery_unit_handle = None
-            recovery_unit = None
-            if txn_participant_dec:
-                txn_participant_observable_state = txn_participant_dec[1]["_o"]
-                txn_resource_stash = get_boost_optional(
-                    txn_participant_observable_state["txnResourceStash"])
-
-                if txn_resource_stash:
-                    output_doc["txnResourceStash"] = str(txn_resource_stash.address)
-                    recovery_unit_handle = get_unique_ptr(txn_resource_stash["_recoveryUnit"])  # pylint: disable=undefined-variable
-                    # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit"
-                    recovery_unit = recovery_unit_handle.dereference().cast(
-                        gdb.lookup_type(recovery_unit_impl_type))
-
-            output_doc["recoveryUnit"] = hex(recovery_unit_handle) if recovery_unit else "0x0"
-            wt_session = get_wt_session(recovery_unit, recovery_unit_impl_type)
-            if wt_session:
-                output_doc["WT_SESSION"] = hex(wt_session)
-            print(json.dumps(output_doc))
-            if recovery_unit:
-                print(recovery_unit)
+            MongoDBDumpRecoveryUnits.dump_session(parent_session, recovery_unit_impl_type)
+            for child_session_kv in child_sessions:
+                child_session = child_session_kv['second']
+                MongoDBDumpRecoveryUnits.dump_session(child_session, recovery_unit_impl_type)
 
         if enabled_at_start:
             gdb.execute("set print static-members on")
 
+    @staticmethod
+    def dump_session(session, recovery_unit_impl_type):
+        """Dump the session."""
+
+        # Prepare structured output doc
+        lsid = session["_sessionId"]
+        output_doc = {"session": str(lsid["_id"])[1:-1], "txnResourceStash": "0x0"}
+        txn_participant_dec = get_decoration(session, "TransactionParticipant")
+        recovery_unit_handle = None
+        recovery_unit = None
+
+        if txn_participant_dec:
+            txn_participant_observable_state = txn_participant_dec[1]["_o"]
+            txn_resource_stash = get_boost_optional(
+                txn_participant_observable_state["txnResourceStash"])
+            if txn_resource_stash:
+                output_doc["txnResourceStash"] = str(txn_resource_stash.address)
+                recovery_unit_handle = get_unique_ptr(txn_resource_stash["_recoveryUnit"])  # pylint: disable=undefined-variable
+                # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit"
+                recovery_unit = recovery_unit_handle.dereference().cast(
+                    gdb.lookup_type(recovery_unit_impl_type))
+
+        output_doc["recoveryUnit"] = hex(recovery_unit_handle) if recovery_unit else "0x0"
+        wt_session = get_wt_session(recovery_unit, recovery_unit_impl_type)
+        if wt_session:
+            output_doc["WT_SESSION"] = hex(wt_session)
+
+        print(json.dumps(output_doc))
+        print(lsid)
+        if recovery_unit:
+            print(recovery_unit)
+
 
 # Register command
 MongoDBDumpRecoveryUnits()
@@ -726,7 +763,7 @@ class MongoDBJavaScriptStack(gdb.Command):
     """Print the JavaScript stack from a MongoDB process."""
 
     # Looking to test your changes to this? Really easy!
-    # 1. install-core to build the mongo shell binary (mongo)
+    # 1. install-jstestshell to build the mongo shell binary (mongo)
     # 2. launch it: ./path/to/bin/mongo --nodb
     # 3. in the shell, run: sleep(99999999999). (do not use --eval)
     # 4. ps ax | grep nodb to find the PID

buildscripts/gdb/mongo_printers.py

@@ -579,6 +579,7 @@ def find_match_brackets(search, opening='<', closing='>'):
     Example:
         'Foo<T>::iterator<U>''
         returns 5
+
     """
     index = search.find(opening)
     if index == -1:

buildscripts/generate_compile_expansions.py

@@ -7,9 +7,7 @@ $ python generate_compile_expansions.py --out compile_expansions.yml
 """
 
 import argparse
-import json
 import os
-import re
 import sys
 import shlex
 import yaml
@@ -25,7 +23,6 @@ def generate_expansions():
     """
     args = parse_args()
     expansions = {}
-    expansions.update(generate_version_expansions())
     expansions.update(generate_scons_cache_expansions())
 
     with open(args.out, "w") as out:
@@ -40,39 +37,6 @@ def parse_args():
     return parser.parse_args()
 
 
-def generate_version_expansions():
-    """Generate expansions from a version.json file if given, or $MONGO_VERSION."""
-    expansions = {}
-
-    if os.path.exists(VERSION_JSON):
-        with open(VERSION_JSON, "r") as fh:
-            data = fh.read()
-            version_data = json.loads(data)
-        version_line = version_data['version']
-        version_parts = match_verstr(version_line)
-        if not version_parts:
-            raise ValueError("Unable to parse version.json")
-    else:
-        if not os.getenv("MONGO_VERSION"):
-            raise Exception("$MONGO_VERSION not set and no version.json provided")
-        version_line = os.getenv("MONGO_VERSION").lstrip("r")
-        version_parts = match_verstr(version_line)
-        if not version_parts:
-            raise ValueError("Unable to parse version from stdin and no version.json provided")
-
-    if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
-        expansions["is_release"] = "false"
-    else:
-        expansions["suffix"] = version_line
-        expansions["src_suffix"] = "r{0}".format(version_line)
-        expansions["is_release"] = "true"
-    expansions["version"] = version_line
-
-    return expansions
-
-
 def generate_scons_cache_expansions():
     """Generate scons cache expansions from some files and environment variables."""
     expansions = {}
@@ -101,21 +65,5 @@ def generate_scons_cache_expansions():
     return expansions
 
 
-def match_verstr(verstr):
-    """Match a version string and capture the "extra" part.
-
-    If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
-    None. If the version is a pre-release like "2.3.4-325-githash" or
-    "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
-    begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
-    r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
-    doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
-    """
-    res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
-    if not res:
-        return False
-    return res.groups()
-
-
 if __name__ == "__main__":
     generate_expansions()

buildscripts/generate_compile_expansions_shared_cache.py

@@ -7,9 +7,7 @@ $ python generate_compile_expansions.py --out compile_expansions.yml
 """
 
 import argparse
-import json
 import os
-import re
 import sys
 import shlex
 import yaml
@@ -25,7 +23,6 @@ def generate_expansions():
     """
     args = parse_args()
     expansions = {}
-    expansions.update(generate_version_expansions())
     expansions.update(generate_scons_cache_expansions())
 
     with open(args.out, "w") as out:
@@ -40,39 +37,6 @@ def parse_args():
     return parser.parse_args()
 
 
-def generate_version_expansions():
-    """Generate expansions from a version.json file if given, or $MONGO_VERSION."""
-    expansions = {}
-
-    if os.path.exists(VERSION_JSON):
-        with open(VERSION_JSON, "r") as fh:
-            data = fh.read()
-            version_data = json.loads(data)
-        version_line = version_data['version']
-        version_parts = match_verstr(version_line)
-        if not version_parts:
-            raise ValueError("Unable to parse version.json")
-    else:
-        if not os.getenv("MONGO_VERSION"):
-            raise Exception("$MONGO_VERSION not set and no version.json provided")
-        version_line = os.getenv("MONGO_VERSION").lstrip("r")
-        version_parts = match_verstr(version_line)
-        if not version_parts:
-            raise ValueError("Unable to parse version from stdin and no version.json provided")
-
-    if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
-        expansions["is_release"] = "false"
-    else:
-        expansions["suffix"] = version_line
-        expansions["src_suffix"] = "r{0}".format(version_line)
-        expansions["is_release"] = "true"
-    expansions["version"] = version_line
-
-    return expansions
-
-
 def generate_scons_cache_expansions():
     """Generate scons cache expansions from some files and environment variables."""
     expansions = {}
@@ -125,21 +89,5 @@ def generate_scons_cache_expansions():
     return expansions
 
 
-def match_verstr(verstr):
-    """Match a version string and capture the "extra" part.
-
-    If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
-    None. If the version is a pre-release like "2.3.4-325-githash" or
-    "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
-    begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
-    r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
-    doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
-    """
-    res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
-    if not res:
-        return False
-    return res.groups()
-
-
 if __name__ == "__main__":
     generate_expansions()

buildscripts/generate_version_expansions.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""
+Generate the version expansions file used by Evergreen as part of the push/release process.
+
+Invoke by specifying an output file.
+$ python generate_build_expansions.py --out version_expansions.yml
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+import yaml
+
+VERSION_JSON = "version.json"
+
+
+def generate_expansions():
+    """Entry point for the script.
+
+    This calls functions to generate version and scons cache expansions and
+    writes them to a file.
+    """
+    args = parse_args()
+    expansions = {}
+    expansions.update(generate_version_expansions())
+
+    with open(args.out, "w") as out:
+        print("saving compile expansions to {0}: ({1})".format(args.out, expansions))
+        yaml.safe_dump(expansions, out, default_flow_style=False)
+
+
+def parse_args():
+    """Parse program arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", required=True)
+    return parser.parse_args()
+
+
+def generate_version_expansions():
+    """Generate expansions from a version.json file if given, or $MONGO_VERSION."""
+    expansions = {}
+
+    if os.path.exists(VERSION_JSON):
+        with open(VERSION_JSON, "r") as fh:
+            data = fh.read()
+            version_data = json.loads(data)
+        version_line = version_data['version']
+        version_parts = match_verstr(version_line)
+        if not version_parts:
+            raise ValueError("Unable to parse version.json")
+    else:
+        version_line = os.getenv("MONGO_VERSION")
+        if not version_line:
+            raise Exception("$MONGO_VERSION not set and no version.json provided")
+
+        version_line = version_line.lstrip("r")
+        version_parts = match_verstr(version_line)
+        if not version_parts:
+            raise ValueError("Unable to parse version from stdin and no version.json provided")
+
+    if version_parts[0]:
+        expansions["suffix"] = "latest"
+        expansions["src_suffix"] = "latest"
+        expansions["is_release"] = "false"
+    else:
+        expansions["suffix"] = version_line
+        expansions["src_suffix"] = "r{0}".format(version_line)
+        expansions["is_release"] = "true"
+    expansions["version"] = version_line
+
+    return expansions
+
+
+def match_verstr(verstr):
+    """Match a version string and capture the "extra" part.
+
+    If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
+    None. If the version is a pre-release like "2.3.4-325-githash" or
+    "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
+    begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
+    r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
+    doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
+    """
+    res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
+    if not res:
+        return False
+    return res.groups()
+
+
+if __name__ == "__main__":
+    generate_expansions()

buildscripts/idl/check_stable_api_commands_have_idl_definitions.py

@@ -96,16 +96,16 @@ def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir:
     if mongod_or_mongos == "mongod":
         logger = loggers.new_fixture_logger("MongoDFixture", 0)
         logger.parent = LOGGER
-        fixture: interface.Fixture = fixturelib.make_fixture("MongoDFixture", logger, 0,
-                                                             dbpath_prefix=dbpath.name,
-                                                             mongod_executable=mongod_executable)
+        fixture: interface.Fixture = fixturelib.make_fixture(
+            "MongoDFixture", logger, 0, dbpath_prefix=dbpath.name,
+            mongod_executable=mongod_executable, mongod_options={"set_parameters": {}})
     else:
         logger = loggers.new_fixture_logger("ShardedClusterFixture", 0)
         logger.parent = LOGGER
-        fixture = fixturelib.make_fixture("ShardedClusterFixture", logger, 0,
-                                          dbpath_prefix=dbpath.name,
-                                          mongos_executable=mongos_executable,
-                                          mongod_executable=mongod_executable, mongod_options={})
+        fixture = fixturelib.make_fixture(
+            "ShardedClusterFixture", logger, 0, dbpath_prefix=dbpath.name,
+            mongos_executable=mongos_executable, mongod_executable=mongod_executable,
+            mongod_options={"set_parameters": {}})
 
     fixture.setup()
     fixture.await_ready()
@@ -178,6 +178,7 @@ def main():
 
         def __init__(self):
             self.INSTALL_DIR = args.install_dir  # pylint: disable=invalid-name
+            self.command = ""
 
     # pylint: disable=protected-access
     configure_resmoke._update_config_vars(FakeArgs())

buildscripts/idl/gen_all_feature_flag_list.py

@@ -30,7 +30,6 @@ Generate a file containing a list of disabled feature flags.
 Used by resmoke.py to run only feature flag tests.
 """
 
-import argparse
 import os
 import sys
 
@@ -43,6 +42,7 @@ sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../
 
 # pylint: disable=wrong-import-position
 import buildscripts.idl.lib as lib
+from buildscripts.idl.idl import parser
 
 
 def is_third_party_idl(idl_path: str) -> bool:
@@ -56,13 +56,14 @@ def is_third_party_idl(idl_path: str) -> bool:
     return False
 
 
-def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
+def gen_all_feature_flags(idl_dir: str = os.getcwd()):
     """Generate a list of all feature flags."""
     all_flags = []
     for idl_path in sorted(lib.list_idls(idl_dir)):
         if is_third_party_idl(idl_path):
             continue
-        for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags:
+        doc = parser.parse_file(open(idl_path), idl_path)
+        for feature_flag in doc.spec.feature_flags:
             if feature_flag.default.literal != "true":
                 all_flags.append(feature_flag.name)
 
@@ -72,18 +73,17 @@ def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
     return list(set(all_flags) - set(force_disabled_flags))
 
 
+def gen_all_feature_flags_file(filename: str = lib.ALL_FEATURE_FLAG_FILE):
+    """Output generated list of feature flags to specified file."""
+    flags = gen_all_feature_flags()
+    with open(filename, "w") as output_file:
+        output_file.write("\n".join(flags))
+        print("Generated: ", os.path.realpath(output_file.name))
+
+
 def main():
     """Run the main function."""
-    arg_parser = argparse.ArgumentParser(description=__doc__)
-    arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append",
-                            help="Directory to search for IDL import files")
-
-    args = arg_parser.parse_args()
-
-    flags = gen_all_feature_flags(os.getcwd(), args.import_dirs)
-    with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file:
-        for flag in flags:
-            output_file.write("%s\n" % flag)
+    gen_all_feature_flags_file()
 
 
 if __name__ == '__main__':

buildscripts/idl/idl/bson.py

@@ -72,6 +72,7 @@ _BINDATA_SUBTYPE = {
     # },
     "uuid": {'scalar': True, 'bindata_enum': 'newUUID'},
     "md5": {'scalar': True, 'bindata_enum': 'MD5Type'},
+    "encrypt": {'scalar': True, 'bindata_enum': 'Encrypt'},
 }
 
 

buildscripts/idl/idl/errors.py

@@ -37,7 +37,6 @@ import inspect
 import os
 import sys
 from typing import List, Union
-from yaml import nodes
 import yaml
 
 from . import common

buildscripts/idl/idl/parser.py

@@ -1001,7 +1001,7 @@ def _propagate_globals(spec):
         idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type)
 
 
-def _parse(stream, error_file_name):
+def parse_file(stream, error_file_name):
     # type: (Any, str) -> syntax.IDLParsedSpec
     """
     Parse a YAML document into an idl.syntax tree.
@@ -1105,7 +1105,7 @@ def parse(stream, input_file_name, resolver):
     """
     # pylint: disable=too-many-locals
 
-    root_doc = _parse(stream, input_file_name)
+    root_doc = parse_file(stream, input_file_name)
 
     if root_doc.errors:
         return root_doc
@@ -1142,7 +1142,7 @@ def parse(stream, input_file_name, resolver):
 
         # Parse imported file
         with resolver.open(resolved_file_name) as file_stream:
-            parsed_doc = _parse(file_stream, resolved_file_name)
+            parsed_doc = parse_file(file_stream, resolved_file_name)
 
         # Check for errors
         if parsed_doc.errors:

buildscripts/idl/idl_check_compatibility.py

@@ -707,7 +707,10 @@ def check_param_or_command_type_recursive(ctxt: IDLCompatibilityContext,
             # Check that new variant types are a superset of old variant types.
             for old_variant_type in old_variant_types:
                 for new_variant_type in new_variant_types:
-                    if old_variant_type.name == new_variant_type.name:
+                    # object->object_owned serialize to the same bson type. object_owned->object is
+                    # not always safe so we only limit this special case to object->object_owned.
+                    if (old_variant_type.name == "object" and new_variant_type.name == "object_owned") or \
+                        old_variant_type.name == new_variant_type.name:
                         # Check that the old and new version of each variant type is also compatible.
                         old = FieldCompatibility(old_variant_type, old_field.idl_file,
                                                  old_field.idl_file_path, old_field.unstable,

buildscripts/linter/git_base.py

@@ -236,6 +236,7 @@ class GitException(Exception):
         process_args: a list containing the git command and arguments (includes 'git' as its first
             element) that were run, if any.
         stderr: the error output of the git command.
+
     """
 
     def __init__(  # pylint: disable=too-many-arguments
@@ -258,6 +259,7 @@ class GitCommandResult(object):
         returncode: the return code.
         stdout: the output of the command.
         stderr: the error output of the command.
+
     """
 
     def __init__(  # pylint: disable=too-many-arguments

buildscripts/linter/parallel.py

@@ -9,7 +9,11 @@ from typing import Any, Callable, List
 
 def parallel_process(items, func):
     # type: (List[Any], Callable[[Any], bool]) -> bool
-    """Run a set of work items to completion and wait."""
+    """
+    Run a set of work items to completion and wait.
+
+    :returns whether all tasks were successful.
+    """
     try:
         cpus = cpu_count()
     except NotImplementedError:
@@ -17,20 +21,16 @@ def parallel_process(items, func):
 
     task_queue = queue.Queue()  # type: queue.Queue
 
-    # Use a list so that worker function will capture this variable
-    pp_event = threading.Event()
-    pp_result = [True]
-    pp_lock = threading.Lock()
+    has_failure_event = threading.Event()
 
     def worker():
         # type: () -> None
         """Worker thread to process work items in parallel."""
-        while not pp_event.is_set():
+        while True:
             try:
                 item = task_queue.get_nowait()
             except queue.Empty:
                 # if the queue is empty, exit the worker thread
-                pp_event.set()
                 return
 
             try:
@@ -39,13 +39,8 @@ def parallel_process(items, func):
                 # Tell the queue we finished with the item
                 task_queue.task_done()
 
-            # Return early if we fail, and signal we are done
             if not ret:
-                with pp_lock:
-                    pp_result[0] = False
-
-                pp_event.set()
-                return
+                has_failure_event.set()
 
     # Enqueue all the work we want to process
     for item in items:
@@ -62,10 +57,10 @@ def parallel_process(items, func):
 
     # Wait for the threads to finish
     # Loop with a timeout so that we can process Ctrl-C interrupts
-    while not pp_event.wait(1):
+    while not queue.Empty:
         time.sleep(1)
 
     for thread in threads:
         thread.join()
 
-    return pp_result[0]
+    return not has_failure_event.is_set()

buildscripts/linter/pydocstyle.py

@@ -11,7 +11,7 @@ class PyDocstyleLinter(base.LinterBase):
     def __init__(self):
         # type: () -> None
         """Create a pydocstyle linter."""
-        super(PyDocstyleLinter, self).__init__("pydocstyle", "2.1.1")
+        super(PyDocstyleLinter, self).__init__("pydocstyle", "6.1.1")
 
     def get_lint_version_cmd_args(self):
         # type: () -> List[str]

buildscripts/moduleconfig.py

@@ -33,16 +33,26 @@ import os
 
 
 def discover_modules(module_root, allowed_modules):
+    # pylint: disable=too-many-branches
     """Scan module_root for subdirectories that look like MongoDB modules.
 
     Return a list of imported build.py module objects.
     """
     found_modules = []
+    found_module_names = []
 
     if allowed_modules is not None:
         allowed_modules = allowed_modules.split(',')
+        # When `--modules=` is passed, the split on empty string is represented
+        # in memory as ['']
+        if allowed_modules == ['']:
+            allowed_modules = []
 
     if not os.path.isdir(module_root):
+        if allowed_modules:
+            raise RuntimeError(
+                f"Requested the following modules: {allowed_modules}, but the module root '{module_root}' could not be found. Check the module root, or remove the module from the scons invocation."
+            )
         return found_modules
 
     for name in os.listdir(module_root):
@@ -66,11 +76,17 @@ def discover_modules(module_root, allowed_modules):
                 if getattr(module, "name", None) is None:
                     module.name = name
                 found_modules.append(module)
+                found_module_names.append(name)
             finally:
                 fp.close()
         except (FileNotFoundError, IOError):
             pass
 
+    if allowed_modules is not None:
+        missing_modules = set(allowed_modules) - set(found_module_names)
+        if missing_modules:
+            raise RuntimeError(f"Failed to locate all modules. Could not find: {missing_modules}")
+
     return found_modules
 
 

buildscripts/mongosymb.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """Script and library for symbolizing MongoDB stack traces.
 
 To use as a script, paste the JSON object on the line after ----- BEGIN BACKTRACE ----- into the
@@ -24,20 +23,42 @@ import signal
 import subprocess
 import sys
 import time
+from abc import ABC, abstractmethod
 from collections import OrderedDict
+from datetime import timedelta
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List, Any, Union, Optional
 
 import requests
 
 # pylint: disable=wrong-import-position
 # pylint: disable=too-many-branches
+from tenacity import wait_fixed, stop_after_delay, retry_if_result, Retrying
+
 sys.path.append(str(Path(os.getcwd(), __file__).parent.parent))
-from buildscripts.util.oauth import Configs, get_oauth_credentials
+from buildscripts.util.oauth import Configs, get_oauth_credentials, get_client_cred_oauth_credentials
 from buildscripts.build_system_options import PathOptions
 
+SYMBOLIZER_PATH_ENV = "MONGOSYMB_SYMBOLIZER_PATH"
+# since older versions may have issues with symbolizing, we are setting the toolchain version to v4
+DEFAULT_SYMBOLIZER_PATH = "/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
 
-class PathDbgFileResolver(object):
+
+class DbgFileResolver(ABC):
+    """Base gdb path resolver class."""
+
+    @abstractmethod
+    def get_dbg_file(self, soinfo: Dict[str, Any]) -> Union[str, None]:
+        """
+        To get path for given build info.
+
+        :param soinfo: soinfo as dict
+        :return: path as string or None (if path not found)
+        """
+        raise NotImplementedError
+
+
+class PathDbgFileResolver(DbgFileResolver):
     """PathDbgFileResolver class."""
 
     def __init__(self, bin_path_guess):
@@ -54,7 +75,7 @@ class PathDbgFileResolver(object):
         return path if path else self._bin_path_guess
 
 
-class S3BuildidDbgFileResolver(object):
+class S3BuildidDbgFileResolver(DbgFileResolver):
     """S3BuildidDbgFileResolver class."""
 
     def __init__(self, cache_dir, s3_bucket):
@@ -73,7 +94,7 @@ class S3BuildidDbgFileResolver(object):
         if not os.path.exists(build_id_path):
             try:
                 self._get_from_s3(build_id)
-            except Exception:  # pylint: disable=broad-except
+            except Exception:  # noqa pylint: disable=broad-except
                 ex = sys.exc_info()[0]
                 sys.stderr.write("Failed to find debug symbols for {} in s3: {}\n".format(
                     build_id, ex))
@@ -147,7 +168,7 @@ class CachedResults(object):
         return self._cached_results.get(key)
 
 
-class PathResolver(object):
+class PathResolver(DbgFileResolver):
     """
     Class to find path for given buildId.
 
@@ -163,16 +184,17 @@ class PathResolver(object):
     # This amount of attributes are necessary.
 
     # the main (API) sever that we'll be sending requests to
-    default_host = 'https://symbolizer-service.server-tig.prod.corp.mongodb.com'
-    default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbolizer_downloads_cache')
-    default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json')
+    default_host = "https://symbolizer-service.server-tig.prod.corp.mongodb.com"
+    default_cache_dir = os.path.join(os.getcwd(), "build", "symbolizer_downloads_cache")
+    default_creds_file_path = os.path.join(os.getcwd(), ".symbolizer_credentials.json")
     default_client_credentials_scope = "servertig-symbolizer-fullaccess"
     default_client_credentials_user_name = "client-user"
+    download_timeout_secs = timedelta(minutes=4).total_seconds()
 
     def __init__(self, host: str = None, cache_size: int = 0, cache_dir: str = None,
                  client_credentials_scope: str = None, client_credentials_user_name: str = None,
-                 client_id: str = None, redirect_port: int = None, scope: str = None,
-                 auth_domain: str = None):
+                 client_id: str = None, client_secret: str = None, redirect_port: int = None,
+                 scope: str = None, auth_domain: str = None):
         """
         Initialize instance.
 
@@ -187,6 +209,7 @@ class PathResolver(object):
         self.client_credentials_scope = client_credentials_scope or self.default_client_credentials_scope
         self.client_credentials_user_name = client_credentials_user_name or self.default_client_credentials_user_name
         self.client_id = client_id
+        self.client_secret = client_secret
         self.redirect_port = redirect_port
         self.scope = scope
         self.auth_domain = auth_domain
@@ -212,11 +235,17 @@ class PathResolver(object):
                 data = json.loads(cfile.read())
                 access_token, expire_time = data.get("access_token"), data.get("expire_time")
                 if time.time() < expire_time:
-                    # credentials hasn't expired yet
+                    # credentials not expired yet
                     self.http_client.headers.update({"Authorization": f"Bearer {access_token}"})
                     return
 
-        credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True)
+        if self.client_id and self.client_secret:
+            # auth using secrets
+            credentials = get_client_cred_oauth_credentials(self.client_id, self.client_secret,
+                                                            self.configs)
+        else:
+            # since we don't have access to secrets, ask user to auth manually
+            credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True)
         self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"})
 
         # write credentials to local file for further useage
@@ -265,7 +294,7 @@ class PathResolver(object):
         :param url: download URL
         :return: full name for local file
         """
-        return url.split('/')[-1]
+        return url.split("/")[-1]
 
     @staticmethod
     def unpack(path: str) -> str:
@@ -273,9 +302,9 @@ class PathResolver(object):
         Use to utar/unzip files.
 
         :param path: full path of file
-        :return: full path of directory of unpacked file
+        :return: full path to directory of unpacked file
         """
-        out_dir = path.replace('.tgz', '', 1)
+        out_dir = path.replace(".tgz", "", 1)
         if not os.path.exists(out_dir):
             os.mkdir(out_dir)
 
@@ -296,12 +325,25 @@ class PathResolver(object):
         filename = self.url_to_filename(url)
         path = os.path.join(self.cache_dir, filename)
         if not os.path.exists(path):
-            subprocess.check_call(['wget', url], cwd=self.cache_dir)
+            print("Downloading the file...")
+            self.get_file_from_service(url, path)
         else:
-            print('File aready exists in cache')
+            print("File already exists in cache")
             exists_locally = True
         return path, exists_locally
 
+    def get_file_from_service(self, url: str, local_path: str) -> None:
+        """
+        Get file from URL and write to a local file.
+
+        :param url: URL string
+        :param local_path: full name for local file
+        """
+        with requests.get(url, stream=True, timeout=self.download_timeout_secs) as response:
+            with open(local_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=2 * 1024 * 1024):
+                    file.write(chunk)
+
     def get_dbg_file(self, soinfo: dict) -> str or None:
         """
         To get path for given buildId.
@@ -310,32 +352,26 @@ class PathResolver(object):
         :return: path as string or None (if path not found)
         """
         build_id = soinfo.get("buildId", "").lower()
-        binary_name = 'mongo'
+        version = soinfo.get("version")
+        binary_name = "mongo"
         # search from cached results
         path = self.get_from_cache(build_id)
         if not path:
             # path does not exist in cache, so we send request to server
             try:
-                response = self.http_client.get(f'{self.host}/find_by_id',
-                                                params={'build_id': build_id})
+                search_parameters = {"build_id": build_id}
+                if version:
+                    search_parameters["version"] = version
+                print(f"Getting data from service... Search parameters: {search_parameters}")
+                response = self.http_client.get(f"{self.host}/find_by_id", params=search_parameters)
                 if response.status_code != 200:
-                    # if we could not find the path of binary, that might be system library.
-                    # we can try using frame's own `path` data.
-                    # symbolization can succeed only if that binary exists on local
-                    # machine (more specifically: in the given path).
-                    system_path = soinfo.get('path')
-                    if system_path:
-                        sys.stdout.write(
-                            f"Could not find path of binary from symbolizer web service. Trying to use the "
-                            f"provided path: {system_path}\n")
-                        return system_path
                     sys.stderr.write(
                         f"Server returned unsuccessful status: {response.status_code}, "
                         f"response body: {response.text}\n")
                     return None
                 else:
-                    data = response.json().get('data', {})
-                    path, binary_name = data.get('debug_symbols_url'), data.get('file_name')
+                    data = response.json().get("data", {})
+                    path, binary_name = data.get("debug_symbols_url"), data.get("file_name")
             except Exception as err:  # noqa pylint: disable=broad-except
                 sys.stderr.write(f"Error occurred while trying to get response from server "
                                  f"for buildId({build_id}): {err}\n")
@@ -352,7 +388,7 @@ class PathResolver(object):
         try:
             dl_path, exists_locally = self.download(path)
             if exists_locally:
-                path = dl_path.replace('.tgz', '', 1)
+                path = dl_path.replace(".tgz", "", 1)
             else:
                 print("Downloaded, now unpacking...")
                 path = self.unpack(dl_path)
@@ -362,8 +398,8 @@ class PathResolver(object):
         # if file has extension, it is good. if not, we should append .debug, because those without extension are
         # from release builds, and their debug symbol files contain .debug extension.
         # we need to map those 2 different file names ('<name>' becomes '<name>.debug').
-        if not binary_name.endswith('.debug') and not binary_name.endswith('.so'):
-            binary_name = f'{binary_name}.debug'
+        if not binary_name.endswith(".debug"):
+            binary_name = f"{binary_name}.debug"
 
         inner_folder_name = self.path_options.get_binary_folder_name(binary_name)
 
@@ -381,6 +417,7 @@ def parse_input(trace_doc, dbg_path_resolver):
         return {so_entry["b"]: so_entry for so_entry in somap_list if "b" in so_entry}
 
     base_addr_map = make_base_addr_map(trace_doc["processInfo"]["somap"])
+    version = get_version(trace_doc)
 
     frames = []
     for frame in trace_doc["backtrace"]:
@@ -390,6 +427,8 @@ def parse_input(trace_doc, dbg_path_resolver):
             )
             continue
         soinfo = base_addr_map.get(frame["b"], {})
+        if version:
+            soinfo["version"] = version
         elf_type = soinfo.get("elfType", 0)
         if elf_type == 3:
             addr_base = "0"
@@ -399,7 +438,7 @@ def parse_input(trace_doc, dbg_path_resolver):
             addr_base = soinfo.get("vmaddr", "0")
         addr = int(addr_base, 16) + int(frame["o"], 16)
         # addr currently points to the return address which is the one *after* the call. x86 is
-        # variable length so going backwards is difficult. However llvm-symbolizer seems to do the
+        # variable length so going backwards is difficult. However, llvm-symbolizer seems to do the
         # right thing if we just subtract 1 byte here. This has the downside of also adjusting the
         # address of instructions that cause signals (such as segfaults and divide-by-zero) which
         # are already correct, but there doesn't seem to be a reliable way to detect that case.
@@ -411,6 +450,16 @@ def parse_input(trace_doc, dbg_path_resolver):
     return frames
 
 
+def get_version(trace_doc: Dict[str, Any]) -> Optional[str]:
+    """
+    Get version from trace doc.
+
+    :param trace_doc: Traceback dict.
+    :return: Version string or None.
+    """
+    return trace_doc.get("processInfo", {}).get("mongodbVersion")
+
+
 def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, input_format,
                      **kwargs):
     """Return a list of symbolized stack frames from a trace_doc in MongoDB stack dump format."""
@@ -418,24 +467,23 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i
     # Keep frames in kwargs to avoid changing the function signature.
     frames = kwargs.get("frames")
     if frames is None:
-        frames = preprocess_frames(dbg_path_resolver, trace_doc, input_format)
+        total_seconds_for_retries = kwargs.get("total_seconds_for_retries", 0)
+        frames = preprocess_frames_with_retries(dbg_path_resolver, trace_doc, input_format,
+                                                total_seconds_for_retries)
 
     if not symbolizer_path:
-        symbolizer_path_env = "MONGOSYMB_SYMBOLIZER_PATH"
-        default_symbolizer_path = "llvm-symbolizer"
-        symbolizer_path = os.environ.get(symbolizer_path_env)
+        symbolizer_path = os.environ.get(SYMBOLIZER_PATH_ENV)
         if not symbolizer_path:
-            print(
-                f"Env value for '{symbolizer_path_env}' not found, using '{default_symbolizer_path}' "
-                f"as a defualt executable path.")
-            symbolizer_path = default_symbolizer_path
+            print(f"Env value for '{SYMBOLIZER_PATH_ENV}' not found, using"
+                  f" '{DEFAULT_SYMBOLIZER_PATH}' as a default executable path.")
+            symbolizer_path = DEFAULT_SYMBOLIZER_PATH
 
     symbolizer_args = [symbolizer_path]
     for dh in dsym_hint:
         symbolizer_args.append("-dsym-hint={}".format(dh))
     symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True,
                                           stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-                                          stderr=open("/dev/null"))
+                                          stderr=sys.stdout)
 
     def extract_symbols(stdin):
         """Extract symbol information from the output of llvm-symbolizer.
@@ -467,7 +515,7 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i
 
     for frame in frames:
         if frame["path"] is None:
-            print("Path not found in frame:", frame)
+            print(f"Path not found in frame: {frame}")
             continue
         symbol_line = "CODE {path:} {addr:}\n".format(**frame)
         symbolizer_process.stdin.write(symbol_line.encode())
@@ -478,8 +526,17 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i
     return frames
 
 
-def preprocess_frames(dbg_path_resolver, trace_doc, input_format):
-    """Process the paths in frame objects."""
+def preprocess_frames(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any],
+                      input_format: str) -> List[Dict[str, Any]]:
+    """
+    Process the paths in frame objects.
+
+    :param dbg_path_resolver: debug symbols file path resolver
+    :param trace_doc: traceback object
+    :param input_format: format of input
+    :return: the list of traceback frames
+    """
+
     if input_format == "classic":
         frames = parse_input(trace_doc, dbg_path_resolver)
     elif input_format == "thin":
@@ -488,9 +545,43 @@ def preprocess_frames(dbg_path_resolver, trace_doc, input_format):
             frame["path"] = dbg_path_resolver.get_dbg_file(frame)
     else:
         raise ValueError('Unknown input format "{}"'.format(input_format))
+
     return frames
 
 
+def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool:
+    """
+    Check whether not found paths in frames ratio is higher than 0.5.
+
+    :param frames: the list of traceback frames
+    :return: True if ratio is higher than 0.5
+    """
+    not_found = [1 for f in frames if f.get("path") is None]
+    not_found_ratio = len(not_found) / (len(frames) or 1)
+    return not_found_ratio >= 0.5
+
+
+def preprocess_frames_with_retries(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any],
+                                   input_format: str,
+                                   total_seconds_for_retries: int = 0) -> List[Dict[str, Any]]:
+    """
+    Process the paths in frame objects.
+
+    :param dbg_path_resolver: debug symbols file path resolver
+    :param trace_doc: traceback object
+    :param input_format: format of input
+    :param total_seconds_for_retries: max wait time for retries in seconds
+    :return: the list of traceback frames
+    """
+
+    retrying = Retrying(
+        retry=retry_if_result(has_high_not_found_paths_ratio), wait=wait_fixed(60),
+        stop=stop_after_delay(total_seconds_for_retries),
+        retry_error_callback=lambda retry_state: retry_state.outcome.result())
+
+    return retrying(preprocess_frames, dbg_path_resolver, trace_doc, input_format)
+
+
 def classic_output(frames, outfile, **kwargs):  # pylint: disable=unused-argument
     """Provide classic output."""
     for frame in frames:
@@ -516,6 +607,12 @@ def make_argument_parser(parser=None, **kwargs):
     parser.add_argument('--debug-file-resolver', choices=['path', 's3', 'pr'], default='pr')
     parser.add_argument('--src-dir-to-move', action="store", type=str, default=None,
                         help="Specify a src dir to move to /data/mci/{original_buildid}/src")
+    parser.add_argument(
+        '--total-seconds-for-retries', default=0, type=int,
+        help="If web service fails to find path for given build id, it could be because mapping "
+        "process was not finished yet. We can wait for it to finish and retry again. Each retry"
+        " adds 2 minutes to previous wait time. It is guaranteed that total wait time does not exceed this "
+        "specified amount.")
 
     parser.add_argument('--live', action='store_true')
     s3_group = parser.add_argument_group(
@@ -531,6 +628,8 @@ def make_argument_parser(parser=None, **kwargs):
                           help='URL of web service running the API to get debug symbol URL')
     pr_group.add_argument('--pr-cache-dir', default='',
                           help='Full path to a directory to store cache/files')
+    pr_group.add_argument('--client-secret', default='', help='Secret key for Okta Oauth')
+    pr_group.add_argument('--client-id', default='', help='Client id for Okta Oauth')
     # caching mechanism is currently not fully developed and needs more advanced cleaning techniques, we add an option
     # to enable it after completing the implementation
 
@@ -565,11 +664,17 @@ def substitute_stdin(options, resolver):
             if not trace_doc["backtrace"]:
                 print("Trace is empty, skipping...")
                 continue
-            frames = symbolize_frames(trace_doc, resolver, options.symbolizer_path, [],
-                                      options.output_format)
+            frames = symbolize_frames(
+                trace_doc,
+                resolver,
+                options.symbolizer_path,
+                [],
+                options.output_format,
+            )
             print(prefix)
             print("Symbolizing...")
             classic_output(frames, sys.stdout, indent=2)
+            print("Completed, waiting for input...")
         else:
             print(line)
 
@@ -583,7 +688,8 @@ def main(options):
     elif options.debug_file_resolver == 's3':
         resolver = S3BuildidDbgFileResolver(options.s3_cache_dir, options.s3_bucket)
     elif options.debug_file_resolver == 'pr':
-        resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir)
+        resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir,
+                                client_secret=options.client_secret, client_id=options.client_id)
 
     if options.live:
         print("Entering live mode")
@@ -597,7 +703,7 @@ def main(options):
 
     if not trace_doc or not trace_doc.strip():
         print("Please provide the backtrace through stdin for symbolization;"
-              "e.g. `your/symbolization/command < /file/with/stacktrace`")
+              " e.g. `your/symbolization/command < /file/with/stacktrace`")
 
     # Search the trace_doc for an object having "backtrace" and "processInfo" keys.
     def bt_search(obj):
@@ -624,8 +730,8 @@ def main(options):
         except json.JSONDecodeError:
             pass
     else:
-        print("could not find json backtrace object in input", file=sys.stderr)
-        exit(1)
+        sys.stderr.write("could not find json backtrace object in input\n")
+        sys.exit(1)
 
     output_fn = None
     if options.output_format == 'json':
@@ -633,7 +739,8 @@ def main(options):
     if options.output_format == 'classic':
         output_fn = classic_output
 
-    frames = preprocess_frames(resolver, trace_doc, options.input_format)
+    frames = preprocess_frames_with_retries(resolver, trace_doc, options.input_format,
+                                            options.total_seconds_for_retries)
 
     if options.src_dir_to_move and resolver.mci_build_dir is not None:
         try:

buildscripts/package_test/kitchen.legacy.yml

@@ -41,21 +41,6 @@ platforms:
       instance_type: c6g.medium
     transport:
       username: ec2-user
-  - name: debian71-x86-64
-    driver:
-      image_id: ami-4b124a22
-    transport:
-      username: admin
-  - name: debian81-x86-64
-    driver:
-      image_id: ami-896d85e2
-    transport:
-      username: admin
-  - name: debian92-x86-64
-    driver:
-      image_id: ami-71b7750b
-    transport:
-      username: admin
   - name: debian10-x86-64
     driver:
       image_id: ami-0dedf6a6502877301

buildscripts/package_test/kitchen.yml

@@ -23,7 +23,7 @@ provisioner:
   name: chef_zero
   log_level: info
   product_name: cinc
-  product_version: 16.7.61
+  product_version: 17.9.52
   download_url: https://omnitruck.cinc.sh/install.sh
 
 platforms:
@@ -34,7 +34,7 @@ platforms:
       username: ec2-user
   - name: amazon2-x86-64
     driver:
-      image_id: ami-428aa838
+      image_id: ami-02013ed1a71752ea7
     transport:
       username: ec2-user
   - name: amazon2-arm64
@@ -53,11 +53,6 @@ platforms:
       image_id: ami-896d85e2
     transport:
       username: admin
-  - name: debian92-x86-64
-    driver:
-      image_id: ami-71b7750b
-    transport:
-      username: admin
   - name: debian10-x86-64
     driver:
       image_id: ami-0dedf6a6502877301
@@ -70,7 +65,7 @@ platforms:
       username: admin
   - name: rhel70-x86-64
     driver:
-      image_id: ami-2051294a
+      image_id: ami-0051b1b2c5a166c8c
     transport:
       username: root
   - name: rhel80-x86-64
@@ -84,6 +79,12 @@ platforms:
       instance_type: m6g.medium
     transport:
       username: ec2-user
+  - name: rhel90-x86-64
+    driver:
+      image_id: ami-0c41531b8d18cc72b
+      user_data: rhel-user-data.sh
+    transport:
+      username: ec2-user
   - name: suse11-x86-64
     driver:
       image_id: ami-7f2e6015
@@ -132,6 +133,19 @@ platforms:
     driver:
       image_id: ami-00579fbb15b954340
       instance_type: m6g.medium
+  - name: ubuntu2204-x86-64
+    driver:
+      image_id: ami-051dcca84f1edfff1
+      user_data: ubuntu-user-data.sh
+    transport:
+      username: ubuntu
+  - name: ubuntu2204-arm64
+    driver:
+      image_id: ami-070650c005cce4203
+      instance_type: m6g.medium
+      user_data: ubuntu-user-data.sh
+    transport:
+      username: ubuntu
 
 transport:
   ssh_key: ~/.ssh/kitchen.pem

buildscripts/package_test/recipes/install_mongodb.rb

@@ -51,11 +51,21 @@ if platform_family? 'debian'
   ENV['DEBIAN_FRONTEND'] = 'noninteractive'
   package 'openssl'
 
-  # the ubuntu 16.04 image does not have some dependencies installed by default
+  # the ubuntu image does not have some dependencies installed by default
   # and it is required for the install_compass script
-  execute 'install dependencies' do
-    command 'apt-get install -y python libsasl2-modules-gssapi-mit'
-    live_stream true
+  if node['platform'] == 'ubuntu' and node['platform_version'] >= '20.04'
+    execute 'install dependencies ubuntu 20.04' do
+      command 'apt-get install -y python3 libsasl2-modules-gssapi-mit'
+      live_stream true
+    end
+    link '/usr/bin/python' do
+      to '/usr/bin/python3'
+    end
+  else
+    execute 'install dependencies' do
+      command 'apt-get install -y python libsasl2-modules-gssapi-mit'
+      live_stream true
+    end
   end
 
   # dpkg returns 1 if dependencies are not satisfied, which they will not be
@@ -82,12 +92,6 @@ if platform_family? 'debian'
     command 'apt update && apt -y -f install'
     live_stream true
   end
-
-  execute 'install mongo shell' do
-    command 'dpkg -i `find . -name "*shell*.deb"`'
-    live_stream true
-    cwd homedir
-  end
 end
 
 if platform_family? 'rhel' or platform_family? 'amazon'
@@ -96,6 +100,14 @@ if platform_family? 'rhel' or platform_family? 'amazon'
       sleep 120
     EOH
   end
+  #rhel9 doesn't have Gconf2 without epel
+  if node['platform'] == 'redhat' and node['platform_version'] == '9.0'
+    execute 'install epel' do
+      command 'dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm -y'
+      live_stream true
+      cwd homedir
+    end
+  end
   execute 'install mongod' do
     command 'yum install -y `find . -name "*server*.rpm"`'
     live_stream true
@@ -108,12 +120,6 @@ if platform_family? 'rhel' or platform_family? 'amazon'
     live_stream true
     cwd homedir
   end
-
-  execute 'install mongo shell' do
-    command 'yum install -y `find . -name "*shell*.rpm"`'
-    live_stream true
-    cwd homedir
-  end
 end
 
 if platform_family? 'suse'
@@ -145,32 +151,4 @@ if platform_family? 'suse'
     live_stream true
     cwd homedir
   end
-
-  execute 'install mongo' do
-    command 'zypper --no-gpg-checks -n install `find . -name "*shell*.rpm"`'
-    live_stream true
-    cwd homedir
-  end
-end
-
-inspec_wait = <<HEREDOC
-#!/bin/bash -x
-ulimit -v unlimited
-for i in {1..60}
-do
-  mongo --eval "db.smoke.insert({answer: 42})"
-  if [ $? -eq 0 ]
-  then
-    exit 0
-  else
-    echo "sleeping"
-    sleep 1
-  fi
-done
-exit 1
-HEREDOC
-
-file '/inspec_wait.sh' do
-  content inspec_wait
-  mode '0755'
 end

buildscripts/package_test/rhel-user-data.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+sed -i '/^PubkeyAcceptedAlgorithms/s/$/,ssh-rsa/'  /etc/crypto-policies/back-ends/opensshserver.config
+sed -i '/^HostKeyAlgorithms/s/$/,ssh-rsa/'  /etc/crypto-policies/back-ends/opensshserver.config
+update-crypto-policies --set LEGACY
+systemctl restart sshd
+

buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb

@@ -78,11 +78,6 @@ else
   end
 end
 
-# wait to make sure mongod is ready
-describe command("/inspec_wait.sh") do
-  its('exit_status') { should eq 0 }
-end
-
 ############################################################
 # This section verifies files, directories, and users
 # - files and directories exist and have correct attributes
@@ -169,8 +164,7 @@ if deb
     # account shells
     its('shell') {
       if ((os[:name] == 'debian' and os[:release].split('.')[0] >= '10') or
-          (os[:name] == 'ubuntu' and os[:release] == '18.04') or
-          (os[:name] == 'ubuntu' and os[:release] == '20.04'))
+          (os[:name] == 'ubuntu' and os[:release] >= '18.04'))
         should eq '/usr/sbin/nologin'
       else
         should eq '/bin/false'
@@ -199,23 +193,6 @@ ulimits.each do |limit, value|
   end
 end
 
-############################################################
-# This section verifies reads and writes.
-# - insert a document into the database
-# - verify that findOne() returns a matching document
-############################################################
-
-describe command('sh -c "ulimit -v unlimited && mongo --eval \"db.smoke.insert({answer: 42})\""') do
-  its('exit_status') { should eq 0 }
-  its('stdout') { should match(/.+WriteResult\({ "nInserted" : 1 }\).+/m) }
-end
-
-# read a document from the db
-describe command('sh -c "ulimit -v unlimited && mongo --eval \"db.smoke.findOne()\""') do
-  its('exit_status') { should eq 0 }
-  its('stdout') { should match(/.+"answer" : 42.+/m) }
-end
-
 ############################################################
 # This section verifies uninstall.
 ############################################################

buildscripts/package_test/ubuntu-user-data.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+echo "HostkeyAlgorithms +ssh-rsa" >> /etc/ssh/sshd_config
+echo "PubkeyAcceptedAlgorithms +ssh-rsa" >> /etc/ssh/sshd_config
+
+systemctl restart sshd
+
+touch /root/userdata_ran

buildscripts/packager.py

@@ -302,6 +302,8 @@ class Distro(object):
                 return "bionic"
             elif build_os == 'ubuntu2004':
                 return "focal"
+            elif build_os == 'ubuntu2204':
+                return "jammy"
             else:
                 raise Exception("unsupported build_os: %s" % build_os)
         elif self.dname == 'debian':
@@ -341,7 +343,17 @@ class Distro(object):
         if re.search("(suse)", self.dname):
             return ["suse11", "suse12", "suse15"]
         elif re.search("(redhat|fedora|centos)", self.dname):
-            return ["rhel82", "rhel80", "rhel70", "rhel71", "rhel72", "rhel62", "rhel55", "rhel67"]
+            return [
+                "rhel90",
+                "rhel82",
+                "rhel80",
+                "rhel70",
+                "rhel71",
+                "rhel72",
+                "rhel62",
+                "rhel55",
+                "rhel67",
+            ]
         elif self.dname in ['amazon', 'amazon2']:
             return [self.dname]
         elif self.dname == 'ubuntu':
@@ -351,6 +363,7 @@ class Distro(object):
                 "ubuntu1604",
                 "ubuntu1804",
                 "ubuntu2004",
+                "ubuntu2204",
             ]
         elif self.dname == 'debian':
             return ["debian81", "debian92", "debian10", "debian11"]

buildscripts/packager_enterprise.py

@@ -119,7 +119,7 @@ class EnterpriseDistro(packager.Distro):
     def build_os(self, arch):  # pylint: disable=too-many-branches
         """Return the build os label in the binary package to download.
 
-        The labels "rhel57", "rhel62", "rhel67", "rhel70", "rhel80" are for redhat,
+        The labels "rhel57", "rhel62", "rhel67", "rhel70", "rhel80", "rhel90" are for redhat,
         the others are delegated to the super class.
         """
         # pylint: disable=too-many-return-statements
@@ -139,7 +139,7 @@ class EnterpriseDistro(packager.Distro):
             return []
         if arch == "arm64":
             if self.dname == 'ubuntu':
-                return ["ubuntu1804", "ubuntu2004"]
+                return ["ubuntu1804", "ubuntu2004", "ubuntu2204"]
         if arch == "aarch64":
             if self.dname == 'redhat':
                 return ["rhel82"]
@@ -148,7 +148,7 @@ class EnterpriseDistro(packager.Distro):
             return []
 
         if re.search("(redhat|fedora|centos)", self.dname):
-            return ["rhel80", "rhel70", "rhel62", "rhel57"]
+            return ["rhel90", "rhel80", "rhel70", "rhel62", "rhel57"]
         return super(EnterpriseDistro, self).build_os(arch)
         # pylint: enable=too-many-return-statements
 

buildscripts/resmoke_tests_runtime_validate.py

@@ -3,7 +3,6 @@
 import json
 import sys
 from collections import namedtuple
-from datetime import datetime, timedelta
 from statistics import mean
 from typing import Dict, List
 
@@ -13,7 +12,8 @@ import structlog
 from buildscripts.resmokelib.testing.report import TestInfo, TestReport
 from buildscripts.resmokelib.utils import get_task_name_without_suffix
 from buildscripts.util.cmdutils import enable_logging
-from evergreen import RetryingEvergreenApi, TestStats
+
+from buildscripts.util.teststats import HistoricTaskData, HistoricalTestInformation
 
 LOGGER = structlog.get_logger("buildscripts.resmoke_tests_runtime_validate")
 
@@ -34,17 +34,12 @@ def parse_resmoke_report(report_file: str) -> List[TestInfo]:
     return [test_info for test_info in test_report.test_infos if "jstests" in test_info.test_file]
 
 
-def get_historic_stats(evg_api_config: str, project_id: str, test_files: List[str], task_name: str,
-                       build_variant: str) -> List[TestStats]:
+def get_historic_stats(project_id: str, task_name: str,
+                       build_variant: str) -> List[HistoricalTestInformation]:
     """Get historic test stats."""
-    evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
-    before_date = datetime.today()
-    after_date = before_date - timedelta(days=LOOK_BACK_NUM_DAYS)
     base_task_name = get_task_name_without_suffix(task_name, build_variant).replace(
         BURN_IN_PREFIX, "")
-    return evg_api.test_stats_by_project(project_id=project_id, after_date=after_date,
-                                         before_date=before_date, tests=test_files,
-                                         tasks=[base_task_name], variants=[build_variant])
+    return HistoricTaskData.get_stats_from_s3(project_id, base_task_name, build_variant)
 
 
 def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]:
@@ -63,13 +58,10 @@ def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]:
 @click.command()
 @click.option("--resmoke-report-file", type=str, required=True,
               help="Location of resmoke's report JSON file.")
-@click.option("--evg-api-config", type=str, required=True,
-              help="Location of evergreen api configuration.")
 @click.option("--project-id", type=str, required=True, help="Evergreen project id.")
 @click.option("--build-variant", type=str, required=True, help="Evergreen build variant name.")
 @click.option("--task-name", type=str, required=True, help="Evergreen task name.")
-def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_variant: str,
-         task_name: str) -> None:
+def main(resmoke_report_file: str, project_id: str, build_variant: str, task_name: str) -> None:
     """Compare resmoke tests runtime with historic stats."""
     enable_logging(verbose=False)
 
@@ -79,10 +71,9 @@ def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_v
         for test_info in current_test_infos
     ])
 
-    historic_stats = get_historic_stats(evg_api_config, project_id, list(current_stats_map.keys()),
-                                        task_name, build_variant)
+    historic_stats = get_historic_stats(project_id, task_name, build_variant)
     historic_stats_map = make_stats_map([
-        _TestData(test_stats.test_file, test_stats.avg_duration_pass)
+        _TestData(test_stats.test_name, test_stats.avg_duration_pass)
         for test_stats in historic_stats
     ])
 

buildscripts/resmokeconfig/fully_disabled_feature_flags.yml

@@ -8,3 +8,8 @@
 # Disable featureFlagRequireTenantID until all paths pass tenant id to TenantNamespace
 # and TenantDatabase constructors.
 - featureFlagRequireTenantID
+- featureFlagSbePlanCache
+# This flag exists to help users in managed environments that upgraded to 6.0 before 6.0.0-rc8 was
+# released create the transactions collection index and is only meant to be enabled adhoc, so only
+# its targeted tests should enable it.
+- featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp

buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml

@@ -53,6 +53,8 @@
 
         # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary.
         - jstests/core/geo_s2ordering.js
+        - jstests/fle2/**/*.js
+        - src/mongo/db/modules/*/jstests/fle2/**/*.js
 
 
 - name: reconfig_kill_primary_jscore_passthrough_exclude_files
@@ -121,6 +123,8 @@
 
         # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary.
         - jstests/core/geo_s2ordering.js
+        - jstests/fle2/**/*.js
+        - src/mongo/db/modules/*/jstests/fle2/**/*.js
 
 
 - name: kill_primary_jscore_passthrough_exclude_with_any_tags

buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml

@@ -190,16 +190,6 @@ evergreen_buildvariants:
     platform: suse15
     architecture: x86_64
 
-  - name: debian92
-    edition: targeted
-    platform: debian92
-    architecture: x86_64
-
-  - name: enterprise-debian92-64
-    edition: enterprise
-    platform: debian92
-    architecture: x86_64
-
   - name: debian10
     edition: targeted
     platform: debian10

buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests are run in causally_consistent_jscore_txns_passthrough.yml.
   - jstests/core/txns/**/*.js

buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml

@@ -12,6 +12,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   - jstests/core/txns/**/*.js
   # Skip any tests that run with auth explicitly.

buildscripts/resmokeconfig/suites/causally_consistent_read_concern_snapshot_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Only the first command in a transaction allows readConcern and this suite only tests
   # non-transaction snapshot reads.

buildscripts/resmokeconfig/suites/change_streams_downgrade.yml

@@ -4,6 +4,8 @@ test_kind: multi_stmt_txn_passthrough
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests run in the jscore_txn passthrough suites.
   - jstests/core/txns/**/*.js
@@ -147,7 +149,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/change_streams_per_shard_cursor_passthrough.yml

@@ -3,6 +3,9 @@ test_kind: js_test
 selector:
   roots:
   - jstests/change_streams/**/*.js
+  exclude_files:
+  # Expects a change stream cursor to be open on each of 2 shards.
+  - jstests/change_streams/projection_fakes_internal_event.js
   exclude_with_any_tags:
   ##
   # The next tags correspond to the special errors thrown by the

buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml

@@ -5,6 +5,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
 
   # Assumes the _id index is real.

buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml

@@ -31,6 +31,8 @@ selector:
   - uses_write_concern
   # Sharding is not supported for live exports and imports.
   - requires_sharding
+  # Workloads that kill random sessions may interrupt the export/import commands.
+  - kills_random_sessions
 
 executor:
   config:

buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml

@@ -23,6 +23,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled due to MongoDB restrictions and/or workload restrictions

buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml

@@ -23,6 +23,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled due to MongoDB restrictions and/or workload restrictions

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml

@@ -23,6 +23,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled due to MongoDB restrictions and/or workload restrictions

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml

@@ -28,6 +28,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled because MapReduce can lose cursors if the primary goes down during the operation.

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml

@@ -28,6 +28,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled because MapReduce can lose cursors if the primary goes down during the operation.

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml

@@ -23,6 +23,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled due to MongoDB restrictions and/or workload restrictions
@@ -129,6 +130,12 @@ selector:
   # for dbCheck. TODO (SERVER-63951): Remove this exclusion.
   - jstests/concurrency/fsm_workloads/create_collection_and_view.js
 
+  # This suite runs the RunDBCheckInBackground hook and the dbCheck command generates oplog entries
+  # but those oplog entries are not supported by resharding.
+  # TODO (SERVER-66011): Enable internal_transactions_resharding.js in the
+  # concurrency_sharded_multi_stmt_txn_with_balancer suite
+  - jstests/concurrency/fsm_workloads/internal_transactions_resharding.js
+
 
   exclude_with_any_tags:
   - assumes_balancer_off

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml

@@ -28,6 +28,7 @@ selector:
 
   # $lookup and $graphLookup are not supported on sharded collections with transactions.
   - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
+  - jstests/concurrency/fsm_workloads/agg_lookup.js
   - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js
 
   # Disabled due to MongoDB restrictions and/or workload restrictions

buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml

@@ -32,11 +32,6 @@ selector:
   # TODO: SERVER-39939.
   - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
 
-  # This workload kills random sessions and a different FSM workload wouldn't be able to handle
-  # the error response from the op being killed.
-  - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js
-  - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js
-
   # This workload may restart running transactions on a different client, causing deadlock if
   # there is a concurrent dropDatabase waiting for the global X lock.
   # TODO: SERVER-37876
@@ -54,8 +49,13 @@ selector:
   - jstests/concurrency/fsm_workloads/reindex_background.js
   - jstests/concurrency/fsm_workloads/reindex_writeconflict.js
 
+  # This workload involves upgrading and downgrading the FCV, and can cause failure on other
+  # FSM workloads which depend on features which are only available in the latest FCV.
+  - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js
+
   exclude_with_any_tags:
   - requires_sharding
+  - kills_random_sessions
 
   group_size: 10
   group_count_multiplier: 1.0

buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml

@@ -32,11 +32,6 @@ selector:
   # TODO: SERVER-39939.
   - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
 
-  # This workload kills random sessions and a different FSM workload wouldn't be able to handle
-  # the error response from the op being killed.
-  - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js
-  - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js
-
   # This workload may restart running transactions on a different client, causing deadlock if
   # there is a concurrent dropDatabase waiting for the global X lock.
   # TODO: SERVER-37876
@@ -59,8 +54,13 @@ selector:
   - jstests/concurrency/fsm_workloads/reindex_background.js
   - jstests/concurrency/fsm_workloads/reindex_writeconflict.js
 
+  # This workload involves upgrading and downgrading the FCV, and can cause failure on other
+  # FSM workloads which depend on features which are only available in the latest FCV.
+  - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js
+
   exclude_with_any_tags:
   - requires_sharding
+  - kills_random_sessions
 
   group_size: 10
   group_count_multiplier: 1.0

buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml

@@ -32,11 +32,6 @@ selector:
   # TODO: SERVER-39939.
   - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
 
-  # This workload kills random sessions and a different FSM workload wouldn't be able to handle
-  # the error response from the op being killed.
-  - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js
-  - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js
-
   # This workload may restart running transactions on a different client, causing deadlock if
   # there is a concurrent dropDatabase waiting for the global X lock.
   # TODO: SERVER-37876
@@ -60,8 +55,13 @@ selector:
   - jstests/concurrency/fsm_workloads/reindex.js
   - jstests/concurrency/fsm_workloads/reindex_background.js
 
+  # This workload involves upgrading and downgrading the FCV, and can cause failure on other
+  # FSM workloads which depend on features which are only available in the latest FCV.
+  - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js
+
   exclude_with_any_tags:
   - requires_sharding
+  - kills_random_sessions
 
   group_size: 10
   group_count_multiplier: 1.0

buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml

@@ -6,6 +6,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   - jstests/core_standalone/**/*.js
   exclude_files:
   # Transactions are not supported on MongoDB standalone nodes, so we do not run these tests in the
@@ -254,9 +256,6 @@ selector:
   - jstests/core/index_check6.js
   - jstests/core/index_check7.js
   - jstests/core/index_decimal.js
-  - jstests/core/index_elemmatch1.js
-  - jstests/core/index_elemmatch2.js
-  - jstests/core/index_elemmatch2.js
   - jstests/core/index_filter_commands.js
   - jstests/core/index_filter_on_hidden_index.js
   - jstests/core/index_multiple_compatibility.js
@@ -506,8 +505,6 @@ selector:
   - jstests/core/in7.js
   - jstests/core/index13.js
   - jstests/core/index_check2.js
-  - jstests/core/index_elemmatch1.js
-  - jstests/core/index_elemmatch2.js
   - jstests/core/indexl.js
   - jstests/core/json_schema/misc_validation.js
   - jstests/core/ne_array.js
@@ -756,7 +753,6 @@ selector:
   - jstests/core/geonear_key.js
   - jstests/core/getmore_invalidated_documents.js
   - jstests/core/hidden_index.js
-  - jstests/core/index_elemmatch2.js
   - jstests/core/index_partial_2dsphere.js
   - jstests/core/json_schema/misc_validation.js
   - jstests/core/list_collections_filter.js

buildscripts/resmokeconfig/suites/cwrwc_rc_majority_passthrough.yml

@@ -6,6 +6,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Transactions only a readConcern argument on the first command.
   - jstests/core/txns/**/*.js

buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml

@@ -0,0 +1,32 @@
+test_kind: js_test
+selector:
+  roots:
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
+  - src/mongo/db/modules/*/jstests/fle2/query/*.js
+  exclude_with_any_tags:
+  # Not compatible with tests the expect fle to always using $in in queries,
+  # i.e. verify explain output
+  - requires_fle_in_always
+
+executor:
+  archive:
+    hooks:
+      - ValidateCollections
+  config:
+    shell_options:
+      eval: "testingReplication = true; testingFLESharding = false;"
+  hooks:
+  # We don't execute dbHash or oplog consistency checks since there is only a single replica set
+  # node.
+  - class: ValidateCollections
+  - class: CleanEveryN
+    n: 20
+  fixture:
+    class: ReplicaSetFixture
+    mongod_options:
+      set_parameters:
+        enableTestCommands: 1
+        internalQueryFLEAlwaysUseHighCardinalityMode: 1
+    # Use a 2-node replica set.
+    num_nodes: 2

buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml

@@ -0,0 +1,37 @@
+test_kind: js_test
+selector:
+  roots:
+  - jstests/fle2/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
+  - src/mongo/db/modules/*/jstests/fle2/query/*.js
+  exclude_with_any_tags:
+  # Not compatible with tests the expect fle to always using $in in queries,
+  # i.e. verify explain output
+  - requires_fle_in_always
+
+executor:
+  archive:
+    hooks:
+      - CheckReplDBHash
+      - ValidateCollections
+  config:
+    shell_options:
+      eval: "testingReplication = false; testingFLESharding = true;"
+  hooks:
+  - class: CheckReplDBHash
+  - class: ValidateCollections
+  - class: CleanEveryN
+    n: 20
+  fixture:
+    class: ShardedClusterFixture
+    mongos_options:
+      set_parameters:
+        enableTestCommands: 1
+        internalQueryFLEAlwaysUseHighCardinalityMode: 1
+    mongod_options:
+      set_parameters:
+        enableTestCommands: 1
+        internalQueryFLEAlwaysUseHighCardinalityMode: 1
+    num_rs_nodes_per_shard: 2
+    enable_sharding:
+    - test

buildscripts/resmokeconfig/suites/jstestfuzz.yml

@@ -20,7 +20,6 @@ executor:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
           skipValidationOnNamespaceNotFound: false
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: MongoDFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_interrupt.yml

@@ -24,7 +24,6 @@ executor:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
           skipValidationOnNamespaceNotFound: false
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: MongoDFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_interrupt_replication.yml

@@ -34,7 +34,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ReplicaSetFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml

@@ -34,7 +34,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ReplicaSetFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_replication_continuous_stepdown.yml

@@ -32,7 +32,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ReplicaSetFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_replication_session.yml

@@ -31,7 +31,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ReplicaSetFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_replication_write_conflicts.yml

@@ -26,7 +26,6 @@ executor:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
           skipValidationOnNamespaceNotFound: false
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ReplicaSetFixture
     mongod_options:

buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml

@@ -29,7 +29,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ShardedClusterFixture
     mongos_options:

buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml

@@ -29,7 +29,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ShardedClusterFixture
     mongos_options:

buildscripts/resmokeconfig/suites/jstestfuzz_sharded_continuous_stepdown.yml

@@ -35,7 +35,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ShardedClusterFixture
     mongos_options:

buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml

@@ -26,7 +26,6 @@ executor:
       global_vars:
         TestData:
           skipValidationOnInvalidViewDefinitions: true
-          forceValidationWithFeatureCompatibilityVersion: latestFCV
   fixture:
     class: ShardedClusterFixture
     mongos_options:

buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # The set_param1.js test attempts to compare the response from running the {getParameter: "*"}
   # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds"
@@ -12,7 +14,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal
   # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same

buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # The set_param1.js test attempts to compare the response from running the {getParameter: "*"}
   # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds"
@@ -12,7 +14,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal
   # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same

buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # The set_param1.js test attempts to compare the response from running the {getParameter: "*"}
   # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds"
@@ -12,7 +14,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal
   # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same

buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # The set_param1.js test attempts to compare the response from running the {getParameter: "*"}
   # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds"
@@ -12,7 +14,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal
   # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same

buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml.
   - jstests/core/txns/**/*.js
@@ -55,7 +57,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded
   # queries with a limit or for distinct commands.

buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml.
   - jstests/core/txns/**/*.js
@@ -55,7 +57,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded
   # queries with a limit or for distinct commands.

buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml.
   - jstests/core/txns/**/*.js
@@ -55,7 +57,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded
   # queries with a limit or for distinct commands.

buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml.
   - jstests/core/txns/**/*.js
@@ -55,7 +57,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded
   # queries with a limit or for distinct commands.

buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml

@@ -13,7 +13,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # These tests verify that an expected number of update operations were tracked in the server
   # status metrics, but the logical session cache refresh causes additional updates to be recorded.

buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml

@@ -13,7 +13,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # These tests verify that an expected number of update operations were tracked in the server
   # status metrics, but the logical session cache refresh causes additional updates to be recorded.

buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml

@@ -13,7 +13,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # These tests verify that an expected number of update operations were tracked in the server
   # status metrics, but the logical session cache refresh causes additional updates to be recorded.

buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml

@@ -13,7 +13,6 @@ selector:
   # can be triggered deterministically.
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
   # These tests verify that an expected number of update operations were tracked in the server
   # status metrics, but the logical session cache refresh causes additional updates to be recorded.

buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml

@@ -151,7 +151,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml

@@ -146,7 +146,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml

@@ -147,7 +147,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml

@@ -159,7 +159,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Transactions only a readConcern argument on the first command.
   - jstests/core/txns/**/*.js

buildscripts/resmokeconfig/suites/replica_sets_api_version_jscore_passthrough.yml

@@ -5,6 +5,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # This test initiates a txn with API parameters, but runs txn-continuing commands in a separate
   # Thread object. Since those commands will not have API parameters appended, they will fail due

buildscripts/resmokeconfig/suites/replica_sets_fcbis_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Tests that query the system.profile collection cannot run in this suite since an initial sync
   # may insert unexpected operations into the profile collection.

buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Tests that query the system.profile collection cannot run in this suite since an initial sync
   # may insert unexpected operations into the profile collection.

buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Duplicate namespace may cause cloner to try and clone view instead of collection
   - jstests/core/views/duplicate_ns.js

buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests change the transactionLifetimeLimitSeconds server parameter which conflicts with how
   # the CheckReplDBHashInBackground hook doesn't want transactions to be reaped while it is running.

buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml

@@ -96,7 +96,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $indexStats

buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml

@@ -87,7 +87,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml

@@ -86,7 +86,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml

@@ -84,7 +84,6 @@ selector:
   # $listSessions
   - jstests/core/list_all_local_sessions.js
   - jstests/core/list_all_sessions.js
-  - jstests/core/list_local_sessions.js
   - jstests/core/list_sessions.js
 
   # $collStats

buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_passthrough.yml

@@ -7,6 +7,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # Transactions do not support retryability of individual operations.
   # TODO: Remove this once it is supported (SERVER-33952).

buildscripts/resmokeconfig/suites/replica_sets_update_v1_oplog_jscore_passthrough.yml

@@ -3,6 +3,8 @@ test_kind: js_test
 selector:
   roots:
   - jstests/core/**/*.js
+  - jstests/fle2/**/*.js
+  - src/mongo/db/modules/*/jstests/fle2/*.js
   exclude_files:
   # These tests change the transactionLifetimeLimitSeconds server parameter which conflicts with how
   # the CheckReplDBHashInBackground hook doesn't want transactions to be reaped while it is running.