SERVER-122576 Make kernel 6.19 crash more graceful than status quo (#51045)

GitOrigin-RevId: e67138ffda9f90b59531128bc43678bb5ad105ed

buildscripts/resmokeconfig/suites/no_passthrough.yml

@@ -18,6 +18,7 @@ selector:
   exclude_files:
   - jstests/noPassthrough/libs/*.js
   exclude_with_any_tags:
+    - requires_kernel_619
 
 # noPassthrough tests start their own mongod's.
 executor:

buildscripts/resmokeconfig/suites/rseq_kernel_check.yml

@@ -0,0 +1,20 @@
+test_kind: js_test
+description: |
+  Tests for rseq/tcmalloc kernel compatibility on Linux >= 6.19.
+  Only runs on the ubuntu2404-kernel619-rseq-check variant.
+
+selector:
+  roots:
+    - jstests/noPassthrough/rseq_linux_compatibility/**/*.js
+  include_with_any_tags:
+    - requires_kernel_619
+
+executor:
+  config:
+    shell_options:
+      nodb: ""
+      process_kwargs:
+        env_vars:
+          # We need to set the environment variable at the suite level because
+          # mongo shell would also crash due to the tcmalloc issue otherwise.
+          GLIBC_TUNABLES: "glibc.pthread.rseq=1"

jstests/noPassthrough/rseq_linux_compatibility/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@aspect_rules_js//js:defs.bzl", "js_library")
+
+js_library(
+    name = "all_javascript_files",
+    srcs = glob([
+        "*.js",
+    ]),
+    target_compatible_with = select({
+        "//bazel/config:ppc_or_s390x": ["@platforms//:incompatible"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+)

jstests/noPassthrough/rseq_linux_compatibility/OWNERS.yml

@@ -0,0 +1,5 @@
+version: 2.0.0
+filters:
+  - "*":
+    approvers:
+      - 10gen/server-workload-resilience

jstests/noPassthrough/rseq_linux_compatibility/rseq_kernel_compatibility_check.js

@@ -0,0 +1,87 @@
+/**
+ * This test checks if mongod correctly crash on startup on linux 6.19 with
+ * tcmalloc per-CPU cache.
+ *
+ * @tags: [requires_kernel_619]
+ */
+
+const gracefulExitLogID = 12257600;
+const findGracefulExitLogLine = new RegExp(`"id":${gracefulExitLogID}`);
+
+function checkGracefulExitOnIncompatibleEnv(conn) {
+    const exitCode = waitProgram(conn.pid);
+    assert.eq(exitCode, 1, `Expected server to exit with code 1, got ${exitCode}`);
+    assert(
+        rawMongoProgramOutput(".*").search(findGracefulExitLogLine) >= 0,
+        `Expected fatal log message with ID ${gracefulExitLogID} in server output`,
+    );
+}
+
+function checkGracefulExitOnCompatibleEnv(conn) {
+    const exitCode = waitProgram(conn.pid);
+    assert.eq(exitCode, 0, `Expected server to exit with code 0, got ${exitCode}`);
+    assert(
+        rawMongoProgramOutput(".*").search(findGracefulExitLogLine) === -1,
+        `Unexpected fatal log message with ID ${gracefulExitLogID} in server output`,
+    );
+}
+
+function testMongodPerCPUCacheEnabled() {
+    clearRawMongoProgramOutput();
+    const conn = MongoRunner.runMongod({
+        env: {GLIBC_TUNABLES: "glibc.pthread.rseq=0"},
+        waitForConnect: false,
+        setParameter: {
+            "failpoint.shutdownAtStartup": '{mode:"alwaysOn"}',
+        },
+    });
+    checkGracefulExitOnIncompatibleEnv(conn);
+}
+
+function testMongodPerCPUCacheDisabled() {
+    clearRawMongoProgramOutput();
+    const conn = MongoRunner.runMongod({
+        env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"},
+        waitForConnect: false,
+        setParameter: {
+            "failpoint.shutdownAtStartup": '{mode:"alwaysOn"}',
+        },
+    });
+    checkGracefulExitOnCompatibleEnv(conn);
+}
+
+function testMongosPerCPUCacheEnabled() {
+    const configRS = new ReplSetTest({nodes: 1});
+    configRS.startSet({configsvr: "", env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"}});
+    configRS.initiate();
+    clearRawMongoProgramOutput();
+    const conn = MongoRunner.runMongos({
+        configdb: configRS.getURL(),
+        env: {GLIBC_TUNABLES: "glibc.pthread.rseq=0"},
+        waitForConnect: false,
+        setParameter: {
+            "failpoint.shutdownAtStartup": '{mode:"alwaysOn"}',
+        },
+    });
+    checkGracefulExitOnIncompatibleEnv(conn);
+    configRS.stopSet();
+}
+
+function testMongosPerCPUCacheDisabled() {
+    const configRS = new ReplSetTest({nodes: 1});
+    configRS.startSet({configsvr: "", env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"}});
+    configRS.initiate();
+    clearRawMongoProgramOutput();
+    const conn = MongoRunner.runMongos({
+        configdb: configRS.getURL(),
+        env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"},
+    });
+    conn.getDB('admin').shutdownServer();
+    checkGracefulExitOnCompatibleEnv(conn);
+    configRS.stopSet();
+}
+
+testMongodPerCPUCacheEnabled();
+testMongodPerCPUCacheDisabled();
+testMongosPerCPUCacheEnabled();
+testMongosPerCPUCacheDisabled();

src/mongo/db/BUILD.bazel

@@ -203,6 +203,24 @@ mongo_cc_library(
     ],
 )
 
+mongo_cc_library(
+    name = "startup_check_rseq",
+    srcs = ["startup_check_rseq.cpp"],
+    hdrs = ["startup_check_rseq.h"],
+    deps = [
+        "//src/mongo:base",
+    ],
+)
+
+mongo_cc_unit_test(
+    name = "startup_check_rseq_test",
+    srcs = ["startup_check_rseq_test.cpp"],
+    tags = ["mongo_unittest_seventh_group"],
+    deps = [
+        "startup_check_rseq",
+    ],
+)
+
 mongo_cc_library(
     name = "profile_filter",
     srcs = [
@@ -2664,6 +2682,7 @@ mongo_cc_library(
         # satisfy symbol dependencies from the files listed above in `sources`. If you need to add a
         # library to inject a static or mongo initializer to mongod, please add that library as a
         # private libdep of mongod_initializers.
+        "startup_check_rseq",
         "//src/mongo/client:clientdriver_minimal",
         "//src/mongo/db/auth:user_cache_invalidator",
         "//src/mongo/idl:cluster_server_parameter_idl",

src/mongo/db/mongod_main.cpp

@@ -211,6 +211,7 @@
 #include "mongo/db/session/session_killer.h"
 #include "mongo/db/session_manager_mongod.h"
 #include "mongo/db/set_change_stream_state_coordinator.h"
+#include "mongo/db/startup_check_rseq.h"
 #include "mongo/db/startup_recovery.h"
 #include "mongo/db/startup_warnings_mongod.h"
 #include "mongo/db/storage/backup_cursor_hooks.h"
@@ -2157,6 +2158,8 @@ int mongod_main(int argc, char* argv[]) {
 
     setupSignalHandlers();
 
+    validateRseqKernelCompat();
+
     srand(static_cast<unsigned>(curTimeMicros64()));  // NOLINT
 
     Status status = mongo::runGlobalInitializers(std::vector<std::string>(argv, argv + argc));

src/mongo/db/startup_check_rseq.cpp

@@ -0,0 +1,109 @@
+/**
+ *    Copyright (C) 2026-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/startup_check_rseq.h"
+
+#include "mongo/base/parse_number.h"
+#include "mongo/base/string_data.h"
+#include "mongo/config.h"
+#include "mongo/logv2/log.h"
+#include "mongo/util/exit_code.h"
+#include "mongo/util/quick_exit.h"
+
+#include <boost/optional.hpp>
+
+#ifdef MONGO_CONFIG_TCMALLOC_GOOGLE
+#include <tcmalloc/malloc_extension.h>
+#endif
+
+#ifdef __linux__
+#include <sys/utsname.h>
+#endif
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kControl
+
+namespace mongo {
+
+bool isKernelVersionSafeForTCMallocPerCPUCache(StringData release) {
+    int major = 0, minor = 0;
+    char* end = nullptr;
+    if (!NumberParser::strToAny(10)(release, &major, &end).isOK() || *end != '.' ||
+        !NumberParser::strToAny(10)(end + 1, &minor).isOK()) {
+        // If the version cannot be parsed, assume the kernel is compatible
+        LOGV2_WARNING(12257601,
+                      "Unable to parse kernel version, cannot check for kernel "
+                      "version compatibility",
+                      "kernel-version"_attr = release);
+        return true;
+    }
+    return major < 6 || (major == 6 && minor < 19);
+}
+
+namespace {
+
+bool isKernelSafeForTCMallocPerCPUCache() {
+#ifdef __linux__
+    struct utsname u;
+    if (uname(&u) != 0) {
+        LOGV2_WARNING(12257602,
+                      "Unable to determine kernel version via uname, cannot check for kernel "
+                      "version compatibility");
+        return true;
+    }
+    StringData release{u.release};
+    if (!isKernelVersionSafeForTCMallocPerCPUCache(release)) {
+        return false;
+    }
+#endif
+    return true;
+}
+
+bool isTCMallocPerCPUCacheActive() {
+#ifdef MONGO_CONFIG_TCMALLOC_GOOGLE
+    return tcmalloc::MallocExtension::PerCpuCachesActive();
+#else
+    return false;
+#endif
+}
+
+}  // namespace
+
+void validateRseqKernelCompat() {
+    if (isTCMallocPerCPUCacheActive() && !isKernelSafeForTCMallocPerCPUCache()) {
+        LOGV2_FATAL_OPTIONS(
+            12257600,
+            logv2::LogOptions(logv2::LogComponent::kControl, logv2::FatalMode::kContinue),
+            "MongoDB cannot start: Linux kernel versions 6.19 and newer has a known "
+            "incompatibility with this version of MongoDB. See "
+            "https://jira.mongodb.org/browse/SERVER-121912 for more information.");
+        quickExit(ExitCode::fail);
+    }
+}
+
+}  // namespace mongo

src/mongo/db/startup_check_rseq.h

@@ -0,0 +1,39 @@
+/**
+ *    Copyright (C) 2026-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/string_data.h"
+
+namespace mongo {
+
+void validateRseqKernelCompat();
+bool isKernelVersionSafeForTCMallocPerCPUCache(StringData release);
+
+}  // namespace mongo

src/mongo/db/startup_check_rseq_test.cpp

@@ -0,0 +1,67 @@
+/**
+ *    Copyright (C) 2026-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/startup_check_rseq.h"
+
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+namespace {
+
+TEST(StartupCheckRseq, SafeKernels) {
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("5.15.0-generic"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.18.99"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.0.0"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("4.19.0-aws"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("5.4.0"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.18.0"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.18"));
+}
+
+TEST(StartupCheckRseq, UnsafeKernels) {
+    ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.19"));
+    ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.19.0"));
+    ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.19.3-generic"));
+    ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.20.0"));
+    ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("7.0.0"));
+    ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("10.0.0"));
+}
+
+TEST(StartupCheckRseq, UnparseableReturnsTrue) {
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache(""));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("invalid"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("abc.def"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6."));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache(".19"));
+    ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache(".6.19.0"));
+}
+
+}  // namespace
+}  // namespace mongo

src/mongo/s/BUILD.bazel

@@ -705,6 +705,7 @@ mongo_cc_library(
         # please add that library as a private libdep of
         # mongos_initializers.
         "mongos_options_idl",
+        "//src/mongo/db:startup_check_rseq",
         "//src/mongo/client:remote_command_targeter",
         "//src/mongo/db/admission:queues_server_status_section",
         "//src/mongo/db:audit",

src/mongo/s/mongos_main.cpp

@@ -93,6 +93,7 @@
 #include "mongo/db/session/session_catalog.h"
 #include "mongo/db/session/session_killer.h"
 #include "mongo/db/shard_id.h"
+#include "mongo/db/startup_check_rseq.h"
 #include "mongo/db/startup_warnings_common.h"
 #include "mongo/db/wire_version.h"
 #include "mongo/executor/task_executor.h"
@@ -1090,6 +1091,8 @@ ExitCode mongos_main(int argc, char* argv[]) {
 
     setupSignalHandlers();
 
+    validateRseqKernelCompat();
+
     Status status = runGlobalInitializers(std::vector<std::string>(argv, argv + argc));
     if (!status.isOK()) {
         LOGV2_FATAL_OPTIONS(