diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index 79ad753bfed..21db590f50e 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -18,6 +18,7 @@ selector: exclude_files: - jstests/noPassthrough/libs/*.js exclude_with_any_tags: + - requires_kernel_619 # noPassthrough tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/rseq_kernel_check.yml b/buildscripts/resmokeconfig/suites/rseq_kernel_check.yml new file mode 100644 index 00000000000..ec0a8522a14 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/rseq_kernel_check.yml @@ -0,0 +1,20 @@ +test_kind: js_test +description: | + Tests for rseq/tcmalloc kernel compatibility on Linux >= 6.19. + Only runs on the ubuntu2404-kernel619-rseq-check variant. + +selector: + roots: + - jstests/noPassthrough/rseq_linux_compatibility/**/*.js + include_with_any_tags: + - requires_kernel_619 + +executor: + config: + shell_options: + nodb: "" + process_kwargs: + env_vars: + # We need to set the environment variable at the suite level because + # mongo shell would also crash due to the tcmalloc issue otherwise. + GLIBC_TUNABLES: "glibc.pthread.rseq=1" diff --git a/jstests/noPassthrough/rseq_linux_compatibility/BUILD.bazel b/jstests/noPassthrough/rseq_linux_compatibility/BUILD.bazel new file mode 100644 index 00000000000..e27c62dadcc --- /dev/null +++ b/jstests/noPassthrough/rseq_linux_compatibility/BUILD.bazel @@ -0,0 +1,13 @@ +load("@aspect_rules_js//js:defs.bzl", "js_library") + +js_library( + name = "all_javascript_files", + srcs = glob([ + "*.js", + ]), + target_compatible_with = select({ + "//bazel/config:ppc_or_s390x": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) diff --git a/jstests/noPassthrough/rseq_linux_compatibility/OWNERS.yml b/jstests/noPassthrough/rseq_linux_compatibility/OWNERS.yml new file mode 100644 index 00000000000..79117d50c1a --- /dev/null +++ b/jstests/noPassthrough/rseq_linux_compatibility/OWNERS.yml @@ -0,0 +1,5 @@ +version: 2.0.0 +filters: + - "*": + approvers: + - 10gen/server-workload-resilience diff --git a/jstests/noPassthrough/rseq_linux_compatibility/rseq_kernel_compatibility_check.js b/jstests/noPassthrough/rseq_linux_compatibility/rseq_kernel_compatibility_check.js new file mode 100644 index 00000000000..df86ef1cb49 --- /dev/null +++ b/jstests/noPassthrough/rseq_linux_compatibility/rseq_kernel_compatibility_check.js @@ -0,0 +1,87 @@ +/** + * This test checks if mongod correctly crash on startup on linux 6.19 with + * tcmalloc per-CPU cache. + * + * @tags: [requires_kernel_619] + */ + +const gracefulExitLogID = 12257600; +const findGracefulExitLogLine = new RegExp(`"id":${gracefulExitLogID}`); + +function checkGracefulExitOnIncompatibleEnv(conn) { + const exitCode = waitProgram(conn.pid); + assert.eq(exitCode, 1, `Expected server to exit with code 1, got ${exitCode}`); + assert( + rawMongoProgramOutput(".*").search(findGracefulExitLogLine) >= 0, + `Expected fatal log message with ID ${gracefulExitLogID} in server output`, + ); +} + +function checkGracefulExitOnCompatibleEnv(conn) { + const exitCode = waitProgram(conn.pid); + assert.eq(exitCode, 0, `Expected server to exit with code 0, got ${exitCode}`); + assert( + rawMongoProgramOutput(".*").search(findGracefulExitLogLine) === -1, + `Unexpected fatal log message with ID ${gracefulExitLogID} in server output`, + ); +} + +function testMongodPerCPUCacheEnabled() { + clearRawMongoProgramOutput(); + const conn = MongoRunner.runMongod({ + env: {GLIBC_TUNABLES: "glibc.pthread.rseq=0"}, + waitForConnect: false, + setParameter: { + "failpoint.shutdownAtStartup": '{mode:"alwaysOn"}', + }, + }); + checkGracefulExitOnIncompatibleEnv(conn); +} + +function testMongodPerCPUCacheDisabled() { + clearRawMongoProgramOutput(); + const conn = MongoRunner.runMongod({ + env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"}, + waitForConnect: false, + setParameter: { + "failpoint.shutdownAtStartup": '{mode:"alwaysOn"}', + }, + }); + checkGracefulExitOnCompatibleEnv(conn); +} + +function testMongosPerCPUCacheEnabled() { + const configRS = new ReplSetTest({nodes: 1}); + configRS.startSet({configsvr: "", env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"}}); + configRS.initiate(); + clearRawMongoProgramOutput(); + const conn = MongoRunner.runMongos({ + configdb: configRS.getURL(), + env: {GLIBC_TUNABLES: "glibc.pthread.rseq=0"}, + waitForConnect: false, + setParameter: { + "failpoint.shutdownAtStartup": '{mode:"alwaysOn"}', + }, + }); + checkGracefulExitOnIncompatibleEnv(conn); + configRS.stopSet(); +} + +function testMongosPerCPUCacheDisabled() { + const configRS = new ReplSetTest({nodes: 1}); + configRS.startSet({configsvr: "", env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"}}); + configRS.initiate(); + clearRawMongoProgramOutput(); + const conn = MongoRunner.runMongos({ + configdb: configRS.getURL(), + env: {GLIBC_TUNABLES: "glibc.pthread.rseq=1"}, + }); + conn.getDB('admin').shutdownServer(); + checkGracefulExitOnCompatibleEnv(conn); + configRS.stopSet(); +} + +testMongodPerCPUCacheEnabled(); +testMongodPerCPUCacheDisabled(); +testMongosPerCPUCacheEnabled(); +testMongosPerCPUCacheDisabled(); diff --git a/src/mongo/db/BUILD.bazel b/src/mongo/db/BUILD.bazel index edc6ef2cb6c..7aaf64dce85 100644 --- a/src/mongo/db/BUILD.bazel +++ b/src/mongo/db/BUILD.bazel @@ -203,6 +203,24 @@ mongo_cc_library( ], ) +mongo_cc_library( + name = "startup_check_rseq", + srcs = ["startup_check_rseq.cpp"], + hdrs = ["startup_check_rseq.h"], + deps = [ + "//src/mongo:base", + ], +) + +mongo_cc_unit_test( + name = "startup_check_rseq_test", + srcs = ["startup_check_rseq_test.cpp"], + tags = ["mongo_unittest_seventh_group"], + deps = [ + "startup_check_rseq", + ], +) + mongo_cc_library( name = "profile_filter", srcs = [ @@ -2664,6 +2682,7 @@ mongo_cc_library( # satisfy symbol dependencies from the files listed above in `sources`. If you need to add a # library to inject a static or mongo initializer to mongod, please add that library as a # private libdep of mongod_initializers. + "startup_check_rseq", "//src/mongo/client:clientdriver_minimal", "//src/mongo/db/auth:user_cache_invalidator", "//src/mongo/idl:cluster_server_parameter_idl", diff --git a/src/mongo/db/mongod_main.cpp b/src/mongo/db/mongod_main.cpp index 2780da362d1..97179bf1a41 100644 --- a/src/mongo/db/mongod_main.cpp +++ b/src/mongo/db/mongod_main.cpp @@ -211,6 +211,7 @@ #include "mongo/db/session/session_killer.h" #include "mongo/db/session_manager_mongod.h" #include "mongo/db/set_change_stream_state_coordinator.h" +#include "mongo/db/startup_check_rseq.h" #include "mongo/db/startup_recovery.h" #include "mongo/db/startup_warnings_mongod.h" #include "mongo/db/storage/backup_cursor_hooks.h" @@ -2157,6 +2158,8 @@ int mongod_main(int argc, char* argv[]) { setupSignalHandlers(); + validateRseqKernelCompat(); + srand(static_cast(curTimeMicros64())); // NOLINT Status status = mongo::runGlobalInitializers(std::vector(argv, argv + argc)); diff --git a/src/mongo/db/startup_check_rseq.cpp b/src/mongo/db/startup_check_rseq.cpp new file mode 100644 index 00000000000..0f77b4a466d --- /dev/null +++ b/src/mongo/db/startup_check_rseq.cpp @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2026-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/startup_check_rseq.h" + +#include "mongo/base/parse_number.h" +#include "mongo/base/string_data.h" +#include "mongo/config.h" +#include "mongo/logv2/log.h" +#include "mongo/util/exit_code.h" +#include "mongo/util/quick_exit.h" + +#include + +#ifdef MONGO_CONFIG_TCMALLOC_GOOGLE +#include +#endif + +#ifdef __linux__ +#include +#endif + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kControl + +namespace mongo { + +bool isKernelVersionSafeForTCMallocPerCPUCache(StringData release) { + int major = 0, minor = 0; + char* end = nullptr; + if (!NumberParser::strToAny(10)(release, &major, &end).isOK() || *end != '.' || + !NumberParser::strToAny(10)(end + 1, &minor).isOK()) { + // If the version cannot be parsed, assume the kernel is compatible + LOGV2_WARNING(12257601, + "Unable to parse kernel version, cannot check for kernel " + "version compatibility", + "kernel-version"_attr = release); + return true; + } + return major < 6 || (major == 6 && minor < 19); +} + +namespace { + +bool isKernelSafeForTCMallocPerCPUCache() { +#ifdef __linux__ + struct utsname u; + if (uname(&u) != 0) { + LOGV2_WARNING(12257602, + "Unable to determine kernel version via uname, cannot check for kernel " + "version compatibility"); + return true; + } + StringData release{u.release}; + if (!isKernelVersionSafeForTCMallocPerCPUCache(release)) { + return false; + } +#endif + return true; +} + +bool isTCMallocPerCPUCacheActive() { +#ifdef MONGO_CONFIG_TCMALLOC_GOOGLE + return tcmalloc::MallocExtension::PerCpuCachesActive(); +#else + return false; +#endif +} + +} // namespace + +void validateRseqKernelCompat() { + if (isTCMallocPerCPUCacheActive() && !isKernelSafeForTCMallocPerCPUCache()) { + LOGV2_FATAL_OPTIONS( + 12257600, + logv2::LogOptions(logv2::LogComponent::kControl, logv2::FatalMode::kContinue), + "MongoDB cannot start: Linux kernel versions 6.19 and newer has a known " + "incompatibility with this version of MongoDB. See " + "https://jira.mongodb.org/browse/SERVER-121912 for more information."); + quickExit(ExitCode::fail); + } +} + +} // namespace mongo diff --git a/src/mongo/db/startup_check_rseq.h b/src/mongo/db/startup_check_rseq.h new file mode 100644 index 00000000000..84a5668142f --- /dev/null +++ b/src/mongo/db/startup_check_rseq.h @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2026-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/string_data.h" + +namespace mongo { + +void validateRseqKernelCompat(); +bool isKernelVersionSafeForTCMallocPerCPUCache(StringData release); + +} // namespace mongo diff --git a/src/mongo/db/startup_check_rseq_test.cpp b/src/mongo/db/startup_check_rseq_test.cpp new file mode 100644 index 00000000000..b72d78b6f44 --- /dev/null +++ b/src/mongo/db/startup_check_rseq_test.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2026-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/startup_check_rseq.h" + +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +TEST(StartupCheckRseq, SafeKernels) { + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("5.15.0-generic")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.18.99")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.0.0")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("4.19.0-aws")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("5.4.0")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.18.0")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.18")); +} + +TEST(StartupCheckRseq, UnsafeKernels) { + ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.19")); + ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.19.0")); + ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.19.3-generic")); + ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("6.20.0")); + ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("7.0.0")); + ASSERT_FALSE(isKernelVersionSafeForTCMallocPerCPUCache("10.0.0")); +} + +TEST(StartupCheckRseq, UnparseableReturnsTrue) { + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("invalid")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("abc.def")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache("6.")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache(".19")); + ASSERT_TRUE(isKernelVersionSafeForTCMallocPerCPUCache(".6.19.0")); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/s/BUILD.bazel b/src/mongo/s/BUILD.bazel index 89ee55aa776..15435eaacc9 100644 --- a/src/mongo/s/BUILD.bazel +++ b/src/mongo/s/BUILD.bazel @@ -705,6 +705,7 @@ mongo_cc_library( # please add that library as a private libdep of # mongos_initializers. "mongos_options_idl", + "//src/mongo/db:startup_check_rseq", "//src/mongo/client:remote_command_targeter", "//src/mongo/db/admission:queues_server_status_section", "//src/mongo/db:audit", diff --git a/src/mongo/s/mongos_main.cpp b/src/mongo/s/mongos_main.cpp index 15a82c2af8f..ff21c300eb7 100644 --- a/src/mongo/s/mongos_main.cpp +++ b/src/mongo/s/mongos_main.cpp @@ -93,6 +93,7 @@ #include "mongo/db/session/session_catalog.h" #include "mongo/db/session/session_killer.h" #include "mongo/db/shard_id.h" +#include "mongo/db/startup_check_rseq.h" #include "mongo/db/startup_warnings_common.h" #include "mongo/db/wire_version.h" #include "mongo/executor/task_executor.h" @@ -1090,6 +1091,8 @@ ExitCode mongos_main(int argc, char* argv[]) { setupSignalHandlers(); + validateRseqKernelCompat(); + Status status = runGlobalInitializers(std::vector(argv, argv + argc)); if (!status.isOK()) { LOGV2_FATAL_OPTIONS(