SERVER-102306 Skip WT cache eviction when generating FTDC (#43062)

Co-authored-by: Erin McNulty <erin.mcnulty@mongodb.com>
GitOrigin-RevId: c4429c0fa50a8fda2c3194b22436b7d7fd3e33a1

src/mongo/db/ftdc/ftdc_mongod.cpp

@@ -42,6 +42,7 @@
 #include "mongo/db/ftdc/ftdc_mongod_gen.h"
 #include "mongo/db/ftdc/ftdc_mongos.h"
 #include "mongo/db/ftdc/ftdc_server.h"
+#include "mongo/db/local_catalog/shard_role_api/transaction_resources.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/repl/replication_coordinator.h"
@@ -90,6 +91,15 @@ public:
     void collect(OperationContext* opCtx, BSONObjBuilder& builder) override {
         std::vector<std::string> namespaces = gDiagnosticDataCollectionStatsNamespaces.get();
 
+        auto ru = shard_role_details::getRecoveryUnit(opCtx);
+        if (ru) {
+            // Set the cache max wait timeout very low as we do not want any FTDC
+            // operation to get blocked on cache eviction. 1 is a magic number that
+            // opts
+            // this thread out of all optional eviction without any waiting.
+            ru->setCacheMaxWaitTimeout(Milliseconds(1));
+        }
+
         for (const auto& nsStr : namespaces) {
 
             try {

src/mongo/db/ftdc/ftdc_server.cpp

@@ -41,6 +41,7 @@
 #include "mongo/db/ftdc/controller.h"
 #include "mongo/db/ftdc/ftdc_server_gen.h"
 #include "mongo/db/ftdc/ftdc_system_stats.h"
+#include "mongo/db/local_catalog/shard_role_api/transaction_resources.h"
 #include "mongo/db/mirror_maestro.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/service_context.h"
@@ -287,6 +288,15 @@ public:
         // the oplog window.
         // "lockContentionMetrics" is included to collect metrics on mutexes
 
+        auto ru = shard_role_details::getRecoveryUnit(opCtx);
+        if (ru) {
+            // Set the cache max wait timeout very low as we do not want any FTDC
+            // operation to get blocked on cache eviction. 1 is a magic number that
+            // opts
+            // this thread out of all optional eviction without any waiting.
+            ru->setCacheMaxWaitTimeout(Milliseconds(1));
+        }
+
         BSONObjBuilder commandBuilder;
         commandBuilder.append(kCommand, 1);
         commandBuilder.append("sharding", false);

src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp

@@ -123,6 +123,12 @@ void WiredTigerRecoveryUnit::_ensureSession() {
         _managedSession = _connection->getUninterruptibleSession(getIsolationConfig(_isolation));
     }
     _session = _managedSession.get();
+
+    if (_cacheMaxWaitTimeout.count()) {
+        _session->modifyConfiguration(
+            fmt::format("cache_max_wait_ms={}", durationCount<Milliseconds>(_cacheMaxWaitTimeout)),
+            "cache_max_wait_ms=0");
+    }
 }
 
 WiredTigerSession* WiredTigerRecoveryUnit::getSessionNoTxn() {
@@ -950,12 +956,15 @@ void WiredTigerRecoveryUnit::setOperationContext(OperationContext* opCtx) {
 }
 
 void WiredTigerRecoveryUnit::setCacheMaxWaitTimeout(Milliseconds timeout) {
+    // Save timeout because if there is currently no session, the next session that is opened will
+    // set the timeout.
     _cacheMaxWaitTimeout = timeout;
-    auto session = getSessionNoTxn();
-
-    session->modifyConfiguration(
-        fmt::format("cache_max_wait_ms={}", durationCount<Milliseconds>(_cacheMaxWaitTimeout)),
-        "cache_max_wait_ms=0");
+    WiredTigerConnection::BlockShutdown blockShutdown(_connection);
+    if (_session && !_connection->isShuttingDown()) {
+        _session->modifyConfiguration(
+            fmt::format("cache_max_wait_ms={}", durationCount<Milliseconds>(_cacheMaxWaitTimeout)),
+            "cache_max_wait_ms=0");
+    }
 }
 
 size_t WiredTigerRecoveryUnit::getCacheDirtyBytes() {

src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp

@@ -1004,7 +1004,11 @@ std::unique_ptr<WiredTigerSession> WiredTigerUtil::getStatisticsSession(
 
     // Obtain a session that can be used during shut down, potentially before the storage engine
     // itself shuts down.
-    return std::make_unique<WiredTigerSession>(&engine.getConnection(), handler, permit);
+    auto session = std::make_unique<WiredTigerSession>(&engine.getConnection(), handler, permit);
+    // Configure the session to avoid being coopted into cache eviction. We never want to block stat
+    // fetching on workload issues.
+    session->modifyConfiguration("cache_max_wait_ms=1", "cache_max_wait_ms=0");
+    return session;
 }
 
 bool WiredTigerUtil::collectConnectionStatistics(WiredTigerKVEngineBase& engine,