SERVER-118736 Added disagg test suite which forces checkpoints while metadata is inconsistent (#48459)

GitOrigin-RevId: 3d97246b70a9cd3d890893878add8e0a2b09e835

etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml

@@ -2493,6 +2493,24 @@ tasks:
           # Secondaries need extra WT cache on disagg to avoid filling up the cache in between checkpoints.
           resmoke_args: --storageEngineCacheSizeGB=5
 
+  - <<: *gen_task_template
+    name: disagg_repl_force_checkpoint_gen
+    tags:
+      [
+        "assigned_to_jira_team_server_disagg",
+        "default",
+        "large",
+        "clustered_collections",
+        "requires_extra_system_deps",
+      ]
+    commands:
+      - func: "generate resmoke tasks"
+        vars:
+          suite: disagg_repl_force_checkpoint
+          use_large_distro: "true"
+          # Secondaries need extra WT cache on disagg to avoid filling up the cache in between checkpoints.
+          resmoke_args: --storageEngineCacheSizeGB=5
+
   - <<: *gen_task_template
     name: disagg_backup_restore_jscore_passthrough_gen
     tags:

src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp

@@ -37,7 +37,6 @@
 #include "mongo/db/server_options.h"
 #include "mongo/db/storage/exceptions.h"
 #include "mongo/db/storage/execution_context.h"
-#include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_connection.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_error_util.h"
@@ -73,6 +72,7 @@ namespace {
 logv2::LogSeverity kSlowTransactionSeverity = logv2::LogSeverity::Debug(1);
 
 MONGO_FAIL_POINT_DEFINE(doUntimestampedWritesForIdempotencyTests);
+MONGO_FAIL_POINT_DEFINE(forceCheckpointOnTableCreate);
 
 const char* getIsolationConfig(RecoveryUnit::Isolation isolation) {
     switch (isolation) {
@@ -390,6 +390,10 @@ void WiredTigerRecoveryUnit::_txnClose(bool commit) {
             _session->modifyConfiguration("cache_max_wait_ms=1", "cache_max_wait_ms=0");
         }
 
+        if (!_createdTables.empty() && _isTimestamped) {
+            _forceCheckpointOnTableCreateForTestingIfEnabled();
+        }
+
         wtRet = _session->commit_transaction(nullptr);
 
         LOGV2_DEBUG(
@@ -465,6 +469,20 @@ void WiredTigerRecoveryUnit::_txnClose(bool commit) {
     }
 }
 
+void WiredTigerRecoveryUnit::_forceCheckpointOnTableCreateForTestingIfEnabled() {
+    if (MONGO_unlikely(forceCheckpointOnTableCreate.shouldFail())) {
+        auto cpSession = _connection->getUninterruptibleSession();
+        int cpRet = cpSession->checkpoint("use_timestamp=true");
+        if (cpRet != EBUSY && cpRet != ENOTSUP && cpRet != EINVAL) {
+            invariantWTOK(cpRet, *cpSession);
+        }
+        LOGV2(11873600,
+              "Forced checkpoint before committing table-creating transaction",
+              "numCreatedTables"_attr = _createdTables.size(),
+              "result"_attr = cpRet);
+    }
+}
+
 Status WiredTigerRecoveryUnit::majorityCommittedSnapshotAvailable() const {
     invariant(_timestampReadSource == ReadSource::kMajorityCommitted);
     auto snapshotName = _connection->snapshotManager().getMinSnapshotForNextCommittedRead();

src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h

@@ -243,6 +243,13 @@ private:
     void _txnClose(bool commit);
     void _txnOpen();
 
+    /**
+     * Forces a checkpoint on table create if the failpoint forceCheckpointOnTableCreate is
+     * enabled. This is used to reproduce excess-table-in-checkpoint inconsistencies for testing.
+     * The checkpoint is forced before committing the transaction that created the table.
+     */
+    void _forceCheckpointOnTableCreateForTestingIfEnabled();
+
     /**
      * Starts a transaction at the current all_durable timestamp.
      * Returns the timestamp the transaction was started at.