SERVER-105625 Add initial replay validation (#40910)

GitOrigin-RevId: b9140d578abfc72ea0464062ae02d0f542a9d8d6

BUILD.bazel

@@ -283,6 +283,7 @@ mongo_install(
         "//src/mongo/db/query/search/mongotmock",
         "//src/mongo/db/query/stage_builder/sbe/abt:optimizer_gdb_test_program",
         "//src/mongo/db/storage/key_string:ksdecode",
+        "//src/mongo/replay:mongor",
         "//src/mongo/s:mongos",
         "//src/mongo/shell:mongo",
         "//src/mongo/tools/mongobridge_tool:mongobridge",

jstests/noPassthrough/traffic_recording/traffic_replaying.js

@@ -1,20 +1,11 @@
 // tests for the traffic recording and replaying commands.
 // @tags: [requires_auth]
-function createDirectories(baseDir, customSubDir) {
-    const pathSep = _isWindows() ? "\\" : "/";
-    const recordingDirGlobal = MongoRunner.toRealDir("$dataDir" + pathSep + baseDir);
-    const recordingDir = MongoRunner.toRealDir(recordingDirGlobal + pathSep + customSubDir + pathSep);
-    jsTest.log("Creating a new directory: " + recordingDirGlobal);
-    jsTest.log("Creating a new directory: " + recordingDir);
-    assert(mkdir(recordingDirGlobal));
-    assert(mkdir(recordingDir));
-    return {recordingDirGlobal, recordingDir};
-}
 
-function cleanUpDirectory(directoryPath) {
-    jsTest.log(`Cleaning up directory: ${directoryPath}`);
-    removeFile(directoryPath); // Deletes the directory and its contents
-}
+import {
+    createDirectories,
+    cleanUpDirectory,
+    recordOperations,
+} from "jstests/noPassthrough/traffic_recording/traffic_replaying_lib.js";
 
 function parseRecordedTraffic(recordingFilePath) {
     const recordedTraffic = convertTrafficRecordingToBSON(recordingFilePath);
@@ -26,41 +17,27 @@ function parseRecordedTraffic(recordingFilePath) {
     return {opTypes, recordedTraffic};
 }
 
-function recordOperations(recordingDirGlobal, customRecordingDir, workflowCallback) {
-    const opts = {auth: "", setParameter: "trafficRecordingDirectory=" + recordingDirGlobal};
-    const mongodInstance = MongoRunner.runMongod(opts);
-
-    const adminDB = mongodInstance.getDB("admin");
-    const testDB = mongodInstance.getDB("test");
-    const coll = testDB.getCollection("foo");
-
-    adminDB.createUser({user: "admin", pwd: "pass", roles: jsTest.adminUserRoles});
-    adminDB.auth("admin", "pass");
-
-    assert.commandWorked(adminDB.runCommand({startTrafficRecording: 1, destination: customRecordingDir}));
-
-    const dbContext = {adminDB, testDB, coll, serverURI: `mongodb://${mongodInstance.host}`};
-
-    workflowCallback(dbContext);
-
-    const serverStatus = assert.commandWorked(testDB.runCommand({serverStatus: 1}));
-    const recordingFilePath = serverStatus.trafficRecording.recordingDir;
-
-    assert.commandWorked(adminDB.runCommand({stopTrafficRecording: 1}));
+function recordAndParseOperations(recordingDirGlobal, customRecordingDir, workflowCallback) {
+    const {mongodInstance, coll, recordingFilePath} = recordOperations(
+        recordingDirGlobal,
+        customRecordingDir,
+        workflowCallback,
+    );
+    const serverURI = `mongodb://${mongodInstance.host}`;
 
     MongoRunner.stopMongod(mongodInstance, null, {user: "admin", pwd: "pass"});
 
     return {
         ...parseRecordedTraffic(recordingFilePath),
         recordingFilePath,
-        serverURI: `mongodb://${mongodInstance.host}`,
+        serverURI,
         recordingDirGlobal,
     };
 }
 
 function runInstances(baseDir, customSubDir, workflowCallback) {
     const {recordingDirGlobal, recordingDir} = createDirectories(baseDir, customSubDir);
-    return recordOperations(recordingDirGlobal, customSubDir, workflowCallback);
+    return recordAndParseOperations(recordingDirGlobal, customSubDir, workflowCallback);
 }
 
 const defaultOperationsLambda = (dbContext) => {

jstests/noPassthrough/traffic_recording/traffic_replaying_lib.js

@@ -0,0 +1,50 @@
+/*
+ * Utilities for traffic replaying tests.
+ */
+
+export function createDirectories(baseDir, customSubDir) {
+    const pathSep = _isWindows() ? "\\" : "/";
+    const recordingDirGlobal = MongoRunner.toRealDir("$dataDir" + pathSep + baseDir);
+    const recordingDir = MongoRunner.toRealDir(recordingDirGlobal + pathSep + customSubDir + pathSep);
+    jsTest.log("Creating a new directory: " + recordingDirGlobal);
+    jsTest.log("Creating a new directory: " + recordingDir);
+    assert(mkdir(recordingDirGlobal));
+    assert(mkdir(recordingDir));
+    return {recordingDirGlobal, recordingDir};
+}
+
+export function cleanUpDirectory(directoryPath) {
+    jsTest.log(`Cleaning up directory: ${directoryPath}`);
+    removeFile(directoryPath); // Deletes the directory and its contents
+}
+
+/**
+ * This function sets up a mongod instance, runs 'opsToRecord' callback function to run some
+ * operations against the mongod instance with a default namespace. Operations in 'opsToRecord' are
+ * recorded. The running mongod instance(including the collection) and the recording file path are
+ * returned.
+ */
+export function recordOperations(recordingDirGlobal, customRecordingDir, opsToRecord) {
+    const opts = {auth: "", setParameter: "trafficRecordingDirectory=" + recordingDirGlobal};
+    const mongodInstance = MongoRunner.runMongod(opts);
+
+    const adminDB = mongodInstance.getDB("admin");
+    const testDB = mongodInstance.getDB("test");
+    const coll = testDB.getCollection("coll");
+
+    adminDB.createUser({user: "admin", pwd: "pass", roles: jsTest.adminUserRoles});
+    adminDB.auth("admin", "pass");
+
+    assert.commandWorked(adminDB.runCommand({startTrafficRecording: 1, destination: customRecordingDir}));
+
+    const dbContext = {adminDB, testDB, coll, serverURI: `mongodb://${mongodInstance.host}`};
+
+    opsToRecord(dbContext);
+
+    const serverStatus = assert.commandWorked(testDB.runCommand({serverStatus: 1}));
+    const recordingFilePath = serverStatus.trafficRecording.recordingDir;
+
+    assert.commandWorked(adminDB.runCommand({stopTrafficRecording: 1}));
+
+    return {mongodInstance, coll, recordingFilePath};
+}

jstests/noPassthrough/traffic_recording/traffic_replaying_with_mongor.js

@@ -0,0 +1,71 @@
+// Tests for the traffic recording and replaying with 'mongor'
+// @tags: [requires_auth]
+
+import {
+    cleanUpDirectory,
+    createDirectories,
+    recordOperations,
+} from "jstests/noPassthrough/traffic_recording/traffic_replaying_lib.js";
+
+function opsToRecord(dbContext) {
+    const {testDB, coll} = dbContext;
+    assert.commandWorked(coll.insert({_id: 1, val: 1}));
+    assert.eq(1, coll.findOne().val);
+    assert.commandWorked(coll.insert({_id: 2, val: "2"}));
+    assert.commandWorked(coll.deleteOne({val: 1}));
+    assert.eq(1, coll.aggregate().toArray().length);
+    assert.commandWorked(coll.update({_id: 2}, {val: 2}));
+}
+
+/**
+ * This function runs a set of operations against a collection (with namespace 'db.test.coll').
+ * These operations will be recorded and replayed later against a shadow instance. This test will
+ * compare documents on both instances to test 'mongor's replaying function.
+ *
+ * A recording file path and documents on the collection will be returned.
+ */
+function runTrafficRecording(baseDir, customSubDir) {
+    const {recordingDirGlobal, recordingDir} = createDirectories(baseDir, customSubDir);
+    const {mongodInstance, coll, recordingFilePath} = recordOperations(recordingDirGlobal, customSubDir, opsToRecord);
+
+    const docs = coll.find({}).toArray();
+
+    MongoRunner.stopMongod(mongodInstance, null, {user: "admin", pwd: "pass"});
+
+    return {recordingFilePath, docs, recordingDirGlobal};
+}
+
+const {recordingFilePath, docs, recordingDirGlobal} = runTrafficRecording("traffic_recording", "recordings");
+jsTest.log("Documents on the original instance: ");
+printjson(docs);
+
+function setupShadowInstance() {
+    mkdir("shadow");
+    const opts = {dbpath: "shadow"};
+    const shadowMongod = MongoRunner.runMongod(opts);
+
+    const testDB = shadowMongod.getDB("test");
+    const shadowColl = testDB.getCollection("coll");
+    shadowColl.drop();
+
+    const shadowURI = `mongodb://${shadowMongod.host}`;
+    return {
+        shadowURI,
+        shadowMongod,
+        shadowColl,
+    };
+}
+const {shadowURI, shadowMongod, shadowColl} = setupShadowInstance();
+
+// Runs 'mongor' to replay the recorded operations against the shadow instance.
+runProgram("mongor", "-i", recordingFilePath, "-t", shadowURI);
+
+const shadowDocs = shadowColl.find().toArray();
+jsTest.log("Documents on the shadow instance: ");
+printjson(shadowDocs);
+
+assert.eq(docs, shadowDocs);
+assert.eq(shadowDocs.length, 1);
+
+MongoRunner.stopMongod(shadowMongod);
+cleanUpDirectory(recordingDirGlobal);