SERVER-106933: Add testing for cluster restart scenarios for extension enablement / upgrade / downgrade (#40179)

GitOrigin-RevId: fc426a268b98549741fabdd41c325d2337891efa

.github/CODEOWNERS

@@ -879,6 +879,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 /jstests/libs/**/check_metadata_consistency_helpers.js @10gen/server-catalog-and-routing @svc-auto-approve-bot
 /jstests/libs/**/catalog_list_operations_consistency_validator.js @10gen/server-catalog-and-routing @svc-auto-approve-bot
 /jstests/libs/**/raw_operation_utils.js @10gen/server-collection-write-path @svc-auto-approve-bot
+/jstests/libs/**/json_utils.js @10gen/query-integration-extensions @svc-auto-approve-bot
 
 # The following patterns are parsed from ./jstests/libs/clustered_collections/OWNERS.yml
 /jstests/libs/clustered_collections/**/* @10gen/server-collection-write-path @svc-auto-approve-bot
@@ -971,6 +972,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 # The following patterns are parsed from ./jstests/multiVersion/genericBinVersion/capped_collections/OWNERS.yml
 /jstests/multiVersion/genericBinVersion/capped_collections/**/* @10gen/server-collection-write-path @svc-auto-approve-bot
 
+# The following patterns are parsed from ./jstests/multiVersion/genericBinVersion/extensions_api/OWNERS.yml
+/jstests/multiVersion/genericBinVersion/extensions_api/ @10gen/query-integration-extensions-api @svc-auto-approve-bot
+
 # The following patterns are parsed from ./jstests/multiVersion/genericBinVersion/index_builds/OWNERS.yml
 /jstests/multiVersion/genericBinVersion/index_builds/**/* @10gen/server-index-builds @svc-auto-approve-bot
 

BUILD.bazel

@@ -449,6 +449,7 @@ extensions_with_config(
         # "_mongo_extension" suffix.
         "//src/mongo/db/extension/test_examples:no_symbol_bad_extension",
         "//src/mongo/db/extension/test_examples:duplicate_stage_descriptor_bad_extension",
+        "//src/mongo/db/extension/test_examples:foo_extension_v2",
         "//src/mongo/db/extension/test_examples:vector_search_extension",
     ],
 )

buildscripts/resmokeconfig/matrix_suites/generated_suites/multiversion_auth_future_git_tag.yml

@@ -42,6 +42,7 @@ selector:
   exclude_with_any_tags:
     - featureFlagToaster
     - featureFlagSpoon
+    - auth_incompatible
     - DISABLED_TEMPORARILY_DUE_TO_FCV_UPGRADE
   roots:
     - jstests/multiVersion/**/*.js

buildscripts/resmokeconfig/matrix_suites/generated_suites/multiversion_future_git_tag.yml

@@ -30,6 +30,7 @@ selector:
   exclude_with_any_tags:
     - featureFlagToaster
     - featureFlagSpoon
+    - requires_auth
     - DISABLED_TEMPORARILY_DUE_TO_FCV_UPGRADE
   roots:
     - jstests/multiVersion/**/*.js

buildscripts/resmokeconfig/suites/multiversion.yml

@@ -19,6 +19,9 @@ selector:
   exclude_with_any_tags:
     - featureFlagToaster
     - featureFlagSpoon
+    # This suite's fixture does not set up authentication. There is a multiversion_auth variant
+    # which does. Those tests which require auth should not run here, only in the auth variant.
+    - requires_auth
 
 # Multiversion tests start their own mongod's.
 executor:

buildscripts/resmokeconfig/suites/multiversion_auth.yml

@@ -26,6 +26,7 @@ selector:
   exclude_with_any_tags:
     - featureFlagToaster
     - featureFlagSpoon
+    - auth_incompatible
 
 # Multiversion tests start their own mongod's.
 executor:

jstests/libs/OWNERS.yml

@@ -84,3 +84,6 @@ filters:
   - "raw_operation_utils.js":
     approvers:
       - 10gen/server-collection-write-path
+  - "json_utils.js":
+    approvers:
+      - 10gen/query-integration-extensions

jstests/libs/json_utils.js

@@ -0,0 +1,8 @@
+// JSON-related utility functions.
+
+/**
+ * Makes a deep copy of the provided JSON.
+ */
+export function copyJSON(json) {
+    return JSON.parse(JSON.stringify(json));
+}

jstests/multiVersion/genericBinVersion/extensions_api/BUILD.bazel

@@ -0,0 +1,12 @@
+load("//bazel:mongo_js_rules.bzl", "all_subpackage_javascript_files", "mongo_js_library")
+
+package(default_visibility = ["//visibility:public"])
+
+mongo_js_library(
+    name = "all_javascript_files",
+    srcs = glob([
+        "*.js",
+    ]),
+)
+
+all_subpackage_javascript_files()

jstests/multiVersion/genericBinVersion/extensions_api/OWNERS.yml

@@ -0,0 +1,5 @@
+version: 2.0.0
+filters:
+  - "*":
+    approvers:
+      - 10gen/query-integration-extensions-api

jstests/multiVersion/genericBinVersion/extensions_api/extension_foo_upgrade_downgrade.js

@@ -0,0 +1,208 @@
+/**
+ * This test simulates a scenario when you're upgrading/downgrading an extension between V1 and V2
+ * while maintaining the same server version. For the sake of testing, $testFoo in V1 must provide
+ * an empty stage definition like {$testFoo: {}} or it will reject the query at parsing; $testFoo in
+ * V2 loosens those restrictions. We test this expected behavior for upgrade:
+ *     - The nodes are spun up with extension V1 loaded. $testFoo is accepted in queries and views,
+ *       but it is always rejected (in queries and views) with a non-empty stage definition.
+ *     - In a ReplSet, $testFoo V1 queries must be accepted the whole time. $testFoo V2 queries
+ *       are only accepted once all binaries are upgraded (the primary last). Note that we are
+ *       missing coverage for ReplSet upgrade/downgrade where we aren't just connected to the
+ *       primary the whole time (TODO SERVER-109457).
+ *     - In a sharded cluster, again $testFoo V1 queries must be accepted the whole time, and
+ *       $testFoo V2 queries are only accepted once all binaries are upgraded (the mongos last).
+ *        * However, there is one difference: in the period between when the shards start restarting
+ *          until mongos has restarted, the cluster may allow you to create a view with $testFoo V2
+ *          even though you cannot run queries on the view until mongos has upgraded. This behavior
+ *          is flaky, as it occasionally rejects the view creation to begin with. Note that this
+ *          only happens when auth is not enabled (see extension_foo_upgrade_downgrade_auth.js for
+ *          testing with more consistent behavior when auth is enabled).
+ *     - The downgrade behavior is reverse and more consistent (no flaky view issues).
+ *
+ * This test is technically not multiversion since we only use latest binaries, but it stays with
+ * multiversion tests since we use the upgrade/downgrade library utils.
+ *
+ * TODO SERVER-109457 Investigate more ReplSet scenarios.
+ *
+ * TODO SERVER-109450 Auth-off cluster should have same behavior as auth-on cluster.
+ * Extensions are only available on Linux machines.
+ * @tags: [
+ *   auth_incompatible,
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ * ]
+ */
+import "jstests/multiVersion/libs/multi_rs.js";
+import "jstests/multiVersion/libs/multi_cluster.js";
+
+import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
+import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+import {isLinux} from "jstests/libs/os_helpers.js";
+import {
+    testPerformReplSetRollingRestart
+} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
+import {
+    testPerformShardedClusterRollingRestart
+} from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
+
+if (!isLinux()) {
+    jsTest.log.info("Skipping test since extensions are only available on Linux platforms.");
+    quit();
+}
+
+const pathToExtensionFoo = MongoRunner.getExtensionPath("libfoo_mongo_extension.so");
+const pathToExtensionFooV2 = MongoRunner.getExtensionPath("libfoo_extension_v2.so");
+
+const collName = jsTestName();
+const viewName = "foo_view";
+const getDB = (primaryConnection) => primaryConnection.getDB(jsTestName());
+const data = [{_id: 0, test: "a"}, {_id: 1, test: "b"}, {_id: 2, test: "c"}];
+
+const fooParseErrorCodes = [10624200, 10624201];
+
+export const fooV1Options = {
+    loadExtensions: [pathToExtensionFoo],
+    setParameter: {featureFlagExtensionsAPI: true}
+};
+export const fooV2Options = {
+    loadExtensions: [pathToExtensionFooV2],
+    setParameter: {featureFlagExtensionsAPI: true}
+};
+
+export function setupCollection(primaryConn, shardingTest = null) {
+    const coll = assertDropAndRecreateCollection(getDB(primaryConn), collName);
+    if (shardingTest) {
+        // Enable sharding on this collection by _id. After inserting the data below,
+        // this will distribute 2 documents to one shard and 1 document to the other.
+        shardingTest.shardColl(coll, {_id: 1});
+    }
+    assert.commandWorked(coll.insertMany(data));
+}
+
+export function assertFooStageAcceptedV1Only(primaryConn) {
+    const db = getDB(primaryConn);
+    db[viewName].drop();
+
+    // $testFoo in v1 should reject a non-empty stage definition.
+    assert.commandFailedWithCode(
+        db.runCommand({aggregate: collName, pipeline: [{$testFoo: {nonEmpty: true}}], cursor: {}}),
+        fooParseErrorCodes);
+
+    // $testFoo with empty stage definition is accepted in a plain aggregation command.
+    const result = db.runCommand({aggregate: collName, pipeline: [{$testFoo: {}}], cursor: {}});
+    assert.commandWorked(result);
+    assertArrayEq({actual: result.cursor.firstBatch, expected: data});
+
+    // $testFoo with empty stage definition is accepted in a view pipeline.
+    assert.commandWorked(db.createView(viewName, collName, [{$testFoo: {}}]));
+    const viewResult = assert.commandWorked(
+        db.runCommand({aggregate: viewName, pipeline: [{$match: {_id: 0}}], cursor: {}}));
+    assertArrayEq({actual: viewResult.cursor.firstBatch, expected: [data[0]]});
+}
+
+export function assertFooStageAcceptedV1AndV2(primaryConn) {
+    const db = getDB(primaryConn);
+    db[viewName].drop();
+
+    // $testFoo with empty stage definition is accepted in a plain aggregation command.
+    let result = db.runCommand({aggregate: collName, pipeline: [{$testFoo: {}}], cursor: {}});
+    assert.commandWorked(result);
+    assertArrayEq({actual: result.cursor.firstBatch, expected: data});
+
+    // $testFoo V2 now accepts a non-empty stage definition.
+    result =
+        db.runCommand({aggregate: collName, pipeline: [{$testFoo: {nonEmpty: true}}], cursor: {}});
+    assert.commandWorked(result);
+    assertArrayEq({actual: result.cursor.firstBatch, expected: data});
+
+    // $testFoo with empty stage definition is accepted in a view pipeline.
+    assert.commandWorked(db.createView(viewName, collName, [{$testFoo: {}}]));
+    let viewResult = assert.commandWorked(
+        db.runCommand({aggregate: viewName, pipeline: [{$match: {_id: 0}}], cursor: {}}));
+    assertArrayEq({actual: viewResult.cursor.firstBatch, expected: [data[0]]});
+
+    db[viewName].drop();
+
+    // $testFoo with a non-empty stage definition is now accepted in a view pipeline.
+    assert.commandWorked(db.createView(viewName, collName, [{$testFoo: {nonEmpty: true}}]));
+    viewResult = assert.commandWorked(
+        db.runCommand({aggregate: viewName, pipeline: [{$match: {_id: 0}}], cursor: {}}));
+    assertArrayEq({actual: viewResult.cursor.firstBatch, expected: [data[0]]});
+}
+
+function assertFooStageAcceptedV1OnlyPlusV2ViewCreation(primaryConn) {
+    const db = getDB(primaryConn);
+    db[viewName].drop();
+
+    // $testFoo should reject a non-empty stage definition in an aggregation command.
+    assert.commandFailedWithCode(
+        db.runCommand({aggregate: collName, pipeline: [{$testFoo: {nonEmpty: true}}], cursor: {}}),
+        fooParseErrorCodes);
+
+    // $testFoo with empty stage definition is accepted in a plain aggregation command.
+    const result = db.runCommand({aggregate: collName, pipeline: [{$testFoo: {}}], cursor: {}});
+    assert.commandWorked(result);
+    assertArrayEq({actual: result.cursor.firstBatch, expected: data});
+
+    // $testFoo with empty stage definition is accepted in a view pipeline.
+    assert.commandWorked(db.createView(viewName, collName, [{$testFoo: {}}]));
+    const viewResult = assert.commandWorked(
+        db.runCommand({aggregate: viewName, pipeline: [{$match: {_id: 0}}], cursor: {}}));
+    assertArrayEq({actual: viewResult.cursor.firstBatch, expected: [data[0]]});
+
+    // This behavior is flaky. Most of the time here, $testFoo with a non-empty stage definition is
+    // allowed in a view pipeline, but queries over that view cannot yet be run. Occasionally,
+    // however, the createView() command is rejected first for not recognizing $testFoo V2 yet.
+    db[viewName].drop();
+    const createViewResult = db.createView(viewName, collName, [{$testFoo: {nonEmpty: true}}]);
+    if (createViewResult["ok"]) {
+        assert.commandFailedWithCode(db.runCommand({aggregate: viewName, pipeline: [], cursor: {}}),
+                                     fooParseErrorCodes);
+    } else {
+        assert.commandFailedWithCode(createViewResult, fooParseErrorCodes);
+    }
+}
+
+// Test upgrading foo extension in a replica set.
+testPerformReplSetRollingRestart({
+    startingNodeOptions: fooV1Options,
+    restartNodeOptions: fooV2Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1Only,
+    afterSecondariesHaveRestarted: assertFooStageAcceptedV1Only,
+    afterPrimariesHaveRestarted: assertFooStageAcceptedV1AndV2,
+});
+
+// Test upgrading foo extension in a sharded cluster.
+testPerformShardedClusterRollingRestart({
+    startingNodeOptions: fooV1Options,
+    restartNodeOptions: fooV2Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1Only,
+    afterConfigHasRestarted: assertFooStageAcceptedV1Only,
+    afterSecondaryShardHasRestarted: assertFooStageAcceptedV1OnlyPlusV2ViewCreation,
+    afterPrimaryShardHasRestarted: assertFooStageAcceptedV1OnlyPlusV2ViewCreation,
+    afterMongosHasRestarted: assertFooStageAcceptedV1AndV2,
+});
+
+// Test downgrading foo extension in a replica set.
+testPerformReplSetRollingRestart({
+    startingNodeOptions: fooV2Options,
+    restartNodeOptions: fooV1Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1AndV2,
+    afterSecondariesHaveRestarted: assertFooStageAcceptedV1AndV2,
+    afterPrimariesHaveRestarted: assertFooStageAcceptedV1Only,
+});
+
+// Test downgrading foo extension in a sharded cluster.
+testPerformShardedClusterRollingRestart({
+    startingNodeOptions: fooV2Options,
+    restartNodeOptions: fooV1Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1AndV2,
+    afterConfigHasRestarted: assertFooStageAcceptedV1AndV2,
+    afterSecondaryShardHasRestarted: assertFooStageAcceptedV1Only,
+    afterPrimaryShardHasRestarted: assertFooStageAcceptedV1Only,
+    afterMongosHasRestarted: assertFooStageAcceptedV1Only,
+});

jstests/multiVersion/genericBinVersion/extensions_api/extension_foo_upgrade_downgrade_auth.js

@@ -0,0 +1,72 @@
+/**
+ * This is an extension of extension_foo_upgrade_downgrade.js that only runs in auth scenarios. See
+ * the header comment in that file for an explanation about expected behavior.
+ *
+ * TODO SERVER-109450 Delete this test once auth and non-auth have the same behavior.
+ *
+ * Extensions are only available on Linux machines.
+ * @tags: [
+ *   requires_auth,
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ * ]
+ */
+import {isLinux} from "jstests/libs/os_helpers.js";
+import {
+    assertFooStageAcceptedV1AndV2,
+    assertFooStageAcceptedV1Only,
+    fooV1Options,
+    fooV2Options,
+    setupCollection
+} from "jstests/multiVersion/genericBinVersion/extensions_api/extension_foo_upgrade_downgrade.js";
+import {
+    testPerformReplSetRollingRestart
+} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
+import {
+    testPerformShardedClusterRollingRestart
+} from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
+
+if (!isLinux()) {
+    jsTest.log.info("Skipping test since extensions are only available on Linux platforms.");
+    quit();
+}
+
+testPerformReplSetRollingRestart({
+    startingNodeOptions: fooV1Options,
+    restartNodeOptions: fooV2Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1Only,
+    afterSecondariesHaveRestarted: assertFooStageAcceptedV1Only,
+    afterPrimariesHaveRestarted: assertFooStageAcceptedV1AndV2,
+});
+
+testPerformShardedClusterRollingRestart({
+    startingNodeOptions: fooV1Options,
+    restartNodeOptions: fooV2Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1Only,
+    afterConfigHasRestarted: assertFooStageAcceptedV1Only,
+    afterSecondaryShardHasRestarted: assertFooStageAcceptedV1Only,
+    afterPrimaryShardHasRestarted: assertFooStageAcceptedV1Only,
+    afterMongosHasRestarted: assertFooStageAcceptedV1AndV2,
+});
+
+testPerformReplSetRollingRestart({
+    startingNodeOptions: fooV2Options,
+    restartNodeOptions: fooV1Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1AndV2,
+    afterSecondariesHaveRestarted: assertFooStageAcceptedV1AndV2,
+    afterPrimariesHaveRestarted: assertFooStageAcceptedV1Only,
+});
+
+testPerformShardedClusterRollingRestart({
+    startingNodeOptions: fooV2Options,
+    restartNodeOptions: fooV1Options,
+    setupFn: setupCollection,
+    beforeRestart: assertFooStageAcceptedV1AndV2,
+    afterConfigHasRestarted: assertFooStageAcceptedV1AndV2,
+    afterSecondaryShardHasRestarted: assertFooStageAcceptedV1Only,
+    afterPrimaryShardHasRestarted: assertFooStageAcceptedV1Only,
+    afterMongosHasRestarted: assertFooStageAcceptedV1Only,
+});

jstests/multiVersion/genericBinVersion/extensions_api/upgrade_enables_extension_foo.js

@@ -0,0 +1,137 @@
+/**
+ * This test simulates a scenario where you're upgrading from a MongoDB version with no extensions
+ * to a loaded to a higher MongoDB version with an extension loaded. We test this expected behavior:
+ *    - The nodes are spun up at a lower version with no extensions loaded. All $testFoo queries
+ *      should be rejected, both in an agg command and in a createView pipeline.
+ *    - In a ReplSet, $testFoo queries should be accepted once all binaries are upgraded (the
+ *      primary last). Nothing is dependent on FCV. Note that we are missing coverage for ReplSet
+ *      upgrade/downgrade where we aren't just connected to the primary the whole time
+ *      (TODO SERVER-109457).
+ *    - In a sharded cluster, $testFoo queries should be accepted once all binaries are upgraded
+ *      (the mongos last), and again nothing is dependent on FCV.
+ *       * However, there is one difference: in the period between when the shards start restarting
+ *         until mongos has restarted, the cluster may allow you to create a view with $testFoo
+ *         even though you cannot run queries on the view until mongos has upgraded. This behavior
+ *         is flaky, as it occasionally rejets the view creation to begin with. Note that this only
+ *         happens when auth is not enabed (see upgrade_enables_extension_foo_auth.js for testing
+ *         with more consistent behavior when auth is enabled).
+ *
+ * TODO SERVER-109457 Investigate more ReplSet scenarios.
+ *
+ * TODO SERVER-109450 Auth-off cluster should have same behavior as auth-on cluster.
+ * Extensions are only available on Linux machines.
+ * @tags: [
+ *   auth_incompatible,
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ * ]
+ */
+import {assertArrayEq} from "jstests/aggregation/extras/utils.js";
+import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+import {isLinux} from "jstests/libs/os_helpers.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
+import {
+    testPerformUpgradeSharded
+} from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
+
+if (!isLinux()) {
+    jsTest.log.info("Skipping test since extensions are only available on Linux platforms.");
+    quit();
+}
+
+const pathToExtensionFoo = MongoRunner.getExtensionPath("libfoo_mongo_extension.so");
+
+const fooStageUnrecognizedErrCode = 40324;
+export const fooExtensionNodeOptions = {
+    loadExtensions: [pathToExtensionFoo],
+    setParameter: {featureFlagExtensionsAPI: true}
+};
+
+const collName = jsTestName();
+const viewName = "foo_view";
+const getDB = (primaryConnection) => primaryConnection.getDB(jsTestName());
+const docs =
+    [{_id: 0, foo: "dreaming"}, {_id: 1, foo: "about"}, {_id: 2, foo: "super cool extensions"}];
+
+export function setupCollection(primaryConn, shardingTest = null) {
+    const coll = assertDropAndRecreateCollection(getDB(primaryConn), collName);
+
+    if (shardingTest) {
+        // Enable sharding on this collection by _id. After inserting the data below,
+        // this will distribute 2 documents to one shard and 1 document to the other.
+        shardingTest.shardColl(coll, {_id: 1});
+    }
+
+    assert.commandWorked(coll.insertMany(docs));
+}
+
+export function assertFooStageRejected(primaryConn) {
+    const db = getDB(primaryConn);
+    db[viewName].drop();
+
+    // $testFoo is rejected in a plain aggregation command.
+    assert.commandFailedWithCode(
+        db.runCommand({aggregate: collName, pipeline: [{$testFoo: {}}], cursor: {}}),
+        fooStageUnrecognizedErrCode);
+
+    // $testFoo is rejected in a view pipeline.
+    assert.commandFailedWithCode(db.createView(viewName, collName, [{$testFoo: {}}]),
+                                 fooStageUnrecognizedErrCode);
+}
+
+export function assertFooStageAccepted(primaryConn) {
+    const db = getDB(primaryConn);
+    db[viewName].drop();
+
+    // $testFoo is accepted in a plain aggregation command.
+    const result = db.runCommand({aggregate: collName, pipeline: [{$testFoo: {}}], cursor: {}});
+    assert.commandWorked(result);
+    assertArrayEq({actual: result.cursor.firstBatch, expected: docs});
+
+    // $testFoo is accepted in a view pipeline.
+    assert.commandWorked(db.createView(viewName, collName, [{$testFoo: {}}]));
+    const viewResult = assert.commandWorked(
+        db.runCommand({aggregate: viewName, pipeline: [{$match: {_id: 0}}], cursor: {}}));
+    assertArrayEq({actual: viewResult.cursor.firstBatch, expected: [docs[0]]});
+}
+
+function assertFooViewCreationAllowedButQueriesRejected(primaryConn) {
+    const db = getDB(primaryConn);
+    db[viewName].drop();
+
+    // This behavior is flaky. Most of the time here, $testFoo is permitted in a pipeline to
+    // createView(), but you cannot run queries over that view yet. Occasionally, however,
+    // the createView() command is rejected first for not recognizing $testFoo.
+    const createViewResult = db.createView(viewName, collName, [{$testFoo: {}}]);
+    if (createViewResult["ok"]) {
+        assert.commandFailedWithCode(db.runCommand({aggregate: viewName, pipeline: [], cursor: {}}),
+                                     fooStageUnrecognizedErrCode);
+    } else {
+        assert.commandFailedWithCode(createViewResult, fooStageUnrecognizedErrCode);
+    }
+
+    // $testFoo is rejected in a plain aggregation command.
+    assert.commandFailedWithCode(
+        db.runCommand({aggregate: collName, pipeline: [{$testFoo: {}}], cursor: {}}),
+        fooStageUnrecognizedErrCode);
+}
+
+testPerformUpgradeReplSet({
+    upgradeNodeOptions: fooExtensionNodeOptions,
+    setupFn: setupCollection,
+    whenFullyDowngraded: assertFooStageRejected,
+    whenSecondariesAreLatestBinary: assertFooStageRejected,
+    whenBinariesAreLatestAndFCVIsLastLTS: assertFooStageAccepted,
+    whenFullyUpgraded: assertFooStageAccepted,
+});
+
+testPerformUpgradeSharded({
+    upgradeNodeOptions: fooExtensionNodeOptions,
+    setupFn: setupCollection,
+    whenFullyDowngraded: assertFooStageRejected,
+    whenOnlyConfigIsLatestBinary: assertFooStageRejected,
+    whenSecondariesAndConfigAreLatestBinary: assertFooViewCreationAllowedButQueriesRejected,
+    whenMongosBinaryIsLastLTS: assertFooViewCreationAllowedButQueriesRejected,
+    whenBinariesAreLatestAndFCVIsLastLTS: assertFooStageAccepted,
+    whenFullyUpgraded: assertFooStageAccepted,
+});

jstests/multiVersion/genericBinVersion/extensions_api/upgrade_enables_extension_foo_auth.js

@@ -0,0 +1,49 @@
+/**
+ * This is an extension of upgrade_enables_extension_foo.js that only runs in auth scenarios. See
+ * the header comment in that file for an explanation about expected behavior.
+ *
+ * TODO SERVER-109450 Delete this test once auth and non-auth have the same behavior.
+ *
+ * Extensions are only available on Linux machines.
+ * @tags: [
+ *   requires_auth,
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ * ]
+ */
+import {isLinux} from "jstests/libs/os_helpers.js";
+import {
+    assertFooStageAccepted,
+    assertFooStageRejected,
+    fooExtensionNodeOptions,
+    setupCollection
+} from "jstests/multiVersion/genericBinVersion/extensions_api/upgrade_enables_extension_foo.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
+import {
+    testPerformUpgradeSharded
+} from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
+
+if (!isLinux()) {
+    jsTest.log.info("Skipping test since extensions are only available on Linux platforms.");
+    quit();
+}
+
+testPerformUpgradeReplSet({
+    upgradeNodeOptions: fooExtensionNodeOptions,
+    setupFn: setupCollection,
+    whenFullyDowngraded: assertFooStageRejected,
+    whenSecondariesAreLatestBinary: assertFooStageRejected,
+    whenBinariesAreLatestAndFCVIsLastLTS: assertFooStageAccepted,
+    whenFullyUpgraded: assertFooStageAccepted,
+});
+
+testPerformUpgradeSharded({
+    upgradeNodeOptions: fooExtensionNodeOptions,
+    setupFn: setupCollection,
+    whenFullyDowngraded: assertFooStageRejected,
+    whenOnlyConfigIsLatestBinary: assertFooStageRejected,
+    whenSecondariesAndConfigAreLatestBinary: assertFooStageRejected,
+    whenMongosBinaryIsLastLTS: assertFooStageRejected,
+    whenBinariesAreLatestAndFCVIsLastLTS: assertFooStageAccepted,
+    whenFullyUpgraded: assertFooStageAccepted,
+});

jstests/multiVersion/libs/mixed_version_fixture_test.js

@@ -1,58 +1,120 @@
 
 import "jstests/multiVersion/libs/multi_rs.js";
 
+import {copyJSON} from "jstests/libs/json_utils.js";
 import {ReplSetTest} from "jstests/libs/replsettest.js";
 
-export function testPerformUpgradeDowngradeReplSet({
-    setupFn,
-    whenFullyDowngraded,
-    whenSecondariesAreLatestBinary,
-    whenBinariesAreLatestAndFCVIsLastLTS,
-    whenFullyUpgraded
-}) {
-    const lastLTSVersion = {binVersion: "last-lts"};
-    const latestVersion = {binVersion: "latest"};
+const latestVersion = {
+    binVersion: "latest"
+};
+const lastLTSVersion = {
+    binVersion: "last-lts"
+};
 
+/**
+ * Simulates a rolling restart of a Replica Set and calls assertion callbacks at each step along
+ * the way. The procedure works like this:
+ *   1. Spin up a ReplSet with startingVersion and startingNodeOptions.
+ *   2. Call the setupFn() and beforeRestart() assertions.
+ *   3. Restart the secondaries (with restartVersion and restartNodeOptions).
+ *   4. Call the afterSecondariesHaveRestarted() assertions.
+ *   5. Restart the primaries (with restartVersion and restartNodeOptions).
+ *   6. Call the afterPrimariesHaveRestarted() assertions.
+ *   7. [if afterFCVBump is non-null] Upgrade the FCV.
+ *   8. [if afterFCVBump is non-null] Call the afterFCVBump() assertions.
+ *   9. [if afterFCVBump is non-null] Downgrade the FCV.
+ *   10. [if afterFCVBump is non-null] Call the afterPrimariesHaveRestarted() assertions again.
+ */
+export function testPerformReplSetRollingRestart({
+    startingVersion = latestVersion,
+    restartVersion = latestVersion,
+    startingNodeOptions = {},
+    restartNodeOptions = {},
+    setupFn,
+    beforeRestart,
+    afterSecondariesHaveRestarted,
+    afterPrimariesHaveRestarted,
+    afterFCVBump = null,
+}) {
     const rst = new ReplSetTest({
         name: jsTestName(),
         nodes: [
-            {...lastLTSVersion},
-            {...lastLTSVersion},
+            {
+                ...startingVersion,
+            },
+            {
+                ...startingVersion,
+            },
         ],
+        // Create a copy of the options since the callee may modify them.
+        nodeOptions: copyJSON(startingNodeOptions),
     });
+
     rst.startSet();
     rst.initiate(null, null, {initiateWithDefaultElectionTimeout: true});
 
     let primaryConnection = rst.getPrimary();
-    const getAdminDB = () => primaryConnection.getDB("admin");
 
     setupFn(primaryConnection);
+    beforeRestart(primaryConnection);
 
-    whenFullyDowngraded(primaryConnection);
-
-    // Upgrade the secondaries only.
-    rst.upgradeSecondaries({...latestVersion});
+    // Restart the secondaries only.
+    // Create a copy of the options since the callee may modify them.
+    rst.upgradeSecondaries({...restartVersion, ...copyJSON(restartNodeOptions)});
     primaryConnection = rst.getPrimary();
+    afterSecondariesHaveRestarted(primaryConnection);
 
-    whenSecondariesAreLatestBinary(primaryConnection = rst.getPrimary());
+    // TODO SERVER-109457 Try a scenario where you force election here so that the restarted
+    // secondary could become primary while another active secondary has not restarted yet. This
+    // will probably require adding a third node.
 
-    // Upgrade the primaries.
-    rst.upgradeSet({...latestVersion});
+    // Restart the primaries.
+    // Create a copy of the options since the callee may modify them.
+    rst.upgradeSet({...restartVersion, ...copyJSON(restartNodeOptions)});
     primaryConnection = rst.getPrimary();
+    afterPrimariesHaveRestarted(primaryConnection);
 
-    whenBinariesAreLatestAndFCVIsLastLTS(primaryConnection);
+    if (afterFCVBump) {
+        const getAdminDB = () => primaryConnection.getDB("admin");
 
-    // Upgrade the FCV.
-    assert.commandWorked(
-        getAdminDB().runCommand({setFeatureCompatibilityVersion: latestFCV, confirm: true}));
+        // Upgrade the FCV.
+        assert.commandWorked(
+            getAdminDB().runCommand({setFeatureCompatibilityVersion: latestFCV, confirm: true}));
 
-    whenFullyUpgraded(primaryConnection);
+        afterFCVBump(primaryConnection);
 
-    // Downgrade FCV without restarting.
-    assert.commandWorked(
-        getAdminDB().runCommand({setFeatureCompatibilityVersion: lastLTSFCV, confirm: true}));
+        // Downgrade FCV without restarting.
+        assert.commandWorked(
+            getAdminDB().runCommand({setFeatureCompatibilityVersion: lastLTSFCV, confirm: true}));
 
-    whenBinariesAreLatestAndFCVIsLastLTS(primaryConnection);
+        afterPrimariesHaveRestarted(primaryConnection);
+    }
 
     rst.stopSet();
 }
+
+/**
+ * Simulates the upgrade procedure on a replica set from the "last-lts" version to the "latest"
+ * version.
+ */
+export function testPerformUpgradeReplSet({
+    startingNodeOptions = {},
+    upgradeNodeOptions = {},
+    setupFn,
+    whenFullyDowngraded,
+    whenSecondariesAreLatestBinary,
+    whenBinariesAreLatestAndFCVIsLastLTS,
+    whenFullyUpgraded,
+}) {
+    testPerformReplSetRollingRestart({
+        startingVersion: lastLTSVersion,
+        restartVersion: latestVersion,
+        startingNodeOptions,
+        restartNodeOptions: upgradeNodeOptions,
+        setupFn,
+        beforeRestart: whenFullyDowngraded,
+        afterSecondariesHaveRestarted: whenSecondariesAreLatestBinary,
+        afterPrimariesHaveRestarted: whenBinariesAreLatestAndFCVIsLastLTS,
+        afterFCVBump: whenFullyUpgraded,
+    });
+}

jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js

@@ -1,8 +1,115 @@
 import "jstests/multiVersion/libs/multi_cluster.js";
 
+import {copyJSON} from "jstests/libs/json_utils.js";
 import {ShardingTest} from "jstests/libs/shardingtest.js";
 
-export function testPerformUpgradeDowngradeSharded({
+const latestVersion = {
+    binVersion: "latest"
+};
+const lastLTSVersion = {
+    binVersion: "last-lts"
+};
+
+/**
+ * Simulates a rolling restart of a sharded cluster and calls assertion callbacks at each step
+ * along the way. The procedure works like this:
+ *   1. Spin up a 2-shard sharded cluster with all nodes on startingVersion and startingNodeOptions.
+ *   2. Call the setupFn() and beforeRestart() assertions.
+ *   3. Restart the config node with restartVersion and restartNodeOptions.
+ *   4. Call the afterConfigHasRestarted() assertions.
+ *   5. Restart the secondary shard (with restartVersion and restartNodeOptions).
+ *   6. Call the afterSecondaryShardHasRestarted() assertions.
+ *   7. Restart the primary shard (with restartVersion and restartNodeOptions).
+ *   8. Call the afterPrimaryShardHasRestarted() assertions.
+ *   7. Restart the mongos (with restartVersion and restartNodeOptions).
+ *   8. Call the afterMongosHasRestarted() assertions.
+ *   9. [if afterFCVBump is non-null] Upgrade the FCV.
+ *   10. [if afterFCVBump is non-null] Call the afterFCVBump() assertions.
+ *   11. [if afterFCVBump is non-null] Downgrade the FCV.
+ *   12. [if afterFCVBump is non-null] Call the afterMongosHasRestarted() assertions again.
+ */
+export function testPerformShardedClusterRollingRestart({
+    startingVersion = latestVersion,
+    restartVersion = latestVersion,
+    startingNodeOptions = {},
+    restartNodeOptions = {},
+    setupFn,
+    beforeRestart,
+    afterConfigHasRestarted,
+    afterSecondaryShardHasRestarted,
+    afterPrimaryShardHasRestarted,
+    afterMongosHasRestarted,
+    afterFCVBump = null,
+}) {
+    // Create a copy of the options each time they're passed since the callees may modify them.
+    const st = new ShardingTest({
+        shards: 2,
+        rs: {nodes: 2},
+        mongos: 1,
+        config: 1,
+        other: {
+            mongosOptions: {...startingVersion, ...copyJSON(startingNodeOptions)},
+            configOptions: {...startingVersion, ...copyJSON(startingNodeOptions)},
+            rsOptions: {...startingVersion, ...copyJSON(startingNodeOptions)}
+        },
+    });
+    st.configRS.awaitReplication();
+
+    setupFn(st.s, st);
+
+    beforeRestart(st.s);
+
+    const justWaitForStable =
+        {upgradeShards: false, upgradeMongos: false, upgradeConfigs: false, waitUntilStable: true};
+
+    // Upgrade the configs.
+    st.upgradeCluster(restartVersion.binVersion,
+                      {...justWaitForStable, upgradeConfigs: true},
+                      copyJSON(restartNodeOptions));
+    afterConfigHasRestarted(st.s);
+
+    // Upgrade the secondary shard.
+    st.upgradeCluster(restartVersion.binVersion,
+                      {...justWaitForStable, upgradeOneShard: st.rs1},
+                      copyJSON(restartNodeOptions));
+    afterSecondaryShardHasRestarted(st.s);
+
+    // Upgrade the rest of the cluster.
+    st.upgradeCluster(restartVersion.binVersion,
+                      {...justWaitForStable, upgradeShards: true},
+                      copyJSON(restartNodeOptions));
+    afterPrimaryShardHasRestarted(st.s);
+
+    // Upgrade mongos.
+    st.upgradeCluster(restartVersion.binVersion,
+                      {...justWaitForStable, upgradeMongos: true},
+                      copyJSON(restartNodeOptions));
+    afterMongosHasRestarted(st.s);
+
+    if (afterFCVBump) {
+        // Upgrade the FCV.
+        assert.commandWorked(st.s.getDB(jsTestName()).adminCommand({
+            setFeatureCompatibilityVersion: latestFCV,
+            confirm: true
+        }));
+
+        afterFCVBump(st.s);
+
+        // Downgrade FCV without restarting.
+        assert.commandWorked(st.s.getDB(jsTestName()).adminCommand({
+            setFeatureCompatibilityVersion: lastLTSFCV,
+            confirm: true
+        }));
+
+        afterMongosHasRestarted(st.s);
+    }
+
+    st.stop();
+}
+
+export function testPerformUpgradeSharded({
+    startingNodeOptions = {},
+    upgradeNodeOptions = {},
     setupFn,
     whenFullyDowngraded,
     whenOnlyConfigIsLatestBinary,
@@ -11,61 +118,17 @@ export function testPerformUpgradeDowngradeSharded({
     whenBinariesAreLatestAndFCVIsLastLTS,
     whenFullyUpgraded
 }) {
-    const st = new ShardingTest({
-        shards: 2,
-        rs: {nodes: 2},
-        mongos: 1,
-        config: 1,
-        other: {
-            mongosOptions: {binVersion: "last-lts"},
-            configOptions: {binVersion: "last-lts"},
-            rsOptions: {binVersion: "last-lts"}
-        },
+    testPerformShardedClusterRollingRestart({
+        startingVersion: lastLTSVersion,
+        restartVersion: latestVersion,
+        startingNodeOptions,
+        restartNodeOptions: upgradeNodeOptions,
+        setupFn,
+        beforeRestart: whenFullyDowngraded,
+        afterConfigHasRestarted: whenOnlyConfigIsLatestBinary,
+        afterSecondaryShardHasRestarted: whenSecondariesAndConfigAreLatestBinary,
+        afterPrimaryShardHasRestarted: whenMongosBinaryIsLastLTS,
+        afterMongosHasRestarted: whenBinariesAreLatestAndFCVIsLastLTS,
+        afterFCVBump: whenFullyUpgraded,
     });
-    st.configRS.awaitReplication();
-
-    setupFn(st.s, st);
-
-    whenFullyDowngraded(st.s);
-
-    const justWaitForStable =
-        {upgradeShards: false, upgradeMongos: false, upgradeConfigs: false, waitUntilStable: true};
-
-    // Upgrade the configs.
-    st.upgradeCluster('latest', {...justWaitForStable, upgradeConfigs: true});
-
-    whenOnlyConfigIsLatestBinary(st.s);
-
-    // Upgrade the secondary shard.
-    st.upgradeCluster('latest', {...justWaitForStable, upgradeOneShard: st.rs1});
-
-    whenSecondariesAndConfigAreLatestBinary(st.s);
-
-    // Upgrade the rest of the cluster.
-    st.upgradeCluster('latest', {...justWaitForStable, upgradeShards: true});
-
-    whenMongosBinaryIsLastLTS(st.s);
-
-    // Upgrade mongos.
-    st.upgradeCluster('latest', {...justWaitForStable, upgradeMongos: true});
-
-    whenBinariesAreLatestAndFCVIsLastLTS(st.s);
-
-    // Upgrade the FCV.
-    assert.commandWorked(st.s.getDB(jsTestName()).adminCommand({
-        setFeatureCompatibilityVersion: latestFCV,
-        confirm: true
-    }));
-
-    whenFullyUpgraded(st.s);
-
-    // Downgrade the FCV without restarting.
-    assert.commandWorked(st.s.getDB(jsTestName()).adminCommand({
-        setFeatureCompatibilityVersion: lastLTSFCV,
-        confirm: true
-    }));
-
-    whenBinariesAreLatestAndFCVIsLastLTS(st.s);
-
-    st.stop();
 }

jstests/multiVersion/libs/multi_cluster.js

@@ -4,15 +4,16 @@
 
 import "jstests/multiVersion/libs/multi_rs.js";
 
+import {copyJSON} from "jstests/libs/json_utils.js";
 import {ShardingTest} from "jstests/libs/shardingtest.js";
 import {awaitRSClientHosts} from "jstests/replsets/rslib.js";
 
 /**
- * Restarts the specified binaries in options with the specified binVersion.
+ * Restarts the specified binaries in upgradeOptions with the specified binVersion.
  * Note: this does not perform any upgrade operations.
  *
  * @param binVersion {string}
- * @param options {Object} format:
+ * @param upgradeOptions {Object} format:
  *
  * {
  *     upgradeShards: <bool>, // defaults to true
@@ -24,20 +25,20 @@ import {awaitRSClientHosts} from "jstests/replsets/rslib.js";
  *                                 certain tests will likely want a stable cluster after upgrading.
  * }
  */
-ShardingTest.prototype.upgradeCluster = function(binVersion, options) {
-    options = options || {};
-    if (options.upgradeShards == undefined)
-        options.upgradeShards = true;
-    if (options.upgradeOneShard == undefined)
-        options.upgradeOneShard = false;
-    if (options.upgradeConfigs == undefined)
-        options.upgradeConfigs = true;
-    if (options.upgradeMongos == undefined)
-        options.upgradeMongos = true;
-    if (options.waitUntilStable == undefined)
-        options.waitUntilStable = false;
+ShardingTest.prototype.upgradeCluster = function(binVersion, upgradeOptions, nodeOptions = {}) {
+    upgradeOptions = upgradeOptions || {};
+    if (upgradeOptions.upgradeShards == undefined)
+        upgradeOptions.upgradeShards = true;
+    if (upgradeOptions.upgradeOneShard == undefined)
+        upgradeOptions.upgradeOneShard = false;
+    if (upgradeOptions.upgradeConfigs == undefined)
+        upgradeOptions.upgradeConfigs = true;
+    if (upgradeOptions.upgradeMongos == undefined)
+        upgradeOptions.upgradeMongos = true;
+    if (upgradeOptions.waitUntilStable == undefined)
+        upgradeOptions.waitUntilStable = false;
 
-    if (options.upgradeConfigs) {
+    if (upgradeOptions.upgradeConfigs) {
         // Upgrade config servers
         const numConfigs = this.configRS.nodes.length;
 
@@ -45,27 +46,37 @@ ShardingTest.prototype.upgradeCluster = function(binVersion, options) {
             var configSvr = this.configRS.nodes[i];
 
             MongoRunner.stopMongod(configSvr);
-            configSvr = MongoRunner.runMongod(
-                {restart: configSvr, binVersion: binVersion, appendOptions: true});
+            // Must copy the nodeOptions since they are modified by callee.
+            const configSrvOptions = copyJSON(nodeOptions);
+            configSvr = MongoRunner.runMongod({
+                restart: configSvr,
+                binVersion: binVersion,
+                appendOptions: true,
+                ...configSrvOptions
+            });
 
             this["config" + i] = this["c" + i] = this.configRS.nodes[i] = configSvr;
         }
     }
 
-    if (options.upgradeShards) {
+    if (upgradeOptions.upgradeShards) {
         // Upgrade shards
         this._rs.forEach((rs) => {
-            rs.test.upgradeSet({binVersion: binVersion});
+            // Must copy the nodeOptions since they are modified by callee.
+            const replSetOptions = copyJSON(nodeOptions);
+            rs.test.upgradeSet({binVersion: binVersion, ...replSetOptions});
         });
     }
 
-    if (options.upgradeOneShard) {
+    if (upgradeOptions.upgradeOneShard) {
         // Upgrade one shard.
-        let rs = options.upgradeOneShard;
-        rs.upgradeSet({binVersion: binVersion});
+        let rs = upgradeOptions.upgradeOneShard;
+        // Must copy the nodeOptions since they are modified by callee.
+        const replSetOptions = copyJSON(nodeOptions);
+        rs.upgradeSet({binVersion: binVersion, ...replSetOptions});
     }
 
-    if (options.upgradeMongos) {
+    if (upgradeOptions.upgradeMongos) {
         // Upgrade all mongos hosts if specified
         var numMongoses = this._mongos.length;
 
@@ -73,8 +84,10 @@ ShardingTest.prototype.upgradeCluster = function(binVersion, options) {
             var mongos = this._mongos[i];
             MongoRunner.stopMongos(mongos);
 
+            // Must copy the nodeOptions since they are modified by callee.
+            const mongosOptions = copyJSON(nodeOptions);
             mongos = MongoRunner.runMongos(
-                {restart: mongos, binVersion: binVersion, appendOptions: true});
+                {restart: mongos, binVersion: binVersion, appendOptions: true, ...mongosOptions});
 
             this["s" + i] = this._mongos[i] = mongos;
             if (i == 0)
@@ -85,25 +98,25 @@ ShardingTest.prototype.upgradeCluster = function(binVersion, options) {
         this.admin = this.s.getDB("admin");
     }
 
-    if (options.waitUntilStable) {
+    if (upgradeOptions.waitUntilStable) {
         this.waitUntilStable();
     }
 };
 
-ShardingTest.prototype.downgradeCluster = function(binVersion, options) {
-    options = options || {};
-    if (options.downgradeShards == undefined)
-        options.downgradeShards = true;
-    if (options.downgradeOneShard == undefined)
-        options.downgradeOneShard = false;
-    if (options.downgradeConfigs == undefined)
-        options.downgradeConfigs = true;
-    if (options.downgradeMongos == undefined)
-        options.downgradeMongos = true;
-    if (options.waitUntilStable == undefined)
-        options.waitUntilStable = false;
+ShardingTest.prototype.downgradeCluster = function(binVersion, downgradeOptions, nodeOptions = {}) {
+    downgradeOptions = downgradeOptions || {};
+    if (downgradeOptions.downgradeShards == undefined)
+        downgradeOptions.downgradeShards = true;
+    if (downgradeOptions.downgradeOneShard == undefined)
+        downgradeOptions.downgradeOneShard = false;
+    if (downgradeOptions.downgradeConfigs == undefined)
+        downgradeOptions.downgradeConfigs = true;
+    if (downgradeOptions.downgradeMongos == undefined)
+        downgradeOptions.downgradeMongos = true;
+    if (downgradeOptions.waitUntilStable == undefined)
+        downgradeOptions.waitUntilStable = false;
 
-    if (options.downgradeMongos) {
+    if (downgradeOptions.downgradeMongos) {
         // Downgrade all mongos hosts if specified
         var numMongoses = this._mongos.length;
 
@@ -111,8 +124,10 @@ ShardingTest.prototype.downgradeCluster = function(binVersion, options) {
             var mongos = this._mongos[i];
             MongoRunner.stopMongos(mongos);
 
+            // Must copy the nodeOptions since they are modified by callee.
+            const mongosOptions = copyJSON(nodeOptions);
             mongos = MongoRunner.runMongos(
-                {restart: mongos, binVersion: binVersion, appendOptions: true});
+                {restart: mongos, binVersion: binVersion, appendOptions: true, ...mongosOptions});
 
             this["s" + i] = this._mongos[i] = mongos;
             if (i == 0)
@@ -123,35 +138,46 @@ ShardingTest.prototype.downgradeCluster = function(binVersion, options) {
         this.admin = this.s.getDB("admin");
     }
 
-    if (options.downgradeShards) {
+    if (downgradeOptions.downgradeShards) {
         // Downgrade shards
         this._rs.forEach((rs) => {
-            rs.test.upgradeSet({binVersion: binVersion});
+            // Must copy the nodeOptions since they are modified by callee.
+            const replSetOptions = copyJSON(nodeOptions);
+            rs.test.upgradeSet({binVersion: binVersion, ...replSetOptions});
         });
     }
 
-    if (options.downgradeOneShard) {
+    if (downgradeOptions.downgradeOneShard) {
+        // Must copy the nodeOptions since they are modified by callee.
+        const replSetOptions = copyJSON(nodeOptions);
+
         // Downgrade one shard.
-        let rs = options.downgradeOneShard;
-        rs.upgradeSet({binVersion: binVersion});
+        let rs = downgradeOptions.downgradeOneShard;
+        rs.upgradeSet({binVersion: binVersion, ...replSetOptions});
     }
 
-    if (options.downgradeConfigs) {
+    if (downgradeOptions.downgradeConfigs) {
         // Downgrade config servers
         const numConfigs = this.configRS.nodes.length;
 
         for (var i = 0; i < numConfigs; i++) {
             var configSvr = this.configRS.nodes[i];
 
+            // Must copy the nodeOptions since they are modified by callee.
+            const configSvrOptions = copyJSON(nodeOptions);
             MongoRunner.stopMongod(configSvr);
-            configSvr = MongoRunner.runMongod(
-                {restart: configSvr, binVersion: binVersion, appendOptions: true});
+            configSvr = MongoRunner.runMongod({
+                restart: configSvr,
+                binVersion: binVersion,
+                appendOptions: true,
+                ...configSvrOptions
+            });
 
             this["config" + i] = this["c" + i] = this.configRS.nodes[i] = configSvr;
         }
     }
 
-    if (options.waitUntilStable) {
+    if (downgradeOptions.waitUntilStable) {
         this.waitUntilStable();
     }
 };

jstests/multiVersion/targetedTestsLastLtsFeatures/query-execution-classic/bindata_convert_numeric_view_upgrade_downgrade.js

@@ -3,11 +3,9 @@
  */
 
 import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
 import {
-    testPerformUpgradeDowngradeReplSet
-} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
-import {
-    testPerformUpgradeDowngradeSharded
+    testPerformUpgradeSharded
 } from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
 
 const collectionName = "coll";
@@ -255,7 +253,7 @@ function assertQueriesOnViewsFail(primaryConnection) {
     );
 }
 
-testPerformUpgradeDowngradeReplSet({
+testPerformUpgradeReplSet({
     setupFn: setupCollection,
     whenFullyDowngraded: assertViewCanBeCreatedButNotExecuted,
     whenSecondariesAreLatestBinary: assertViewCanBeCreatedButNotExecuted,
@@ -263,7 +261,7 @@ testPerformUpgradeDowngradeReplSet({
     whenFullyUpgraded: assertViewCanBeCreatedAndExecuted,
 });
 
-testPerformUpgradeDowngradeSharded({
+testPerformUpgradeSharded({
     setupFn: setupCollection,
     whenFullyDowngraded: assertViewCanBeCreatedButNotExecuted,
     whenOnlyConfigIsLatestBinary: assertViewCanBeCreatedButNotExecuted,

jstests/multiVersion/targetedTestsLastLtsFeatures/query-execution-query-settings/query_settings_upgrade_downgrade.js

@@ -10,11 +10,9 @@ import {describe, it} from "jstests/libs/mochalite.js";
 import {QuerySettingsUtils} from "jstests/libs/query/query_settings_utils.js";
 import {ReplSetTest} from "jstests/libs/replsettest.js";
 import {ShardingTest} from "jstests/libs/shardingtest.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
 import {
-    testPerformUpgradeDowngradeReplSet
-} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
-import {
-    testPerformUpgradeDowngradeSharded
+    testPerformUpgradeSharded
 } from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
 
 const dbName = "test";
@@ -68,7 +66,7 @@ describe("QuerySettings", function() {
                 assertQueryShapeConfigurations(true);
 
             it("in replica set", function() {
-                testPerformUpgradeDowngradeReplSet({
+                testPerformUpgradeReplSet({
                     setupFn,
                     whenFullyDowngraded:
                         assertQueryShapeConfigurationsWithEmptyRepresentativeQueries,
@@ -82,7 +80,7 @@ describe("QuerySettings", function() {
             });
 
             it("in sharded cluster", function() {
-                testPerformUpgradeDowngradeSharded({
+                testPerformUpgradeSharded({
                     setupFn,
                     whenFullyDowngraded:
                         assertQueryShapeConfigurationsWithEmptyRepresentativeQueries,

jstests/multiVersion/targetedTestsLastLtsFeatures/query-integration-search/rank_fusion_upgrade_downgrade.js

@@ -2,11 +2,9 @@
  * Verifies that $rankFusion behaves correctly in FCV upgrade/downgrade scenarios.
  */
 import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
 import {
-    testPerformUpgradeDowngradeReplSet
-} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
-import {
-    testPerformUpgradeDowngradeSharded
+    testPerformUpgradeSharded
 } from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
 
 const collName = jsTestName();
@@ -104,7 +102,7 @@ function assertRankFusionCompletelyAccepted(primaryConn) {
         {aggregate: viewName, pipeline: rankFusionPipelineWithScoreDetails, cursor: {}}));
 }
 
-testPerformUpgradeDowngradeReplSet({
+testPerformUpgradeReplSet({
     setupFn: setupCollection,
     whenFullyDowngraded: assertRankFusionCompletelyRejected,
     whenSecondariesAreLatestBinary: assertRankFusionCompletelyRejected,
@@ -112,7 +110,7 @@ testPerformUpgradeDowngradeReplSet({
     whenFullyUpgraded: assertRankFusionCompletelyAccepted,
 });
 
-testPerformUpgradeDowngradeSharded({
+testPerformUpgradeSharded({
     setupFn: setupCollection,
     whenFullyDowngraded: assertRankFusionCompletelyRejected,
     whenOnlyConfigIsLatestBinary: assertRankFusionCompletelyRejected,

jstests/multiVersion/targetedTestsLastLtsFeatures/query-integration-search/score_fusion_upgrade_downgrade.js

@@ -2,11 +2,9 @@
  * Verifies that $scoreFusion behaves correctly in FCV upgrade/downgrade scenarios.
  */
 import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
 import {
-    testPerformUpgradeDowngradeReplSet
-} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
-import {
-    testPerformUpgradeDowngradeSharded
+    testPerformUpgradeSharded
 } from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
 
 const collName = jsTestName();
@@ -186,7 +184,7 @@ function assertScoreFusionCompletelyAccepted(primaryConn) {
         db.runCommand({aggregate: collName, pipeline: scorePipelineWithScoreDetails, cursor: {}}));
 }
 
-testPerformUpgradeDowngradeReplSet({
+testPerformUpgradeReplSet({
     setupFn: setupCollection,
     whenFullyDowngraded: assertScoreFusionCompletelyRejected,
     whenSecondariesAreLatestBinary: assertScoreFusionCompletelyRejected,
@@ -194,7 +192,7 @@ testPerformUpgradeDowngradeReplSet({
     whenFullyUpgraded: assertScoreFusionCompletelyAccepted,
 });
 
-testPerformUpgradeDowngradeSharded({
+testPerformUpgradeSharded({
     setupFn: setupCollection,
     whenFullyDowngraded: assertScoreFusionCompletelyRejected,
     whenOnlyConfigIsLatestBinary: assertScoreFusionCompletelyRejected,

jstests/multiVersion/targetedTestsLastLtsFeatures/query-optimization/mql_js_engine_gap_upgrade_downgrade.js

@@ -8,11 +8,9 @@
  */
 
 import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+import {testPerformUpgradeReplSet} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
 import {
-    testPerformUpgradeDowngradeReplSet
-} from "jstests/multiVersion/libs/mixed_version_fixture_test.js";
-import {
-    testPerformUpgradeDowngradeSharded
+    testPerformUpgradeSharded
 } from "jstests/multiVersion/libs/mixed_version_sharded_fixture_test.js";
 
 const collectionName = "coll";
@@ -194,7 +192,7 @@ function assertCreateOrEvaluateViewsWithNewFeaturesFail(primaryConnection) {
     }
 }
 
-testPerformUpgradeDowngradeReplSet({
+testPerformUpgradeReplSet({
     setupFn: setupCollection,
     whenFullyDowngraded: assertCreateOrEvaluateViewsWithNewFeaturesFail,
     whenSecondariesAreLatestBinary: assertCreateOrEvaluateViewsWithNewFeaturesFail,
@@ -202,7 +200,7 @@ testPerformUpgradeDowngradeReplSet({
     whenFullyUpgraded: assertCreateAndEvaluateViewsWithNewFeaturesPass,
 });
 
-testPerformUpgradeDowngradeSharded({
+testPerformUpgradeSharded({
     setupFn: setupCollection,
     whenFullyDowngraded: assertCreateOrEvaluateViewsWithNewFeaturesFail,
     whenOnlyConfigIsLatestBinary: assertCreateOrEvaluateViewsWithNewFeaturesFail,

src/mongo/db/extension/test_examples/BUILD.bazel

@@ -10,12 +10,21 @@ package(default_visibility = ["//visibility:public"])
     )
     for extension_name in [
         "foo",
+        "foo_v2",
         "bar",
         # TODO SERVER-109108: Remove this entry when the buzz extension is no longer needed.
         "buzz",
     ]
 ]
 
+# "foo_v2" is used for testing upgrade scenarios as the upgrade of "foo" from above. We cannot use
+# the "mongo_extension" suffix since loading this extension in passthroughs with "foo" V1 should
+# cause duplicate stage errors.
+mongo_cc_extension_shared_library(
+    name = "foo_extension_v2",
+    srcs = ["foo_v2.cpp"],
+)
+
 # "vector_search" is used to test that we can override the existing $vectorSearch implementation.
 # It must not have the _mongo_extension suffix so that it doesn't get loaded in the
 # Extensions-enabled variant since that would break real $vectorSearch tests.

src/mongo/db/extension/test_examples/foo_v2.cpp

@@ -0,0 +1,69 @@
+/**
+ *    Copyright (C) 2025-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/bson/bsonobj.h"
+#include "mongo/db/extension/sdk/aggregation_stage.h"
+#include "mongo/db/extension/sdk/extension_factory.h"
+
+namespace sdk = mongo::extension::sdk;
+
+/**
+ * $testFoo is a no-op stage.
+ *
+ * This file is identical to foo.cpp except this stage does _not_ fail parsing if the
+ * stage definition is empty. This is used for extenison upgrade/downgrade testing.
+ */
+class TestFooLogicalStage : public sdk::LogicalAggregationStage {};
+
+class TestFooStageDescriptor : public sdk::AggregationStageDescriptor {
+public:
+    static inline const std::string kStageName = "$testFoo";
+
+    TestFooStageDescriptor()
+        : sdk::AggregationStageDescriptor(kStageName, MongoExtensionAggregationStageType::kNoOp) {}
+
+    std::unique_ptr<sdk::LogicalAggregationStage> parse(mongo::BSONObj stageBson) const override {
+        // Unlike foo.cpp, this will NOT fail to parse if the stage definition is not empty. Any/all
+        // fields are just quietly ignored.
+        uassert(10624201,
+                "Failed to parse " + kStageName + ", expected object",
+                stageBson.hasField(kStageName) && stageBson.getField(kStageName).isABSONObj());
+
+        return std::make_unique<TestFooLogicalStage>();
+    }
+};
+
+class FooExtension : public sdk::Extension {
+public:
+    void initialize(const sdk::HostPortalHandle& portal) override {
+        _registerStage<TestFooStageDescriptor>(portal);
+    }
+};
+
+REGISTER_EXTENSION(FooExtension);