diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f89182a2538..6239d82eea3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -152,6 +152,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot /buildscripts/resmokeconfig/matrix_suites/mappings/**/core_min_batch_repeat_queries_multiplan_single_solutions_ese_gsm.yml @10gen/query-optimization @svc-auto-approve-bot /buildscripts/resmokeconfig/matrix_suites/mappings/**/core_repeat_queries.yml @10gen/query-optimization @svc-auto-approve-bot /buildscripts/resmokeconfig/matrix_suites/mappings/**/replica_sets_jscore_pqs* @10gen/query-execution-query-settings @svc-auto-approve-bot +/buildscripts/resmokeconfig/matrix_suites/mappings/**/shard* @10gen/server-catalog-and-routing @svc-auto-approve-bot /buildscripts/resmokeconfig/matrix_suites/mappings/**/sharded_collections_pqs* @10gen/query-execution-query-settings @svc-auto-approve-bot /buildscripts/resmokeconfig/matrix_suites/mappings/**/sharded_collections_query_shape_hash_stability* @10gen/query-execution-query-settings @10gen/query-integration-observability @svc-auto-approve-bot /buildscripts/resmokeconfig/matrix_suites/mappings/**/*legacy_timeseries_no_rawdata* @10gen/server-catalog-and-routing-shard-catalog @svc-auto-approve-bot diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml index 0774a9d3d31..55125d22fe0 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -29,7 +29,7 @@ executor: mongos_options: set_parameters: enableTestCommands: 1 - num_mongos: 3 + num_mongos: 1 num_rs_nodes_per_shard: 2 num_shards: 2 old_bin_version: last_continuous diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml index e307767423a..77e25c80a8e 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml @@ -62,6 +62,7 @@ executor: mongos_options: set_parameters: enableTestCommands: 1 + num_mongos: 1 num_rs_nodes_per_shard: 2 num_shards: 2 old_bin_version: last_continuous diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml index 25143f1f4b4..97e0a40c8a8 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -27,7 +27,7 @@ executor: mongos_options: set_parameters: enableTestCommands: 1 - num_mongos: 2 + num_mongos: 1 num_rs_nodes_per_shard: 2 num_shards: 2 old_bin_version: last_continuous diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml index ede80c2733d..5031f83c272 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -60,7 +60,7 @@ executor: mongos_options: set_parameters: enableTestCommands: 1 - num_mongos: 2 + num_mongos: 1 num_rs_nodes_per_shard: 3 num_shards: 2 old_bin_version: last_continuous diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/OWNERS.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/OWNERS.yml index 51d1fd12b15..a8a133ca1f7 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/OWNERS.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/OWNERS.yml @@ -40,6 +40,9 @@ filters: - "replica_sets_jscore_pqs*": approvers: - 10gen/query-execution-query-settings + - "shard*": + approvers: + - 10gen/server-catalog-and-routing - "sharded_collections_pqs*": approvers: - 10gen/query-execution-query-settings diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml index ed8be02c640..508ced5f52d 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_collections_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -3,3 +3,5 @@ overrides: - "multiversion.sharded_collections_temporarily_disable_due_to_fcv_upgrade" - "multiversion.sharded_fixture_last_continuous_new_old_old_new" - "multiversion.sharding_multiversion_mongos_testdata_last_continuous" + # TODO (SERVER-117693) - Remove this override once SERVER-114640 is backported to 8.2 + - "multiversion.sharding_temporarily_enforce_1_mongos" diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml index a3fc8dfb5be..05d02519f3c 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/sharded_retryable_writes_downgrade_last_continuous_new_old_old_new.yml @@ -1,3 +1,5 @@ base_suite: sharded_retryable_writes_downgrade overrides: - "multiversion.sharded_fixture_last_continuous_new_old_old_new" + # TODO (SERVER-117693) - Remove this override once SERVER-114640 is backported to 8.2 + - "multiversion.sharding_temporarily_enforce_1_mongos" diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml index 673a3b02b6b..b42d21464cd 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -3,3 +3,5 @@ overrides: - "multiversion.sharding_temporarily_disable_due_to_fcv_upgrade" - "multiversion.sharded_fixture_last_continuous_new_old_old_new" - "multiversion.sharding_multiversion_mongos_testdata_last_continuous" + # TODO (SERVER-117693) - Remove this override once SERVER-114640 is backported to 8.2 + - "multiversion.sharding_temporarily_enforce_1_mongos" diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml index b945409f021..12994f8f1e4 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -5,6 +5,8 @@ overrides: - "kill_primary.sharding_reconfig_archive" - "kill_primary.sharding_shell_options" - "kill_primary.all_nodes_can_be_primary" + # TODO (SERVER-117693) - Remove this override once SERVER-114640 is backported to 8.2 + - "multiversion.sharding_temporarily_enforce_1_mongos" extends: - "replica_sets_stepdown_selector.kill_primary_jscore_passthrough_exclude_with_any_tags" diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml index 67a79a24dc5..2c63007d275 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml @@ -178,6 +178,12 @@ - requires_profiling - DISABLED_TEMPORARILY_DUE_TO_FCV_UPGRADE +- name: sharding_temporarily_enforce_1_mongos + value: + executor: + fixture: + num_mongos: 1 + - name: multiversion_future_git_tag_exclude_files value: exclude_with_any_tags: diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js index 2dd93ccd188..e3327e16b1a 100644 --- a/jstests/auth/lib/commands_lib.js +++ b/jstests/auth/lib/commands_lib.js @@ -1648,6 +1648,14 @@ export const authCommandsLib = { }, { testname: "aggregate_merge_insert_documents", + setup: function (db) { + assert.commandWorked(db.getSiblingDB(firstDbName).foo.insert({})); + assert.commandWorked(db.getSiblingDB(secondDbName).foo.insert({})); + }, + teardown: function (db) { + assert.commandWorked(db.getSiblingDB(firstDbName).dropDatabase()); + assert.commandWorked(db.getSiblingDB(secondDbName).dropDatabase()); + }, command: function (state, args) { return { aggregate: "foo", @@ -1724,6 +1732,14 @@ export const authCommandsLib = { }, { testname: "aggregate_merge_replace_documents", + setup: function (db) { + assert.commandWorked(db.getSiblingDB(firstDbName).foo.insert({})); + assert.commandWorked(db.getSiblingDB(secondDbName).foo.insert({})); + }, + teardown: function (db) { + assert.commandWorked(db.getSiblingDB(firstDbName).dropDatabase()); + assert.commandWorked(db.getSiblingDB(secondDbName).dropDatabase()); + }, command: function (state, args) { return { aggregate: "foo", diff --git a/jstests/core/ddl/drop_collection.js b/jstests/core/ddl/drop_collection.js index 8c40bf17a8f..3a2a91b7954 100644 --- a/jstests/core/ddl/drop_collection.js +++ b/jstests/core/ddl/drop_collection.js @@ -13,9 +13,6 @@ import {FixtureHelpers} from "jstests/libs/fixture_helpers.js"; -// TODO SERVER-114640 Remove this line once aggregate won't report NamespaceNotFound if the db gets dropped. -TestData.pinToSingleMongos = true; - let _collCounter = 0; function getNewColl(db) { const collNamePrefix = jsTestName() + "_coll_"; diff --git a/jstests/core/timeseries/ddl/timeseries_list_catalog.js b/jstests/core/timeseries/ddl/timeseries_list_catalog.js index 2015e13b2e8..404138d9e92 100644 --- a/jstests/core/timeseries/ddl/timeseries_list_catalog.js +++ b/jstests/core/timeseries/ddl/timeseries_list_catalog.js @@ -20,7 +20,7 @@ import { getTimeseriesCollForDDLOps, } from "jstests/core/timeseries/libs/viewless_timeseries_util.js"; -// TODO SERVER-114640 Remove this line once aggregate won't report NamespaceNotFound if the db gets dropped. +// TODO (SERVER-117871) remove once the topologyTime is guaranteed to be gossiped out to all routers. TestData.pinToSingleMongos = true; assert.commandWorked(db.dropDatabase()); diff --git a/jstests/core_sharding/resharding/reshard_collection_atlas_log_ingestion.js b/jstests/core_sharding/resharding/reshard_collection_atlas_log_ingestion.js index 9c6bd61048a..a36ac4f0ac0 100644 --- a/jstests/core_sharding/resharding/reshard_collection_atlas_log_ingestion.js +++ b/jstests/core_sharding/resharding/reshard_collection_atlas_log_ingestion.js @@ -17,9 +17,6 @@ * ] */ -// TODO SERVER-114640 Remove this line once aggregate won't report NamespaceNotFound if the db gets dropped. -TestData.pinToSingleMongos = true; - import {withSkipRetryOnNetworkError} from "jstests/concurrency/fsm_workload_helpers/stepdown_suite_helpers.js"; import {DiscoverTopology} from "jstests/libs/discover_topology.js"; import {configureFailPointForRS} from "jstests/libs/fail_point_util.js"; diff --git a/src/mongo/db/pipeline/document_source_merge.h b/src/mongo/db/pipeline/document_source_merge.h index e171bebd660..a59c5182439 100644 --- a/src/mongo/db/pipeline/document_source_merge.h +++ b/src/mongo/db/pipeline/document_source_merge.h @@ -148,6 +148,10 @@ public: return true; } + bool isMergeStage() const override { + return true; + } + std::unique_ptr getStageParams() const override { return std::make_unique(_originalBson); } diff --git a/src/mongo/db/pipeline/lite_parsed_document_source.h b/src/mongo/db/pipeline/lite_parsed_document_source.h index 443b7a6a88d..af530764489 100644 --- a/src/mongo/db/pipeline/lite_parsed_document_source.h +++ b/src/mongo/db/pipeline/lite_parsed_document_source.h @@ -463,6 +463,13 @@ public: return false; } + /** + * Returns true if this is the merge stage. + */ + virtual bool isMergeStage() const { + return false; + } + /** * Returns true if this stage is an initial source and should run just once on the entire * cluster. diff --git a/src/mongo/db/pipeline/lite_parsed_pipeline.h b/src/mongo/db/pipeline/lite_parsed_pipeline.h index f54fac102e6..a88af431f6d 100644 --- a/src/mongo/db/pipeline/lite_parsed_pipeline.h +++ b/src/mongo/db/pipeline/lite_parsed_pipeline.h @@ -200,6 +200,10 @@ public: return !_stageSpecs.empty() && _stageSpecs.back()->isWriteStage(); } + bool endsWithMergeStage() const { + return !_stageSpecs.empty() && _stageSpecs.back()->isMergeStage(); + } + /** * Returns true if the pipeline has a $changeStream stage. */ diff --git a/src/mongo/s/query/planner/cluster_aggregate.cpp b/src/mongo/s/query/planner/cluster_aggregate.cpp index dc345d87e0f..e2c72161b5d 100644 --- a/src/mongo/s/query/planner/cluster_aggregate.cpp +++ b/src/mongo/s/query/planner/cluster_aggregate.cpp @@ -1128,27 +1128,28 @@ Status ClusterAggregate::runAggregate( router.createDbImplicitlyOnRoute(); } - // We'll use routerBodyStarted to distinguish whether an error was thrown before or after the - // body function was executed. - bool routerBodyStarted = false; + // We'll use the aggregationStatus to distinguish whether an error was thrown by the + // aggregation command or the refresh loop. + Status aggregationStatus = Status::OK(); auto bodyFn = [&](OperationContext* opCtx, RoutingContext& routingCtx) { - routerBodyStarted = true; - uassertStatusOK(runAggregateImpl(opCtx, - routingCtx, - namespaces, - request, - liteParsedPipeline, - privileges, - boost::none /* resolvedView */, - boost::none /* originalRequest */, - verbosity, - result, - ifrContext)); + aggregationStatus = runAggregateImpl(opCtx, + routingCtx, + namespaces, + request, + liteParsedPipeline, + privileges, + boost::none /* resolvedView */, + boost::none /* originalRequest */, + verbosity, + result, + ifrContext); + + uassertStatusOK(aggregationStatus); return Status::OK(); }; // Route the command and capture the returned status. - Status status = std::invoke([&]() -> Status { + Status finalStatus = std::invoke([&]() -> Status { try { return router.routeWithRoutingContext(comment, bodyFn); } catch (const DBException& ex) { @@ -1156,23 +1157,40 @@ Status ClusterAggregate::runAggregate( } }); - // Error handling for exceptions raised prior to executing the runAggregation operation. - if (!status.isOK() && !routerBodyStarted) { + // Error handling for exceptions raised by the refresh loop. + // We can infer this by comparing the 2 status received: + // - a failed finalStatus might be either an error from the refresh loop or from the the + // aggregate command + // - a failed aggregationStatus can only be from the aggregate command (note this includes + // StaleDb and StaleConfig) + // A refresh error is calculated as follows + // - the finalStatus fails but the aggregation doesn't + // - both finalStatus and aggregationStatus fails, but they are different + bool isRefreshError = + !finalStatus.isOK() && (aggregationStatus.isOK() || aggregationStatus != finalStatus); + if (isRefreshError) { uassert(CollectionUUIDMismatchInfo(request.getDbName(), *request.getCollectionUUID(), std::string{request.getNamespace().coll()}, boost::none), "Database does not exist", - status != ErrorCodes::NamespaceNotFound || !request.getCollectionUUID()); + finalStatus != ErrorCodes::NamespaceNotFound || !request.getCollectionUUID()); if (liteParsedPipeline.startsWithCollStats()) { - uassertStatusOKWithContext(status, + uassertStatusOKWithContext(finalStatus, "Unable to retrieve information for $collStats stage"); } + // $merge is the only stage that requires to report specifically NamespaceNotFound, instead + // of returning an empty batch with status ok. + if (finalStatus == ErrorCodes::NamespaceNotFound && + liteParsedPipeline.endsWithMergeStage()) { + return finalStatus; + } + // Return an empty cursor with the given status. return _parseQueryStatsAndReturnEmptyResult(opCtx, - status, + finalStatus, namespaces, request, liteParsedPipeline, @@ -1182,7 +1200,7 @@ Status ClusterAggregate::runAggregate( result); } - return status; + return finalStatus; } Status ClusterAggregate::runAggregateWithRoutingCtx(