diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_execution_control_with_prioritization.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_execution_control_with_prioritization.yml index 2fc168b1c9a..f43693e26e1 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_execution_control_with_prioritization.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_execution_control_with_prioritization.yml @@ -42,6 +42,7 @@ executor: mongod_options: oplogSize: 1024 hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_continuous_new_old_old_new.yml index 6dd048e17e3..67cf2f92857 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_continuous_new_old_old_new.yml @@ -44,6 +44,7 @@ executor: mongod_options: oplogSize: 1024 hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_lts_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_lts_new_old_old_new.yml index 1f31ddf6aa0..9206f6ad34b 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_lts_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_last_lts_new_old_old_new.yml @@ -44,6 +44,7 @@ executor: mongod_options: oplogSize: 1024 hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_primary_driven_index_builds.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_primary_driven_index_builds.yml index 7e67411f5d3..2600e4d3a9a 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_primary_driven_index_builds.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/concurrency_sharded_replication_primary_driven_index_builds.yml @@ -45,6 +45,7 @@ executor: mongod_options: oplogSize: 1024 hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml index 6965e8cac2f..edefb5103b6 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_continuous_new_old_old_new.yml @@ -10,6 +10,7 @@ executor: archive: hooks: - ContinuousStepdown + - CheckIdleCursors - CheckReplDBHash - CheckMetadataConsistencyInBackground - ValidateCollections @@ -74,6 +75,7 @@ executor: hooks: - class: ContinuousStepdown randomize_kill: true + - class: CheckIdleCursors - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_lts_new_old_old_new.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_lts_new_old_old_new.yml index 3ca22de719e..85009bfafd8 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_lts_new_old_old_new.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_primaries_jscore_passthrough_last_lts_new_old_old_new.yml @@ -10,6 +10,7 @@ executor: archive: hooks: - ContinuousStepdown + - CheckIdleCursors - CheckReplDBHash - CheckMetadataConsistencyInBackground - ValidateCollections @@ -74,6 +75,7 @@ executor: hooks: - class: ContinuousStepdown randomize_kill: true + - class: CheckIdleCursors - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_stepdown_terminate_jscore_passthrough.yml b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_stepdown_terminate_jscore_passthrough.yml index 63f44892214..c7f7228da37 100644 --- a/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_stepdown_terminate_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_kill_stepdown_terminate_jscore_passthrough.yml @@ -9,6 +9,7 @@ executor: archive: hooks: + - CheckIdleCursors - CheckReplDBHash - CheckMetadataConsistencyInBackground - ValidateCollections @@ -70,6 +71,7 @@ executor: hooks: - class: ContinuousStepdown randomize_kill: true + - class: CheckIdleCursors - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/kill_primary.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/kill_primary.yml index 7da337cff52..aa9e3168bee 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/kill_primary.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/kill_primary.yml @@ -287,6 +287,7 @@ # too many network errors occur when re-running a transaction we will run out of retries. - class: ContinuousStepdown randomize_kill: true + - class: CheckIdleCursors - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground - class: ValidateCollections @@ -306,6 +307,7 @@ test: true hooks: - ContinuousStepdown + - CheckIdleCursors - CheckReplDBHash - CheckMetadataConsistencyInBackground - ValidateCollections @@ -384,6 +386,7 @@ archive: test: true hooks: + - CheckIdleCursors - CheckReplDBHash - CheckMetadataConsistencyInBackground - ValidateCollections @@ -392,6 +395,7 @@ hooks: - class: ContinuousStepdown randomize_kill: true + - class: CheckIdleCursors - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml index 692aee3cd3d..1522c05cae4 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml @@ -55,6 +55,7 @@ executor: runningWithBalancer: false implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml index 18a82f85f39..df1e11c38fb 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml @@ -69,6 +69,7 @@ executor: implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml index c9a28e75941..e84cd7b1405 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml @@ -131,6 +131,7 @@ executor: - class: ContinuousInitialSync is_fsm_workload: true sync_interval_secs: 15 + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml index 9a654995225..33d55065afe 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml @@ -106,6 +106,7 @@ executor: implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml index 4cad6389a72..40a92c5c7bf 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml @@ -121,6 +121,7 @@ executor: implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml index 88a9e0c8064..44063984d0b 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml @@ -96,6 +96,7 @@ executor: traceExceptions: false implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary.yml index c4ac92f6e31..35f9cec2036 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary.yml @@ -173,6 +173,7 @@ executor: shard_stepdown: true stepdown_interval_ms: 15000 is_fsm_workload: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml index 9c15225f9ae..a91abf1bc2b 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml @@ -101,6 +101,7 @@ executor: implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer_and_config_transitions_and_add_remove_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer_and_config_transitions_and_add_remove_shard.yml index f488cc9e2cb..c23d9f3a9e4 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer_and_config_transitions_and_add_remove_shard.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer_and_config_transitions_and_add_remove_shard.yml @@ -142,6 +142,7 @@ executor: add_remove_random_shards: false # TODO(SERVER-91474): turn this to true when fixing the ticket move_primary_comment: *movePrimaryComment is_fsm_workload: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml index a0181cecc83..408dd66a62e 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml @@ -35,6 +35,7 @@ executor: runningWithBalancer: false implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata # TODO (SERVER-63855): remove 'RunDBCheckInBackground' or put it back. # - class: RunDBCheckInBackground diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml index 71f080fffbf..cca017741df 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml @@ -71,6 +71,7 @@ executor: runningWithBalancer: true implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions.yml index c8154fd94a5..2810753650c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions.yml @@ -102,6 +102,7 @@ executor: transition_configsvr: true move_primary_comment: *movePrimaryComment is_fsm_workload: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions_and_add_remove_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions_and_add_remove_shard.yml index 29f091ee10e..7fa6a09cd64 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions_and_add_remove_shard.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer_and_config_transitions_and_add_remove_shard.yml @@ -138,6 +138,7 @@ executor: transition_configsvr: true add_remove_random_shards: true is_fsm_workload: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata # Suites that shutdown nodes are not compatible with the CheckReplDBHashInBackground hook, so # this suite does not include that hook diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer.yml index bf69817d06e..cfc3e0111ea 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer.yml @@ -103,6 +103,7 @@ executor: shard_stepdown: true is_fsm_workload: true randomize_kill: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions_and_add_remove_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions_and_add_remove_shard.yml index fcec0296137..e84d4664b79 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions_and_add_remove_shard.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions_and_add_remove_shard.yml @@ -133,6 +133,7 @@ executor: add_remove_random_shards: true move_primary_comment: *movePrimaryComment is_fsm_workload: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml index 17c028c7bfc..da50f8ddc9c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml @@ -70,6 +70,7 @@ executor: implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHashInBackground - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml index ae036c60429..ba5816024ca 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml @@ -57,6 +57,7 @@ executor: runningWithBalancer: false implicitlyShardOnCreateCollectionOnly: true hooks: + - class: CheckIdleCursors - class: CheckShardFilteringMetadata # TODO (SERVER-63855): remove 'RunDBCheckInBackground' or put it back. # - class: RunDBCheckInBackground diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index c19c6d81e06..d970cab909d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -94,6 +94,7 @@ executor: config_stepdown: true shard_stepdown: true is_fsm_workload: true + - class: CheckIdleCursors - class: CheckShardFilteringMetadata - class: CheckReplDBHash - class: CheckMetadataConsistencyInBackground diff --git a/buildscripts/resmokelib/testing/hooks/OWNERS.yml b/buildscripts/resmokelib/testing/hooks/OWNERS.yml index dcd40957075..6d3e7230e52 100644 --- a/buildscripts/resmokelib/testing/hooks/OWNERS.yml +++ b/buildscripts/resmokelib/testing/hooks/OWNERS.yml @@ -60,6 +60,9 @@ filters: - "drop_sessions_collection.py": approvers: - 10gen/server-catalog-and-routing-ddl + - "check_idle_cursors.py": + approvers: + - 10gen/server-jstest-infra - "rotate_execution_control_params.py": approvers: - 10gen/server-workload-resilience diff --git a/buildscripts/resmokelib/testing/hooks/check_idle_cursors.py b/buildscripts/resmokelib/testing/hooks/check_idle_cursors.py new file mode 100644 index 00000000000..8ec4b206686 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/check_idle_cursors.py @@ -0,0 +1,29 @@ +import os.path + +from buildscripts.resmokelib.testing.hooks import jsfile +from buildscripts.resmokelib.utils import jscomment + + +class CheckIdleCursors(jsfile.JSHook): + IS_BACKGROUND = False + + # Tag you’ll set in the JS test files that are allowed to leak cursors. + ALLOW_LEAK_TAG = "can_leak_idle_cursors" + + def __init__(self, hook_logger, fixture, shell_options={}): + description = "Checking for idle cursors in $currentOp" + js_filename = os.path.join("jstests", "hooks", "jstest_infra", "run_check_idle_cursors.js") + super().__init__( + hook_logger, fixture, js_filename, description, shell_options=shell_options + ) + + def after_test(self, test, test_report): + global_vars = self._shell_options.setdefault("global_vars", {}) + test_data = global_vars.setdefault("TestData", {}) + test_data["shouldKillIdleCursors"] = self._does_test_allow_leaking_idle_cursors(test) + + super().after_test(test, test_report) + + def _does_test_allow_leaking_idle_cursors(self, test): + tags = jscomment.get_tags(test.test_name) + return self.ALLOW_LEAK_TAG in tags diff --git a/jstests/concurrency/fsm_libs/worker_thread.js b/jstests/concurrency/fsm_libs/worker_thread.js index 48ff7270f1a..9ca84e3ef79 100644 --- a/jstests/concurrency/fsm_libs/worker_thread.js +++ b/jstests/concurrency/fsm_libs/worker_thread.js @@ -287,19 +287,6 @@ export const workerThread = (function () { }; } } finally { - // Kill this worker thread's session to ensure any possible idle cursors left open by - // the workload are closed. - // TODO SERVER-74993: Remove this. - try { - let session = myDB.getSession(); - if (session) { - myDB.runCommand({killSessions: [session.getSessionId()]}); - } - } catch (e) { - // Ignore errors from killSessions. - jsTest.log("Error running killSessions: " + e); - } - // Avoid retention of connection object configs = null; myDB = null; diff --git a/jstests/concurrency/fsm_workloads/ddl/random_ddl/random_ddl_check_metadata_consistency_killop.js b/jstests/concurrency/fsm_workloads/ddl/random_ddl/random_ddl_check_metadata_consistency_killop.js index 2ebb4ffce14..6173762f988 100644 --- a/jstests/concurrency/fsm_workloads/ddl/random_ddl/random_ddl_check_metadata_consistency_killop.js +++ b/jstests/concurrency/fsm_workloads/ddl/random_ddl/random_ddl_check_metadata_consistency_killop.js @@ -7,6 +7,8 @@ * requires_sharding, * # killOp does not support stepdowns. * does_not_support_stepdowns, + * # TODO: SERVER-114503 Investigate DDL commands FSM tests leaking cursors. + * can_leak_idle_cursors, * ] */ diff --git a/jstests/concurrency/fsm_workloads/query/agg/agg_lookup.js b/jstests/concurrency/fsm_workloads/query/agg/agg_lookup.js index fb8b95b4593..ba42e9a92ce 100644 --- a/jstests/concurrency/fsm_workloads/query/agg/agg_lookup.js +++ b/jstests/concurrency/fsm_workloads/query/agg/agg_lookup.js @@ -2,6 +2,10 @@ * agg_lookup.js * * Runs a $lookup aggregation simultaneously with updates. + * @tags: [ + * # TODO: SERVER-114502 Investigate agg_lookup and long_secondary_reads fsm tests leaking cursors. + * can_leak_idle_cursors, + * ] */ import {interruptedQueryErrors} from "jstests/concurrency/fsm_libs/assert.js"; import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js"; diff --git a/jstests/concurrency/fsm_workloads/query/long_secondary_reads.js b/jstests/concurrency/fsm_workloads/query/long_secondary_reads.js index 61f85a2aeff..6ea112223f1 100644 --- a/jstests/concurrency/fsm_workloads/query/long_secondary_reads.js +++ b/jstests/concurrency/fsm_workloads/query/long_secondary_reads.js @@ -13,7 +13,9 @@ * # Tests which expect commands to fail and catch the error can cause transactions to abort and * # retry indefinitely. * catches_command_failures, - * requires_fcv_82 + * requires_fcv_82, + * # TODO: SERVER-114502 Investigate agg_lookup and long_secondary_reads fsm tests leaking cursors. + * can_leak_idle_cursors, * ] */ diff --git a/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_move_collection.js b/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_move_collection.js index a7abfe49350..2a0b4facc6b 100644 --- a/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_move_collection.js +++ b/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_move_collection.js @@ -13,6 +13,8 @@ * # TODO (SERVER-91251): Run this with stepdowns on sanitizers. * tsan_incompatible, * incompatible_aubsan, + * # TODO: SERVER-114503 Investigate DDL commands FSM tests leaking cursors. + * can_leak_idle_cursors, * ]; */ import {extendWorkload} from "jstests/concurrency/fsm_libs/extend_workload.js"; diff --git a/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_resharding.js b/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_resharding.js index 3e3e1cfca11..623d2e3d5e0 100644 --- a/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_resharding.js +++ b/jstests/concurrency/fsm_workloads/random_update_delete/random_update_delete_with_resharding.js @@ -13,6 +13,8 @@ * # TODO (SERVER-91251): Run this with stepdowns on sanitizers. * tsan_incompatible, * incompatible_aubsan, + * # TODO: SERVER-114503 Investigate DDL commands FSM tests leaking cursors. + * can_leak_idle_cursors, * ]; */ import {extendWorkload} from "jstests/concurrency/fsm_libs/extend_workload.js"; diff --git a/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_crud_ops.js b/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_crud_ops.js index 8f7fb60624d..69fbb80d0d8 100644 --- a/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_crud_ops.js +++ b/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_crud_ops.js @@ -1,7 +1,12 @@ /** * Same as the base workload, but refines to a nested shard key. * - * @tags: [requires_persistence, requires_sharding] + * @tags: [ + * requires_persistence, + * requires_sharding, + * # TODO: SERVER-114503 Investigate DDL commands FSM tests leaking cursors. + * can_leak_idle_cursors, + * ] */ import {extendWorkload} from "jstests/concurrency/fsm_libs/extend_workload.js"; diff --git a/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_zone_ops.js b/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_zone_ops.js index 57943cbff3b..e73d52ef2d1 100644 --- a/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_zone_ops.js +++ b/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_nested_zone_ops.js @@ -1,7 +1,13 @@ /** * Same as the base workload, but refines to a nested shard key. * - * @tags: [requires_persistence, requires_sharding, assumes_stable_shard_list] + * @tags: [ + * requires_persistence, + * requires_sharding, + * assumes_stable_shard_list, + * # TODO: SERVER-114503 Investigate DDL commands FSM tests leaking cursors. + * can_leak_idle_cursors, + * ] */ import {extendWorkload} from "jstests/concurrency/fsm_libs/extend_workload.js"; diff --git a/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_zone_ops.js b/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_zone_ops.js index 37388e102be..c0bff4e4952 100644 --- a/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_zone_ops.js +++ b/jstests/concurrency/fsm_workloads/refine_collection_shard_key/refine_collection_shard_key_zone_ops.js @@ -13,7 +13,13 @@ * - refineCollectionShardKey - Refines the collection's shard key and decreases the latch count * such that the next latch collection will be targeted by the test. * - * @tags: [requires_persistence, requires_sharding, assumes_stable_shard_list] + * @tags: [ + * requires_persistence, + * requires_sharding, + * assumes_stable_shard_list, + * # TODO: SERVER-114503 Investigate DDL commands FSM tests leaking cursors. + * can_leak_idle_cursors, + * ] */ import "jstests/libs/parallelTester.js"; diff --git a/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_catalog_operations.js b/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_catalog_operations.js index 61ce6579e75..0d5fff7316e 100644 --- a/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_catalog_operations.js +++ b/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_catalog_operations.js @@ -5,7 +5,12 @@ * once the read has begun, catalog operations with conflicting locks will block until the read is * finished. * - * @tags: [creates_background_indexes, uses_transactions] + * @tags: [ + * creates_background_indexes, + * uses_transactions, + * # TODO: SERVER-114500 Investigate snapshot_read* FSM tests leaking cursors. + * can_leak_idle_cursors, + * ] */ import {doSnapshotFind, doSnapshotGetMore} from "jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js"; diff --git a/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_op_only.js b/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_op_only.js index 244a6e3c54e..fe7b99a594a 100644 --- a/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_op_only.js +++ b/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_op_only.js @@ -10,6 +10,8 @@ * requires_getmore, * # This test relies on query commands returning specific batch-sized responses. * assumes_no_implicit_cursor_exhaustion, + * # TODO: SERVER-114500 Investigate snapshot_read* FSM tests leaking cursors. + * can_leak_idle_cursors, * ] */ diff --git a/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_operations.js b/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_operations.js index ae19af21c0d..be813ed45a9 100644 --- a/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_operations.js +++ b/jstests/concurrency/fsm_workloads/txns/snapshot_read/snapshot_read_kill_operations.js @@ -8,6 +8,8 @@ * requires_getmore, * # This test relies on query commands returning specific batch-sized responses. * assumes_no_implicit_cursor_exhaustion, + * # TODO: SERVER-114500 Investigate snapshot_read* FSM tests leaking cursors. + * can_leak_idle_cursors, * ] */ diff --git a/jstests/concurrency/fsm_workloads/view_catalog/view_catalog_cycle_lookup.js b/jstests/concurrency/fsm_workloads/view_catalog/view_catalog_cycle_lookup.js index f0b110aef2e..c048c058568 100644 --- a/jstests/concurrency/fsm_workloads/view_catalog/view_catalog_cycle_lookup.js +++ b/jstests/concurrency/fsm_workloads/view_catalog/view_catalog_cycle_lookup.js @@ -7,6 +7,7 @@ * * @tags: [ * requires_fcv_51, + * can_leak_idle_cursors, * ] */ diff --git a/jstests/hooks/jstest_infra/BUILD.bazel b/jstests/hooks/jstest_infra/BUILD.bazel new file mode 100644 index 00000000000..97513de081b --- /dev/null +++ b/jstests/hooks/jstest_infra/BUILD.bazel @@ -0,0 +1,12 @@ +load("//bazel:mongo_js_rules.bzl", "all_subpackage_javascript_files", "mongo_js_library") + +package(default_visibility = ["//visibility:public"]) + +mongo_js_library( + name = "all_javascript_files", + srcs = glob([ + "*.js", + ]), +) + +all_subpackage_javascript_files() diff --git a/jstests/hooks/jstest_infra/OWNERS.yml b/jstests/hooks/jstest_infra/OWNERS.yml new file mode 100644 index 00000000000..2b9e6d8fc08 --- /dev/null +++ b/jstests/hooks/jstest_infra/OWNERS.yml @@ -0,0 +1,5 @@ +version: 1.0.0 +filters: + - "*": + approvers: + - 10gen/server-jstest-infra diff --git a/jstests/hooks/jstest_infra/run_check_idle_cursors.js b/jstests/hooks/jstest_infra/run_check_idle_cursors.js new file mode 100644 index 00000000000..bc4709bec66 --- /dev/null +++ b/jstests/hooks/jstest_infra/run_check_idle_cursors.js @@ -0,0 +1,71 @@ +// This script checks for leaked cursors in the MongoDB system. +import {DiscoverTopology} from "jstests/libs/discover_topology.js"; + +function findIdleCursors(conn) { + try { + const systemIntrospectionFilter = { + $nor: [ + // Commands: + {"cursor.originatingCommand.listDatabases": {$exists: true}}, + {"cursor.originatingCommand.listCollections": {$exists: true}}, + {"cursor.originatingCommand.listIndexes": {$exists: true}}, + + // Aggregation pipelines: + {"cursor.originatingCommand.pipeline.0.$listCatalog": {$exists: true}}, + {"cursor.originatingCommand.pipeline.0.$indexStats": {$exists: true}}, + {"cursor.originatingCommand.pipeline.0.$collStats": {$exists: true}}, + ], + }; + const systemCollections = ["local.oplog.rs", "config.system.sessions"]; + + return conn + .getDB("admin") + .aggregate( + [ + {$currentOp: {localOps: true, idleCursors: true}}, + {$match: {"command.comment": {$ne: "$currentOp"}}}, + {$match: {"type": "idleCursor"}}, + {$match: {"ns": {$nin: systemCollections}}}, + {$match: systemIntrospectionFilter}, + ], + {comment: "$currentOp"}, + ) + .toArray(); + } catch (error) { + jsTest.log.info(`Failed to read currentOps for connection`, {conn, error}); + return []; + } +} + +function assertNoIdleCursors(conn) { + let idleCursors = findIdleCursors(conn); + jsTest.log.info("Idle cursor for connection", {conn, idleCursors}); + + // Currently some tests leak idle cursors. In order to not fail those tests we kill the leaked cursors. + if (TestData && TestData.shouldKillIdleCursors) { + jsTest.log.info("Killing idle cursors for connection", {conn, idleCursors}); + idleCursors.forEach(function (idleCursor) { + assert.commandWorked( + conn.getDB("admin").runCommand({killCursors: idleCursor.ns, cursors: [idleCursor.cursor.cursorId]}), + ); + }); + + idleCursors = findIdleCursors(conn); + } + + assert.eq(0, idleCursors.length, `Expected no idleCursors left over on connection ${conn}`); +} + +(function () { + const nodesToConnectTo = (function () { + const topology = DiscoverTopology.findConnectedNodes(db.getMongo()); + if (topology.mongos) { + const shardConnStrings = Object.values(topology.shards).flatMap((shardInfo) => shardInfo.nodes); + return topology.mongos.nodes.concat(shardConnStrings); + } + + return [topology.nodes]; + })(); + jsTest.log.info("Nodes to check for leaked cursors", {nodesToConnectTo}); + nodesToConnectTo.map((connString) => new Mongo(connString)).forEach(assertNoIdleCursors); +})();