From 18c1f142b5b3fd179a677b2736332dc17770b7c2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 23 Dec 2025 06:43:32 -0600 Subject: [PATCH] PYTHON-5529 Introduce optin setting to await for MinPoolSize population (#2664) --- test/asynchronous/test_client.py | 2 +- test/asynchronous/unified_format.py | 14 ++- test/asynchronous/utils.py | 6 +- test/csot/command-execution.json | 14 ++- test/csot/convenient-transactions.json | 6 +- test/csot/error-transformations.json | 6 +- test/csot/global-timeoutMS.json | 130 ++++++++++++++++------ test/csot/non-tailable-cursors.json | 6 +- test/csot/retryability-timeoutMS.json | 6 +- test/csot/runCursorCommand.json | 6 +- test/csot/sessions-inherit-timeoutMS.json | 6 +- test/test_client.py | 2 +- test/unified_format.py | 14 ++- test/utils.py | 6 +- 14 files changed, 164 insertions(+), 60 deletions(-) diff --git a/test/asynchronous/test_client.py b/test/asynchronous/test_client.py index 37fb23923..5511765ba 100644 --- a/test/asynchronous/test_client.py +++ b/test/asynchronous/test_client.py @@ -2398,7 +2398,7 @@ class TestExhaustCursor(AsyncIntegrationTest): client = self.async_rs_or_single_client() self.addCleanup(client.close) coll = client.pymongo_test.test - pool = async_get_pool(client) + pool = async_get_pool(client) # type:ignore # Patch the pool to delay the connect method. def delayed_connect(*args, **kwargs): diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index bca259b82..be2372f22 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -329,6 +329,17 @@ class EntityMapUtil: kwargs["h"] = uri client = await self.test.async_rs_or_single_client(**kwargs) await client.aconnect() + # Wait for pool to be populated. + if "awaitMinPoolSizeMS" in spec: + pool = await async_get_pool(client) + t0 = time.monotonic() + while True: + if (time.monotonic() - t0) > spec["awaitMinPoolSizeMS"] * 1000: + raise ValueError("Test timed out during awaitMinPoolSize") + async with pool.lock: + if len(pool.conns) + pool.active_sockets >= pool.opts.min_pool_size: + break + await asyncio.sleep(0.1) self[spec["id"]] = client return elif entity_type == "database": @@ -463,7 +474,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.25") + SCHEMA_VERSION = Version.from_string("1.26") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -1551,7 +1562,6 @@ class UnifiedSpecTestMeta(type): if re.search(fail_pattern, description): test_method = unittest.expectedFailure(test_method) break - setattr(cls, test_name, test_method) diff --git a/test/asynchronous/utils.py b/test/asynchronous/utils.py index 02ba46c71..9021b0608 100644 --- a/test/asynchronous/utils.py +++ b/test/asynchronous/utils.py @@ -28,25 +28,25 @@ from inspect import iscoroutinefunction from bson.son import SON from pymongo import AsyncMongoClient +from pymongo.asynchronous.pool import Pool, _CancellationContext, _PoolGeneration from pymongo.errors import ConfigurationError from pymongo.hello import HelloCompat from pymongo.lock import _async_create_lock from pymongo.operations import _Op from pymongo.read_preferences import ReadPreference from pymongo.server_selectors import any_server_selector, writable_server_selector -from pymongo.synchronous.pool import _CancellationContext, _PoolGeneration _IS_SYNC = False -async def async_get_pool(client): +async def async_get_pool(client: AsyncMongoClient) -> Pool: """Get the standalone, primary, or mongos pool.""" topology = await client._get_topology() server = await topology._select_server(writable_server_selector, _Op.TEST) return server.pool -async def async_get_pools(client): +async def async_get_pools(client: AsyncMongoClient) -> list[Pool]: """Get all pools.""" return [ server.pool diff --git a/test/csot/command-execution.json b/test/csot/command-execution.json index aa9c3eb23..212cd4108 100644 --- a/test/csot/command-execution.json +++ b/test/csot/command-execution.json @@ -1,6 +1,6 @@ { "description": "timeoutMS behaves correctly during command execution", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.4.7", @@ -69,8 +69,10 @@ "appName": "reduceMaxTimeMSTest", "w": 1, "timeoutMS": 500, - "heartbeatFrequencyMS": 500 + "heartbeatFrequencyMS": 500, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "observeEvents": [ "commandStartedEvent" ] @@ -185,8 +187,10 @@ "appName": "rttTooHighTest", "w": 1, "timeoutMS": 10, - "heartbeatFrequencyMS": 500 + "heartbeatFrequencyMS": 500, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "observeEvents": [ "commandStartedEvent" ] @@ -316,8 +320,10 @@ "appName": "reduceMaxTimeMSTest", "w": 1, "timeoutMS": 90, - "heartbeatFrequencyMS": 100000 + "heartbeatFrequencyMS": 100000, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "observeEvents": [ "commandStartedEvent" ] diff --git a/test/csot/convenient-transactions.json b/test/csot/convenient-transactions.json index 3868b3026..f9d03429d 100644 --- a/test/csot/convenient-transactions.json +++ b/test/csot/convenient-transactions.json @@ -1,6 +1,6 @@ { "description": "timeoutMS behaves correctly for the withTransaction API", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -21,8 +21,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 500 + "timeoutMS": 500, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" diff --git a/test/csot/error-transformations.json b/test/csot/error-transformations.json index 4889e3958..89be49f0a 100644 --- a/test/csot/error-transformations.json +++ b/test/csot/error-transformations.json @@ -1,6 +1,6 @@ { "description": "MaxTimeMSExpired server errors are transformed into a custom timeout error", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.0", @@ -26,8 +26,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" diff --git a/test/csot/global-timeoutMS.json b/test/csot/global-timeoutMS.json index f1edbe68e..9d8046d1b 100644 --- a/test/csot/global-timeoutMS.json +++ b/test/csot/global-timeoutMS.json @@ -1,6 +1,6 @@ { "description": "timeoutMS can be configured on a MongoClient", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -38,8 +38,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -217,8 +219,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -390,8 +394,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -569,8 +575,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -762,8 +770,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -941,8 +951,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -1120,8 +1132,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -1305,8 +1319,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -1484,8 +1500,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -1663,8 +1681,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -1842,8 +1862,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -2021,8 +2043,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -2194,8 +2218,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -2375,8 +2401,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -2554,8 +2582,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -2733,8 +2763,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -2906,8 +2938,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -3079,8 +3113,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -3258,8 +3294,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -3441,8 +3479,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -3628,8 +3668,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -3807,8 +3849,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -3986,8 +4030,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -4171,8 +4217,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -4360,8 +4408,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -4549,8 +4599,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -4728,8 +4780,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -4913,8 +4967,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -5102,8 +5158,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -5297,8 +5355,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -5482,8 +5542,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" @@ -5677,8 +5739,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 250 + "timeoutMS": 250, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" diff --git a/test/csot/non-tailable-cursors.json b/test/csot/non-tailable-cursors.json index 291c6e72a..58c59cb32 100644 --- a/test/csot/non-tailable-cursors.json +++ b/test/csot/non-tailable-cursors.json @@ -1,6 +1,6 @@ { "description": "timeoutMS behaves correctly for non-tailable cursors", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.4" @@ -17,8 +17,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 200 + "timeoutMS": 200, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" diff --git a/test/csot/retryability-timeoutMS.json b/test/csot/retryability-timeoutMS.json index 9daad260e..5a0c9f360 100644 --- a/test/csot/retryability-timeoutMS.json +++ b/test/csot/retryability-timeoutMS.json @@ -1,6 +1,6 @@ { "description": "timeoutMS behaves correctly for retryable operations", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.0", @@ -26,8 +26,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 100 + "timeoutMS": 100, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent" diff --git a/test/csot/runCursorCommand.json b/test/csot/runCursorCommand.json index 36f774fb5..e5182e338 100644 --- a/test/csot/runCursorCommand.json +++ b/test/csot/runCursorCommand.json @@ -1,6 +1,6 @@ { "description": "runCursorCommand", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.4" @@ -16,6 +16,10 @@ { "client": { "id": "commandClient", + "uriOptions": { + "minPoolSize": 1 + }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent", diff --git a/test/csot/sessions-inherit-timeoutMS.json b/test/csot/sessions-inherit-timeoutMS.json index 13ea91c79..dbf163e48 100644 --- a/test/csot/sessions-inherit-timeoutMS.json +++ b/test/csot/sessions-inherit-timeoutMS.json @@ -1,6 +1,6 @@ { "description": "sessions inherit timeoutMS from their parent MongoClient", - "schemaVersion": "1.9", + "schemaVersion": "1.26", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -21,8 +21,10 @@ "client": { "id": "client", "uriOptions": { - "timeoutMS": 500 + "timeoutMS": 500, + "minPoolSize": 1 }, + "awaitMinPoolSizeMS": 10000, "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent", diff --git a/test/test_client.py b/test/test_client.py index 2d30eb1bd..737b3afe6 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -2353,7 +2353,7 @@ class TestExhaustCursor(IntegrationTest): client = self.rs_or_single_client() self.addCleanup(client.close) coll = client.pymongo_test.test - pool = get_pool(client) + pool = get_pool(client) # type:ignore # Patch the pool to delay the connect method. def delayed_connect(*args, **kwargs): diff --git a/test/unified_format.py b/test/unified_format.py index 06c6abd00..9e3dc1fed 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -328,6 +328,17 @@ class EntityMapUtil: kwargs["h"] = uri client = self.test.rs_or_single_client(**kwargs) client._connect() + # Wait for pool to be populated. + if "awaitMinPoolSizeMS" in spec: + pool = get_pool(client) + t0 = time.monotonic() + while True: + if (time.monotonic() - t0) > spec["awaitMinPoolSizeMS"] * 1000: + raise ValueError("Test timed out during awaitMinPoolSize") + with pool.lock: + if len(pool.conns) + pool.active_sockets >= pool.opts.min_pool_size: + break + time.sleep(0.1) self[spec["id"]] = client return elif entity_type == "database": @@ -462,7 +473,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.25") + SCHEMA_VERSION = Version.from_string("1.26") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -1536,7 +1547,6 @@ class UnifiedSpecTestMeta(type): if re.search(fail_pattern, description): test_method = unittest.expectedFailure(test_method) break - setattr(cls, test_name, test_method) diff --git a/test/utils.py b/test/utils.py index bfc606fe8..7b5a4c821 100644 --- a/test/utils.py +++ b/test/utils.py @@ -34,19 +34,19 @@ from pymongo.lock import _create_lock from pymongo.operations import _Op from pymongo.read_preferences import ReadPreference from pymongo.server_selectors import any_server_selector, writable_server_selector -from pymongo.synchronous.pool import _CancellationContext, _PoolGeneration +from pymongo.synchronous.pool import Pool, _CancellationContext, _PoolGeneration _IS_SYNC = True -def get_pool(client): +def get_pool(client: MongoClient) -> Pool: """Get the standalone, primary, or mongos pool.""" topology = client._get_topology() server = topology._select_server(writable_server_selector, _Op.TEST) return server.pool -def get_pools(client): +def get_pools(client: MongoClient) -> list[Pool]: """Get all pools.""" return [ server.pool