diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index e4fdf25c2..3e4dc482d 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -1195,7 +1195,8 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): ResourceWarning, stacklevel=2, ) - except AttributeError: + except (AttributeError, TypeError): + # Ignore errors at interpreter exit. pass def _close_cursor_soon( diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index a9f02d650..ca0cebd41 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1249,6 +1249,9 @@ class Pool: async with self.lock: conn_id = self.next_connection_id self.next_connection_id += 1 + # Use a temporary context so that interrupt_connections can cancel creating the socket. + tmp_context = _CancellationContext() + self.active_contexts.add(tmp_context) listeners = self.opts._event_listeners if self.enabled_for_cmap: @@ -1267,6 +1270,8 @@ class Pool: try: sock = await _configured_socket(self.address, self.opts) except BaseException as error: + async with self.lock: + self.active_contexts.discard(tmp_context) if self.enabled_for_cmap: assert listeners is not None listeners.publish_connection_closed( @@ -1292,6 +1297,9 @@ class Pool: conn = AsyncConnection(sock, self, self.address, conn_id) # type: ignore[arg-type] async with self.lock: self.active_contexts.add(conn.cancel_context) + self.active_contexts.discard(tmp_context) + if tmp_context.cancelled: + conn.cancel_context.cancel() try: if self.handshake: await conn.hello() @@ -1301,6 +1309,8 @@ class Pool: await conn.authenticate() except BaseException: + async with self.lock: + self.active_contexts.discard(conn.cancel_context) conn.close_conn(ConnectionClosedReason.ERROR) raise diff --git a/pymongo/message.py b/pymongo/message.py index 3e2ae00ae..b6c00f06c 100644 --- a/pymongo/message.py +++ b/pymongo/message.py @@ -252,6 +252,10 @@ def _gen_find_command( if limit < 0: cmd["singleBatch"] = True if batch_size: + # When limit and batchSize are equal we increase batchSize by 1 to + # avoid an unnecessary killCursors. + if limit == batch_size: + batch_size += 1 cmd["batchSize"] = batch_size if read_concern.level and not (session and session.in_transaction): cmd["readConcern"] = read_concern.document diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index f3ed6cd2c..038dbb3b5 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -70,13 +70,14 @@ elif sys.platform == "darwin": "version": platform.mac_ver()[0], } elif sys.platform == "win32": + _ver = sys.getwindowsversion() _METADATA["os"] = { - "type": platform.system(), - # "Windows XP", "Windows 7", "Windows 10", etc. - "name": " ".join((platform.system(), platform.release())), - "architecture": platform.machine(), - # Windows patch level (e.g. 5.1.2600-SP3) - "version": "-".join(platform.win32_ver()[1:3]), + "type": "Windows", + "name": "Windows", + # Avoid using platform calls, see PYTHON-4455. + "architecture": os.environ.get("PROCESSOR_ARCHITECTURE") or platform.machine(), + # Windows patch level (e.g. 10.0.17763-SP0). + "version": ".".join(map(str, _ver[:3])) + f"-SP{_ver[-1] or '0'}", } elif sys.platform.startswith("java"): _name, _ver, _arch = platform.java_ver()[-1] diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 0380d4468..00c6203a9 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -1193,7 +1193,8 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): ResourceWarning, stacklevel=2, ) - except AttributeError: + except (AttributeError, TypeError): + # Ignore errors at interpreter exit. pass def _close_cursor_soon( diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index eb007a347..86baf15b9 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1243,6 +1243,9 @@ class Pool: with self.lock: conn_id = self.next_connection_id self.next_connection_id += 1 + # Use a temporary context so that interrupt_connections can cancel creating the socket. + tmp_context = _CancellationContext() + self.active_contexts.add(tmp_context) listeners = self.opts._event_listeners if self.enabled_for_cmap: @@ -1261,6 +1264,8 @@ class Pool: try: sock = _configured_socket(self.address, self.opts) except BaseException as error: + with self.lock: + self.active_contexts.discard(tmp_context) if self.enabled_for_cmap: assert listeners is not None listeners.publish_connection_closed( @@ -1286,6 +1291,9 @@ class Pool: conn = Connection(sock, self, self.address, conn_id) # type: ignore[arg-type] with self.lock: self.active_contexts.add(conn.cancel_context) + self.active_contexts.discard(tmp_context) + if tmp_context.cancelled: + conn.cancel_context.cancel() try: if self.handshake: conn.hello() @@ -1295,6 +1303,8 @@ class Pool: conn.authenticate() except BaseException: + with self.lock: + self.active_contexts.discard(conn.cancel_context) conn.close_conn(ConnectionClosedReason.ERROR) raise diff --git a/requirements/typing.txt b/requirements/typing.txt index ad799ea36..613eba764 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,5 +1,5 @@ mypy==1.13.0 -pyright==1.1.388 +pyright==1.1.389 typing_extensions -r ./encryption.txt -r ./ocsp.txt diff --git a/test/asynchronous/test_retryable_reads.py b/test/asynchronous/test_retryable_reads.py index b2d86f5d8..bde7a9f2e 100644 --- a/test/asynchronous/test_retryable_reads.py +++ b/test/asynchronous/test_retryable_reads.py @@ -174,9 +174,8 @@ class TestRetryableReads(AsyncIntegrationTest): retryReads=True, ) - async with self.fail_point(fail_command): - with self.assertRaises(AutoReconnect): - await client.t.t.find_one({}) + with self.assertRaises(AutoReconnect): + await client.t.t.find_one({}) # Disable failpoints on each mongos for client in mongos_clients: diff --git a/test/asynchronous/test_retryable_writes.py b/test/asynchronous/test_retryable_writes.py index accbbd003..ca2f0a542 100644 --- a/test/asynchronous/test_retryable_writes.py +++ b/test/asynchronous/test_retryable_writes.py @@ -1,4 +1,4 @@ -# Copyright 2017 MongoDB, Inc. +# Copyright 2017-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -43,7 +43,6 @@ from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.int64 import Int64 from bson.raw_bson import RawBSONDocument from bson.son import SON -from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.errors import ( AutoReconnect, ConnectionFailure, @@ -226,47 +225,6 @@ class TestRetryableWrites(IgnoreDeprecationsTest): f"{msg} sent txnNumber with {event.command_name}", ) - @async_client_context.require_no_standalone - async def test_supported_single_statement_supported_cluster(self): - for method, args, kwargs in retryable_single_statement_ops(self.db.retryable_write_test): - msg = f"{method.__name__}(*{args!r}, **{kwargs!r})" - self.listener.reset() - await method(*args, **kwargs) - commands_started = self.listener.started_events - self.assertEqual(len(self.listener.succeeded_events), 1, msg) - first_attempt = commands_started[0] - self.assertIn( - "lsid", - first_attempt.command, - f"{msg} sent no lsid with {first_attempt.command_name}", - ) - initial_session_id = first_attempt.command["lsid"] - self.assertIn( - "txnNumber", - first_attempt.command, - f"{msg} sent no txnNumber with {first_attempt.command_name}", - ) - - # There should be no retry when the failpoint is not active. - if async_client_context.is_mongos or not async_client_context.test_commands_enabled: - self.assertEqual(len(commands_started), 1) - continue - - initial_transaction_id = first_attempt.command["txnNumber"] - retry_attempt = commands_started[1] - self.assertIn( - "lsid", - retry_attempt.command, - f"{msg} sent no lsid with {first_attempt.command_name}", - ) - self.assertEqual(retry_attempt.command["lsid"], initial_session_id, msg) - self.assertIn( - "txnNumber", - retry_attempt.command, - f"{msg} sent no txnNumber with {first_attempt.command_name}", - ) - self.assertEqual(retry_attempt.command["txnNumber"], initial_transaction_id, msg) - async def test_supported_single_statement_unsupported_cluster(self): if async_client_context.is_rs or async_client_context.is_mongos: raise SkipTest("This cluster supports retryable writes") diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 81feed4d4..db5ed81e2 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -76,6 +76,7 @@ from pymongo.asynchronous.encryption import AsyncClientEncryption from pymongo.asynchronous.helpers import anext from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.errors import ( + AutoReconnect, BulkWriteError, ClientBulkWriteException, ConfigurationError, @@ -545,15 +546,6 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): or "Cancel server check" in spec["description"] ): self.skipTest("MMAPv1 does not support retryWrites=True") - if ( - "AsyncDatabase-level aggregate with $out includes read preference for 5.0+ server" - in spec["description"] - ): - if async_client_context.version[0] == 8: - self.skipTest("waiting on PYTHON-4356") - if "Aggregate with $out includes read preference for 5.0+ server" in spec["description"]: - if async_client_context.version[0] == 8: - self.skipTest("waiting on PYTHON-4356") if "Client side error in command starting transaction" in spec["description"]: self.skipTest("Implement PYTHON-1894") if "timeoutMS applied to entire download" in spec["description"]: @@ -764,9 +756,10 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): for client in clients: try: await client.admin.command("killAllSessions", []) - except OperationFailure: + except (OperationFailure, AutoReconnect): # "operation was interrupted" by killing the command's # own session. + # On 8.0+ killAllSessions sometimes returns a network error. pass async def _databaseOperation_listCollections(self, target, *args, **kwargs): diff --git a/test/asynchronous/utils_spec_runner.py b/test/asynchronous/utils_spec_runner.py index 4d9c4c8f2..f27f52ec2 100644 --- a/test/asynchronous/utils_spec_runner.py +++ b/test/asynchronous/utils_spec_runner.py @@ -46,7 +46,7 @@ from gridfs.asynchronous.grid_file import AsyncGridFSBucket from pymongo.asynchronous import client_session from pymongo.asynchronous.command_cursor import AsyncCommandCursor from pymongo.asynchronous.cursor import AsyncCursor -from pymongo.errors import BulkWriteError, OperationFailure, PyMongoError +from pymongo.errors import AutoReconnect, BulkWriteError, OperationFailure, PyMongoError from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference from pymongo.results import BulkWriteResult, _WriteResult @@ -343,9 +343,10 @@ class AsyncSpecRunner(AsyncIntegrationTest): for client in clients: try: await client.admin.command("killAllSessions", []) - except OperationFailure: + except (OperationFailure, AutoReconnect): # "operation was interrupted" by killing the command's # own session. + # On 8.0+ killAllSessions sometimes returns a network error. pass def check_command_result(self, expected_result, result): diff --git a/test/crud/unified/client-bulkWrite-replaceOne-sort.json b/test/crud/unified/client-bulkWrite-replaceOne-sort.json index 53218c1f4..b86bc5f94 100644 --- a/test/crud/unified/client-bulkWrite-replaceOne-sort.json +++ b/test/crud/unified/client-bulkWrite-replaceOne-sort.json @@ -3,7 +3,8 @@ "schemaVersion": "1.4", "runOnRequirements": [ { - "minServerVersion": "8.0" + "minServerVersion": "8.0", + "serverless": "forbid" } ], "createEntities": [ diff --git a/test/crud/unified/client-bulkWrite-updateOne-sort.json b/test/crud/unified/client-bulkWrite-updateOne-sort.json index 4a07b8b97..ef75dcb37 100644 --- a/test/crud/unified/client-bulkWrite-updateOne-sort.json +++ b/test/crud/unified/client-bulkWrite-updateOne-sort.json @@ -3,7 +3,8 @@ "schemaVersion": "1.4", "runOnRequirements": [ { - "minServerVersion": "8.0" + "minServerVersion": "8.0", + "serverless": "forbid" } ], "createEntities": [ diff --git a/test/crud/unified/distinct-hint.json b/test/crud/unified/distinct-hint.json new file mode 100644 index 000000000..2a6869cbe --- /dev/null +++ b/test/crud/unified/distinct-hint.json @@ -0,0 +1,139 @@ +{ + "description": "distinct-hint", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "7.1.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "distinct-hint-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "distinct-hint-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "distinct with hint string", + "operations": [ + { + "name": "distinct", + "object": "collection0", + "arguments": { + "fieldName": "x", + "filter": { + "_id": 1 + }, + "hint": "_id_" + }, + "expectResult": [ + 11 + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "distinct": "coll0", + "key": "x", + "query": { + "_id": 1 + }, + "hint": "_id_" + }, + "commandName": "distinct", + "databaseName": "distinct-hint-tests" + } + } + ] + } + ] + }, + { + "description": "distinct with hint document", + "operations": [ + { + "name": "distinct", + "object": "collection0", + "arguments": { + "fieldName": "x", + "filter": { + "_id": 1 + }, + "hint": { + "_id": 1 + } + }, + "expectResult": [ + 11 + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "distinct": "coll0", + "key": "x", + "query": { + "_id": 1 + }, + "hint": { + "_id": 1 + } + }, + "commandName": "distinct", + "databaseName": "distinct-hint-tests" + } + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/estimatedDocumentCount.json b/test/crud/unified/estimatedDocumentCount.json index 1b650c1cb..3577d9006 100644 --- a/test/crud/unified/estimatedDocumentCount.json +++ b/test/crud/unified/estimatedDocumentCount.json @@ -249,7 +249,7 @@ "name": "estimatedDocumentCount", "object": "collection0", "expectError": { - "isError": true + "isClientError": true } } ], diff --git a/test/crud/unified/find.json b/test/crud/unified/find.json index 6bf1e4e44..325cd96c2 100644 --- a/test/crud/unified/find.json +++ b/test/crud/unified/find.json @@ -237,6 +237,68 @@ ] } ] + }, + { + "description": "Find with batchSize equal to limit", + "operations": [ + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": 1 + }, + "limit": 4, + "batchSize": 4 + }, + "expectResult": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "limit": 4, + "batchSize": 5 + }, + "commandName": "find", + "databaseName": "find-tests" + } + } + ] + } + ] } ] } diff --git a/test/crud/unified/findOne.json b/test/crud/unified/findOne.json new file mode 100644 index 000000000..826c0f5df --- /dev/null +++ b/test/crud/unified/findOne.json @@ -0,0 +1,158 @@ +{ + "description": "findOne", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "find-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "find-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + } + ] + } + ], + "tests": [ + { + "description": "FindOne with filter", + "operations": [ + { + "object": "collection0", + "name": "findOne", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": { + "_id": 1, + "x": 11 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll0", + "filter": { + "_id": 1 + }, + "batchSize": { + "$$exists": false + }, + "limit": 1, + "singleBatch": true + }, + "commandName": "find", + "databaseName": "find-tests" + } + } + ] + } + ] + }, + { + "description": "FindOne with filter, sort, and skip", + "operations": [ + { + "object": "collection0", + "name": "findOne", + "arguments": { + "filter": { + "_id": { + "$gt": 2 + } + }, + "sort": { + "_id": 1 + }, + "skip": 2 + }, + "expectResult": { + "_id": 5, + "x": 55 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll0", + "filter": { + "_id": { + "$gt": 2 + } + }, + "sort": { + "_id": 1 + }, + "skip": 2, + "batchSize": { + "$$exists": false + }, + "limit": 1, + "singleBatch": true + }, + "commandName": "find", + "databaseName": "find-tests" + } + } + ] + } + ] + } + ] +} diff --git a/test/retryable_writes/unified/aggregate-out-merge.json b/test/retryable_writes/unified/aggregate-out-merge.json new file mode 100644 index 000000000..c46bf8c31 --- /dev/null +++ b/test/retryable_writes/unified/aggregate-out-merge.json @@ -0,0 +1,144 @@ +{ + "description": "aggregate with $out/$merge does not set txnNumber", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "3.6", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "mergeCollection", + "databaseName": "retryable-writes-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "aggregate with $out does not set txnNumber", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$sort": { + "x": 1 + } + }, + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$out": "outCollection" + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "aggregate with $merge does not set txnNumber", + "runOnRequirements": [ + { + "minServerVersion": "4.1.11" + } + ], + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$sort": { + "x": 1 + } + }, + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$merge": { + "into": "mergeCollection" + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } + ] + } + ] +} diff --git a/test/retryable_writes/unified/bulkWrite.json b/test/retryable_writes/unified/bulkWrite.json index 691321746..f2bd9e0eb 100644 --- a/test/retryable_writes/unified/bulkWrite.json +++ b/test/retryable_writes/unified/bulkWrite.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -121,6 +124,53 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "delete", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { @@ -510,6 +560,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { @@ -926,6 +1003,81 @@ ] } ] + }, + { + "description": "collection bulkWrite with updateMany does not set txnNumber", + "operations": [ + { + "object": "collection0", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "updateMany": { + "filter": {}, + "update": { + "$set": { + "x": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "collection bulkWrite with deleteMany does not set txnNumber", + "operations": [ + { + "object": "collection0", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "deleteMany": { + "filter": {} + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } + ] } ] } diff --git a/test/retryable_writes/unified/client-bulkWrite-serverErrors.json b/test/retryable_writes/unified/client-bulkWrite-serverErrors.json index f58c82bcc..a1f7c8152 100644 --- a/test/retryable_writes/unified/client-bulkWrite-serverErrors.json +++ b/test/retryable_writes/unified/client-bulkWrite-serverErrors.json @@ -428,7 +428,10 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "txnNumber": { + "$$exists": false + } } } } @@ -779,7 +782,10 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "txnNumber": { + "$$exists": false + } } } } @@ -861,7 +867,10 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "txnNumber": { + "$$exists": false + } } } } diff --git a/test/retryable_writes/unified/deleteMany.json b/test/retryable_writes/unified/deleteMany.json index 087576cc0..381f37795 100644 --- a/test/retryable_writes/unified/deleteMany.json +++ b/test/retryable_writes/unified/deleteMany.json @@ -15,7 +15,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": true + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -70,6 +73,23 @@ "databaseName": "retryable-writes-tests", "documents": [] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } ] } ] diff --git a/test/retryable_writes/unified/deleteOne.json b/test/retryable_writes/unified/deleteOne.json index c3aaf8865..9e37ff8bc 100644 --- a/test/retryable_writes/unified/deleteOne.json +++ b/test/retryable_writes/unified/deleteOne.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -88,6 +91,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "delete", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/findOneAndDelete.json b/test/retryable_writes/unified/findOneAndDelete.json index 89dbb9d65..ebfb8ce66 100644 --- a/test/retryable_writes/unified/findOneAndDelete.json +++ b/test/retryable_writes/unified/findOneAndDelete.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -94,6 +97,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/findOneAndReplace.json b/test/retryable_writes/unified/findOneAndReplace.json index 6d1cc1797..638d15a41 100644 --- a/test/retryable_writes/unified/findOneAndReplace.json +++ b/test/retryable_writes/unified/findOneAndReplace.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -98,6 +101,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/findOneAndUpdate.json b/test/retryable_writes/unified/findOneAndUpdate.json index eb88fbe9b..eefe98ae1 100644 --- a/test/retryable_writes/unified/findOneAndUpdate.json +++ b/test/retryable_writes/unified/findOneAndUpdate.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -99,6 +102,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/insertMany.json b/test/retryable_writes/unified/insertMany.json index 47181d0a9..35a18c46c 100644 --- a/test/retryable_writes/unified/insertMany.json +++ b/test/retryable_writes/unified/insertMany.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -107,6 +110,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { @@ -172,6 +202,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/insertOne.json b/test/retryable_writes/unified/insertOne.json index 61957415e..a6afdbf22 100644 --- a/test/retryable_writes/unified/insertOne.json +++ b/test/retryable_writes/unified/insertOne.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -101,6 +104,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/replaceOne.json b/test/retryable_writes/unified/replaceOne.json index e58625bb5..ee6e37d3b 100644 --- a/test/retryable_writes/unified/replaceOne.json +++ b/test/retryable_writes/unified/replaceOne.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -98,6 +101,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/retryable_writes/unified/unacknowledged-write-concern.json b/test/retryable_writes/unified/unacknowledged-write-concern.json new file mode 100644 index 000000000..eaa114acf --- /dev/null +++ b/test/retryable_writes/unified/unacknowledged-write-concern.json @@ -0,0 +1,77 @@ +{ + "description": "unacknowledged write does not set txnNumber", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "3.6", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0", + "collectionOptions": { + "writeConcern": { + "w": 0 + } + } + } + } + ], + "tests": [ + { + "description": "unacknowledged write does not set txnNumber", + "operations": [ + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } + ] + } + ] +} diff --git a/test/retryable_writes/unified/updateMany.json b/test/retryable_writes/unified/updateMany.json index 260b7ad1c..12c5204ee 100644 --- a/test/retryable_writes/unified/updateMany.json +++ b/test/retryable_writes/unified/updateMany.json @@ -15,7 +15,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": true + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -86,6 +89,23 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": false + } + } + } + } + ] + } ] } ] diff --git a/test/retryable_writes/unified/updateOne.json b/test/retryable_writes/unified/updateOne.json index 7947cef3c..99ffba8e2 100644 --- a/test/retryable_writes/unified/updateOne.json +++ b/test/retryable_writes/unified/updateOne.json @@ -13,7 +13,10 @@ { "client": { "id": "client0", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] } }, { @@ -99,6 +102,33 @@ } ] } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "update", + "command": { + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } ] }, { diff --git a/test/test_connection_monitoring.py b/test/test_connection_monitoring.py index d576a1184..05411d17b 100644 --- a/test/test_connection_monitoring.py +++ b/test/test_connection_monitoring.py @@ -216,11 +216,6 @@ class TestCMAP(IntegrationTest): def run_scenario(self, scenario_def, test): """Run a CMAP spec test.""" - if ( - scenario_def["description"] - == "clear with interruptInUseConnections = true closes pending connections" - ): - self.skipTest("Skip pending PYTHON-4414") self.logs: list = [] self.assertEqual(scenario_def["version"], 1) self.assertIn(scenario_def["style"], ["unit", "integration"]) diff --git a/test/test_retryable_reads.py b/test/test_retryable_reads.py index d4951db5e..9c3f6b170 100644 --- a/test/test_retryable_reads.py +++ b/test/test_retryable_reads.py @@ -174,9 +174,8 @@ class TestRetryableReads(IntegrationTest): retryReads=True, ) - with self.fail_point(fail_command): - with self.assertRaises(AutoReconnect): - client.t.t.find_one({}) + with self.assertRaises(AutoReconnect): + client.t.t.find_one({}) # Disable failpoints on each mongos for client in mongos_clients: diff --git a/test/test_retryable_writes.py b/test/test_retryable_writes.py index 5df6c41f7..74f3c23e5 100644 --- a/test/test_retryable_writes.py +++ b/test/test_retryable_writes.py @@ -1,4 +1,4 @@ -# Copyright 2017 MongoDB, Inc. +# Copyright 2017-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,7 +65,6 @@ from pymongo.operations import ( UpdateMany, UpdateOne, ) -from pymongo.synchronous.mongo_client import MongoClient from pymongo.write_concern import WriteConcern _IS_SYNC = True @@ -226,47 +225,6 @@ class TestRetryableWrites(IgnoreDeprecationsTest): f"{msg} sent txnNumber with {event.command_name}", ) - @client_context.require_no_standalone - def test_supported_single_statement_supported_cluster(self): - for method, args, kwargs in retryable_single_statement_ops(self.db.retryable_write_test): - msg = f"{method.__name__}(*{args!r}, **{kwargs!r})" - self.listener.reset() - method(*args, **kwargs) - commands_started = self.listener.started_events - self.assertEqual(len(self.listener.succeeded_events), 1, msg) - first_attempt = commands_started[0] - self.assertIn( - "lsid", - first_attempt.command, - f"{msg} sent no lsid with {first_attempt.command_name}", - ) - initial_session_id = first_attempt.command["lsid"] - self.assertIn( - "txnNumber", - first_attempt.command, - f"{msg} sent no txnNumber with {first_attempt.command_name}", - ) - - # There should be no retry when the failpoint is not active. - if client_context.is_mongos or not client_context.test_commands_enabled: - self.assertEqual(len(commands_started), 1) - continue - - initial_transaction_id = first_attempt.command["txnNumber"] - retry_attempt = commands_started[1] - self.assertIn( - "lsid", - retry_attempt.command, - f"{msg} sent no lsid with {first_attempt.command_name}", - ) - self.assertEqual(retry_attempt.command["lsid"], initial_session_id, msg) - self.assertIn( - "txnNumber", - retry_attempt.command, - f"{msg} sent no txnNumber with {first_attempt.command_name}", - ) - self.assertEqual(retry_attempt.command["txnNumber"], initial_transaction_id, msg) - def test_supported_single_statement_unsupported_cluster(self): if client_context.is_rs or client_context.is_mongos: raise SkipTest("This cluster supports retryable writes") diff --git a/test/unified_format.py b/test/unified_format.py index 395d40b2d..3489a8ac8 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -69,6 +69,7 @@ from gridfs import GridFSBucket, GridOut from pymongo import ASCENDING, CursorType, MongoClient, _csot from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.errors import ( + AutoReconnect, BulkWriteError, ClientBulkWriteException, ConfigurationError, @@ -543,15 +544,6 @@ class UnifiedSpecTestMixinV1(IntegrationTest): or "Cancel server check" in spec["description"] ): self.skipTest("MMAPv1 does not support retryWrites=True") - if ( - "Database-level aggregate with $out includes read preference for 5.0+ server" - in spec["description"] - ): - if client_context.version[0] == 8: - self.skipTest("waiting on PYTHON-4356") - if "Aggregate with $out includes read preference for 5.0+ server" in spec["description"]: - if client_context.version[0] == 8: - self.skipTest("waiting on PYTHON-4356") if "Client side error in command starting transaction" in spec["description"]: self.skipTest("Implement PYTHON-1894") if "timeoutMS applied to entire download" in spec["description"]: @@ -760,9 +752,10 @@ class UnifiedSpecTestMixinV1(IntegrationTest): for client in clients: try: client.admin.command("killAllSessions", []) - except OperationFailure: + except (OperationFailure, AutoReconnect): # "operation was interrupted" by killing the command's # own session. + # On 8.0+ killAllSessions sometimes returns a network error. pass def _databaseOperation_listCollections(self, target, *args, **kwargs): diff --git a/test/utils.py b/test/utils.py index 766f209de..9b326e5d7 100644 --- a/test/utils.py +++ b/test/utils.py @@ -925,35 +925,6 @@ def parse_spec_options(opts): if "maxCommitTimeMS" in opts: opts["max_commit_time_ms"] = opts.pop("maxCommitTimeMS") - if "hint" in opts: - hint = opts.pop("hint") - if not isinstance(hint, str): - hint = list(hint.items()) - opts["hint"] = hint - - # Properly format 'hint' arguments for the Bulk API tests. - if "requests" in opts: - reqs = opts.pop("requests") - for req in reqs: - if "name" in req: - # CRUD v2 format - args = req.pop("arguments", {}) - if "hint" in args: - hint = args.pop("hint") - if not isinstance(hint, str): - hint = list(hint.items()) - args["hint"] = hint - req["arguments"] = args - else: - # Unified test format - bulk_model, spec = next(iter(req.items())) - if "hint" in spec: - hint = spec.pop("hint") - if not isinstance(hint, str): - hint = list(hint.items()) - spec["hint"] = hint - opts["requests"] = reqs - return dict(opts) diff --git a/test/utils_spec_runner.py b/test/utils_spec_runner.py index 8a061de0b..8b2679d77 100644 --- a/test/utils_spec_runner.py +++ b/test/utils_spec_runner.py @@ -43,7 +43,7 @@ from bson.int64 import Int64 from bson.son import SON from gridfs import GridFSBucket from gridfs.synchronous.grid_file import GridFSBucket -from pymongo.errors import BulkWriteError, OperationFailure, PyMongoError +from pymongo.errors import AutoReconnect, BulkWriteError, OperationFailure, PyMongoError from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference from pymongo.results import BulkWriteResult, _WriteResult @@ -343,9 +343,10 @@ class SpecRunner(IntegrationTest): for client in clients: try: client.admin.command("killAllSessions", []) - except OperationFailure: + except (OperationFailure, AutoReconnect): # "operation was interrupted" by killing the command's # own session. + # On 8.0+ killAllSessions sometimes returns a network error. pass def check_command_result(self, expected_result, result):