From 9a71be1615048a6f3aa2d23fa03693923572eeee Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 17 Sep 2024 17:54:09 -0700 Subject: [PATCH 1/8] PYTHON-4740 Convert asyncio.TimeoutError to socket.timeout for compat (#1864) --- pymongo/asynchronous/bulk.py | 4 -- pymongo/asynchronous/client_bulk.py | 6 +- pymongo/network_layer.py | 104 +++++++++++++++------------- pymongo/synchronous/bulk.py | 4 -- pymongo/synchronous/client_bulk.py | 6 +- 5 files changed, 56 insertions(+), 68 deletions(-) diff --git a/pymongo/asynchronous/bulk.py b/pymongo/asynchronous/bulk.py index 9fd673693..9d33a990e 100644 --- a/pymongo/asynchronous/bulk.py +++ b/pymongo/asynchronous/bulk.py @@ -313,8 +313,6 @@ class _AsyncBulk: if isinstance(exc, (NotPrimaryError, OperationFailure)): await client._process_response(exc.details, bwc.session) # type: ignore[arg-type] raise - finally: - bwc.start_time = datetime.datetime.now() return reply # type: ignore[return-value] async def unack_write( @@ -403,8 +401,6 @@ class _AsyncBulk: assert bwc.start_time is not None bwc._fail(request_id, failure, duration) raise - finally: - bwc.start_time = datetime.datetime.now() return result # type: ignore[return-value] async def _execute_batch_unack( diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 15a0369f4..dc800c954 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -319,8 +319,6 @@ class _AsyncClientBulk: await self.client._process_response(exc.details, bwc.session) # type: ignore[arg-type] else: await self.client._process_response({}, bwc.session) # type: ignore[arg-type] - finally: - bwc.start_time = datetime.datetime.now() return reply # type: ignore[return-value] async def unack_write( @@ -410,9 +408,7 @@ class _AsyncClientBulk: bwc._fail(request_id, failure, duration) # Top-level error will be embedded in ClientBulkWriteException. reply = {"error": exc} - finally: - bwc.start_time = datetime.datetime.now() - return result # type: ignore[return-value] + return reply async def _execute_batch_unack( self, diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index d99b4fee4..82a6228ac 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -64,65 +64,69 @@ async def async_sendall(sock: Union[socket.socket, _sslConn], buf: bytes) -> Non loop = asyncio.get_event_loop() try: if _HAVE_SSL and isinstance(sock, (SSLSocket, _sslConn)): - if sys.platform == "win32": - await asyncio.wait_for(_async_sendall_ssl_windows(sock, buf), timeout=timeout) - else: - await asyncio.wait_for(_async_sendall_ssl(sock, buf, loop), timeout=timeout) + await asyncio.wait_for(_async_sendall_ssl(sock, buf, loop), timeout=timeout) else: await asyncio.wait_for(loop.sock_sendall(sock, buf), timeout=timeout) # type: ignore[arg-type] + except asyncio.TimeoutError as exc: + # Convert the asyncio.wait_for timeout error to socket.timeout which pool.py understands. + raise socket.timeout("timed out") from exc finally: sock.settimeout(timeout) -async def _async_sendall_ssl( - sock: Union[socket.socket, _sslConn], buf: bytes, loop: AbstractEventLoop -) -> None: - view = memoryview(buf) - fd = sock.fileno() - sent = 0 +if sys.platform != "win32": - def _is_ready(fut: Future) -> None: - loop.remove_writer(fd) - loop.remove_reader(fd) - if fut.done(): - return - fut.set_result(None) + async def _async_sendall_ssl( + sock: Union[socket.socket, _sslConn], buf: bytes, loop: AbstractEventLoop + ) -> None: + view = memoryview(buf) + fd = sock.fileno() + sent = 0 - while sent < len(buf): - try: - sent += sock.send(view[sent:]) - except BLOCKING_IO_ERRORS as exc: - fd = sock.fileno() - # Check for closed socket. - if fd == -1: - raise SSLError("Underlying socket has been closed") from None - if isinstance(exc, BLOCKING_IO_READ_ERROR): - fut = loop.create_future() - loop.add_reader(fd, _is_ready, fut) - await fut - if isinstance(exc, BLOCKING_IO_WRITE_ERROR): - fut = loop.create_future() - loop.add_writer(fd, _is_ready, fut) - await fut - if _HAVE_PYOPENSSL and isinstance(exc, BLOCKING_IO_LOOKUP_ERROR): - fut = loop.create_future() - loop.add_reader(fd, _is_ready, fut) - loop.add_writer(fd, _is_ready, fut) - await fut + def _is_ready(fut: Future) -> None: + loop.remove_writer(fd) + loop.remove_reader(fd) + if fut.done(): + return + fut.set_result(None) - -# The default Windows asyncio event loop does not support loop.add_reader/add_writer: https://docs.python.org/3/library/asyncio-platforms.html#asyncio-platform-support -async def _async_sendall_ssl_windows(sock: Union[socket.socket, _sslConn], buf: bytes) -> None: - view = memoryview(buf) - total_length = len(buf) - total_sent = 0 - while total_sent < total_length: - try: - sent = sock.send(view[total_sent:]) - except BLOCKING_IO_ERRORS: - await asyncio.sleep(0.5) - sent = 0 - total_sent += sent + while sent < len(buf): + try: + sent += sock.send(view[sent:]) + except BLOCKING_IO_ERRORS as exc: + fd = sock.fileno() + # Check for closed socket. + if fd == -1: + raise SSLError("Underlying socket has been closed") from None + if isinstance(exc, BLOCKING_IO_READ_ERROR): + fut = loop.create_future() + loop.add_reader(fd, _is_ready, fut) + await fut + if isinstance(exc, BLOCKING_IO_WRITE_ERROR): + fut = loop.create_future() + loop.add_writer(fd, _is_ready, fut) + await fut + if _HAVE_PYOPENSSL and isinstance(exc, BLOCKING_IO_LOOKUP_ERROR): + fut = loop.create_future() + loop.add_reader(fd, _is_ready, fut) + loop.add_writer(fd, _is_ready, fut) + await fut +else: + # The default Windows asyncio event loop does not support loop.add_reader/add_writer: + # https://docs.python.org/3/library/asyncio-platforms.html#asyncio-platform-support + async def _async_sendall_ssl( + sock: Union[socket.socket, _sslConn], buf: bytes, dummy: AbstractEventLoop + ) -> None: + view = memoryview(buf) + total_length = len(buf) + total_sent = 0 + while total_sent < total_length: + try: + sent = sock.send(view[total_sent:]) + except BLOCKING_IO_ERRORS: + await asyncio.sleep(0.5) + sent = 0 + total_sent += sent def sendall(sock: Union[socket.socket, _sslConn], buf: bytes) -> None: diff --git a/pymongo/synchronous/bulk.py b/pymongo/synchronous/bulk.py index 27fcff620..c658157ea 100644 --- a/pymongo/synchronous/bulk.py +++ b/pymongo/synchronous/bulk.py @@ -313,8 +313,6 @@ class _Bulk: if isinstance(exc, (NotPrimaryError, OperationFailure)): client._process_response(exc.details, bwc.session) # type: ignore[arg-type] raise - finally: - bwc.start_time = datetime.datetime.now() return reply # type: ignore[return-value] def unack_write( @@ -403,8 +401,6 @@ class _Bulk: assert bwc.start_time is not None bwc._fail(request_id, failure, duration) raise - finally: - bwc.start_time = datetime.datetime.now() return result # type: ignore[return-value] def _execute_batch_unack( diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index 23af231d1..f41f0203f 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -319,8 +319,6 @@ class _ClientBulk: self.client._process_response(exc.details, bwc.session) # type: ignore[arg-type] else: self.client._process_response({}, bwc.session) # type: ignore[arg-type] - finally: - bwc.start_time = datetime.datetime.now() return reply # type: ignore[return-value] def unack_write( @@ -410,9 +408,7 @@ class _ClientBulk: bwc._fail(request_id, failure, duration) # Top-level error will be embedded in ClientBulkWriteException. reply = {"error": exc} - finally: - bwc.start_time = datetime.datetime.now() - return result # type: ignore[return-value] + return reply def _execute_batch_unack( self, From 6d472a10a1a0cf511ef42c57cda5545ff78d2c27 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Sep 2024 20:00:06 -0500 Subject: [PATCH 2/8] PYTHON-4738 Skip encryption test_fork on PyPy (#1865) --- test/asynchronous/test_encryption.py | 1 + test/test_encryption.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index f29b0f824..c3f622338 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -380,6 +380,7 @@ class TestClientSimple(AsyncEncryptionIntegrationTest): is_greenthread_patched(), "gevent and eventlet do not support POSIX-style forking.", ) + @unittest.skipIf("PyPy" in sys.version, "PYTHON-4738 fails often on PyPy") @async_client_context.require_sync async def test_fork(self): opts = AutoEncryptionOpts(KMS_PROVIDERS, "keyvault.datakeys") diff --git a/test/test_encryption.py b/test/test_encryption.py index 512c92f4d..43c85e2c5 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -380,6 +380,7 @@ class TestClientSimple(EncryptionIntegrationTest): is_greenthread_patched(), "gevent and eventlet do not support POSIX-style forking.", ) + @unittest.skipIf("PyPy" in sys.version, "PYTHON-4738 fails often on PyPy") @client_context.require_sync def test_fork(self): opts = AutoEncryptionOpts(KMS_PROVIDERS, "keyvault.datakeys") From 2c432b580baf842318d9c0628fcda001da248494 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 18 Sep 2024 09:23:07 -0400 Subject: [PATCH 3/8] PYTHON-4768 - Fix atlas connection tests and cleanup uses of raw MongoClients in tests (#1867) --- test/atlas/test_connection.py | 34 +++++++++---------- test/mockupdb/test_auth_recovering_member.py | 7 ++-- test/mockupdb/test_cluster_time.py | 9 +++-- test/mockupdb/test_cursor_namespace.py | 9 ++--- test/mockupdb/test_getmore_sharded.py | 6 ++-- test/mockupdb/test_initial_ismaster.py | 6 ++-- test/mockupdb/test_list_indexes.py | 7 ++-- test/mockupdb/test_max_staleness.py | 9 +++-- test/mockupdb/test_mixed_version_sharded.py | 5 +-- test/mockupdb/test_op_msg.py | 5 +-- test/mockupdb/test_op_msg_read_preference.py | 11 +++--- test/mockupdb/test_query_read_pref_sharded.py | 6 ++-- test/mockupdb/test_reset_and_request_check.py | 5 +-- test/test_index_management.py | 10 +++--- test/test_retryable_reads.py | 11 +++--- 15 files changed, 69 insertions(+), 71 deletions(-) diff --git a/test/atlas/test_connection.py b/test/atlas/test_connection.py index 762ac3011..4dcbba6d1 100644 --- a/test/atlas/test_connection.py +++ b/test/atlas/test_connection.py @@ -19,6 +19,7 @@ import os import sys import unittest from collections import defaultdict +from test import PyMongoTestCase import pytest @@ -46,38 +47,37 @@ URIS = { } -def connect(uri): - if not uri: - raise Exception("Must set env variable to test.") - client = pymongo.MongoClient(uri) - # No TLS error - client.admin.command("ping") - # No auth error - client.test.test.count_documents({}) +class TestAtlasConnect(PyMongoTestCase): + def connect(self, uri): + if not uri: + raise Exception("Must set env variable to test.") + client = self.simple_client(uri) + # No TLS error + client.admin.command("ping") + # No auth error + client.test.test.count_documents({}) - -class TestAtlasConnect(unittest.TestCase): @unittest.skipUnless(HAS_SNI, "Free tier requires SNI support") def test_free_tier(self): - connect(URIS["ATLAS_FREE"]) + self.connect(URIS["ATLAS_FREE"]) def test_replica_set(self): - connect(URIS["ATLAS_REPL"]) + self.connect(URIS["ATLAS_REPL"]) def test_sharded_cluster(self): - connect(URIS["ATLAS_SHRD"]) + self.connect(URIS["ATLAS_SHRD"]) def test_tls_11(self): - connect(URIS["ATLAS_TLS11"]) + self.connect(URIS["ATLAS_TLS11"]) def test_tls_12(self): - connect(URIS["ATLAS_TLS12"]) + self.connect(URIS["ATLAS_TLS12"]) def test_serverless(self): - connect(URIS["ATLAS_SERVERLESS"]) + self.connect(URIS["ATLAS_SERVERLESS"]) def connect_srv(self, uri): - connect(uri) + self.connect(uri) self.assertIn("mongodb+srv://", uri) @unittest.skipUnless(HAS_SNI, "Free tier requires SNI support") diff --git a/test/mockupdb/test_auth_recovering_member.py b/test/mockupdb/test_auth_recovering_member.py index 95a83a11a..6eadafaf3 100644 --- a/test/mockupdb/test_auth_recovering_member.py +++ b/test/mockupdb/test_auth_recovering_member.py @@ -14,6 +14,7 @@ from __future__ import annotations import unittest +from test import PyMongoTestCase import pytest @@ -30,7 +31,7 @@ from pymongo.errors import ServerSelectionTimeoutError pytestmark = pytest.mark.mockupdb -class TestAuthRecoveringMember(unittest.TestCase): +class TestAuthRecoveringMember(PyMongoTestCase): def test_auth_recovering_member(self): # Test that we don't attempt auth against a recovering RS member. server = MockupDB() @@ -48,12 +49,10 @@ class TestAuthRecoveringMember(unittest.TestCase): server.run() self.addCleanup(server.stop) - client = MongoClient( + client = self.simple_client( server.uri, replicaSet="rs", serverSelectionTimeoutMS=100, socketTimeoutMS=100 ) - self.addCleanup(client.close) - # Should see there's no primary or secondary and raise selection timeout # error. If it raises AutoReconnect we know it actually tried the # server, and that's wrong. diff --git a/test/mockupdb/test_cluster_time.py b/test/mockupdb/test_cluster_time.py index 979484317..761415951 100644 --- a/test/mockupdb/test_cluster_time.py +++ b/test/mockupdb/test_cluster_time.py @@ -16,6 +16,7 @@ from __future__ import annotations import unittest +from test import PyMongoTestCase import pytest @@ -34,7 +35,7 @@ from pymongo.errors import OperationFailure pytestmark = pytest.mark.mockupdb -class TestClusterTime(unittest.TestCase): +class TestClusterTime(PyMongoTestCase): def cluster_time_conversation(self, callback, replies, max_wire_version=6): cluster_time = Timestamp(0, 0) server = MockupDB() @@ -52,8 +53,7 @@ class TestClusterTime(unittest.TestCase): server.run() self.addCleanup(server.stop) - client = MongoClient(server.uri) - self.addCleanup(client.close) + client = self.simple_client(server.uri) with going(callback, client): for reply in replies: @@ -118,8 +118,7 @@ class TestClusterTime(unittest.TestCase): server.run() self.addCleanup(server.stop) - client = MongoClient(server.uri, heartbeatFrequencyMS=500) - self.addCleanup(client.close) + client = self.simple_client(server.uri, heartbeatFrequencyMS=500) request = server.receives("ismaster") # No $clusterTime in first ismaster, only in subsequent ones diff --git a/test/mockupdb/test_cursor_namespace.py b/test/mockupdb/test_cursor_namespace.py index 57a98373f..455a3a923 100644 --- a/test/mockupdb/test_cursor_namespace.py +++ b/test/mockupdb/test_cursor_namespace.py @@ -16,6 +16,7 @@ from __future__ import annotations import unittest +from test import PyMongoTestCase import pytest @@ -32,7 +33,7 @@ from pymongo import MongoClient pytestmark = pytest.mark.mockupdb -class TestCursorNamespace(unittest.TestCase): +class TestCursorNamespace(PyMongoTestCase): server: MockupDB client: MongoClient @@ -40,7 +41,7 @@ class TestCursorNamespace(unittest.TestCase): def setUpClass(cls): cls.server = MockupDB(auto_ismaster={"maxWireVersion": 6}) cls.server.run() - cls.client = MongoClient(cls.server.uri) + cls.client = cls.unmanaged_simple_client(cls.server.uri) @classmethod def tearDownClass(cls): @@ -88,7 +89,7 @@ class TestCursorNamespace(unittest.TestCase): self._test_cursor_namespace(op, "listIndexes") -class TestKillCursorsNamespace(unittest.TestCase): +class TestKillCursorsNamespace(PyMongoTestCase): server: MockupDB client: MongoClient @@ -96,7 +97,7 @@ class TestKillCursorsNamespace(unittest.TestCase): def setUpClass(cls): cls.server = MockupDB(auto_ismaster={"maxWireVersion": 6}) cls.server.run() - cls.client = MongoClient(cls.server.uri) + cls.client = cls.unmanaged_simple_client(cls.server.uri) @classmethod def tearDownClass(cls): diff --git a/test/mockupdb/test_getmore_sharded.py b/test/mockupdb/test_getmore_sharded.py index cf26b10a1..8ba291e4a 100644 --- a/test/mockupdb/test_getmore_sharded.py +++ b/test/mockupdb/test_getmore_sharded.py @@ -17,6 +17,7 @@ from __future__ import annotations import unittest from queue import Queue +from test import PyMongoTestCase import pytest @@ -33,7 +34,7 @@ from pymongo import MongoClient pytestmark = pytest.mark.mockupdb -class TestGetmoreSharded(unittest.TestCase): +class TestGetmoreSharded(PyMongoTestCase): def test_getmore_sharded(self): servers = [MockupDB(), MockupDB()] @@ -47,11 +48,10 @@ class TestGetmoreSharded(unittest.TestCase): server.run() self.addCleanup(server.stop) - client = MongoClient( + client = self.simple_client( "mongodb://%s:%d,%s:%d" % (servers[0].host, servers[0].port, servers[1].host, servers[1].port) ) - self.addCleanup(client.close) collection = client.db.collection cursor = collection.find() with going(next, cursor): diff --git a/test/mockupdb/test_initial_ismaster.py b/test/mockupdb/test_initial_ismaster.py index 8046f08db..3eae98716 100644 --- a/test/mockupdb/test_initial_ismaster.py +++ b/test/mockupdb/test_initial_ismaster.py @@ -15,6 +15,7 @@ from __future__ import annotations import time import unittest +from test import PyMongoTestCase import pytest @@ -31,15 +32,14 @@ from pymongo import MongoClient pytestmark = pytest.mark.mockupdb -class TestInitialIsMaster(unittest.TestCase): +class TestInitialIsMaster(PyMongoTestCase): def test_initial_ismaster(self): server = MockupDB() server.run() self.addCleanup(server.stop) start = time.time() - client = MongoClient(server.uri) - self.addCleanup(client.close) + client = self.simple_client(server.uri) # A single ismaster is enough for the client to be connected. self.assertFalse(client.nodes) diff --git a/test/mockupdb/test_list_indexes.py b/test/mockupdb/test_list_indexes.py index ce6db9e10..ff3363664 100644 --- a/test/mockupdb/test_list_indexes.py +++ b/test/mockupdb/test_list_indexes.py @@ -16,6 +16,7 @@ from __future__ import annotations import unittest +from test import PyMongoTestCase import pytest @@ -28,18 +29,16 @@ except ImportError: from bson import SON -from pymongo import MongoClient pytestmark = pytest.mark.mockupdb -class TestListIndexes(unittest.TestCase): +class TestListIndexes(PyMongoTestCase): def test_list_indexes_command(self): server = MockupDB(auto_ismaster={"maxWireVersion": 6}) server.run() self.addCleanup(server.stop) - client = MongoClient(server.uri) - self.addCleanup(client.close) + client = self.simple_client(server.uri) with going(client.test.collection.list_indexes) as cursor: request = server.receives(listIndexes="collection", namespace="test") request.reply({"cursor": {"firstBatch": [{"name": "index_0"}], "id": 123}}) diff --git a/test/mockupdb/test_max_staleness.py b/test/mockupdb/test_max_staleness.py index 40cf7ef00..7275aaf44 100644 --- a/test/mockupdb/test_max_staleness.py +++ b/test/mockupdb/test_max_staleness.py @@ -14,6 +14,7 @@ from __future__ import annotations import unittest +from test import PyMongoTestCase import pytest @@ -30,7 +31,7 @@ from pymongo import MongoClient pytestmark = pytest.mark.mockupdb -class TestMaxStalenessMongos(unittest.TestCase): +class TestMaxStalenessMongos(PyMongoTestCase): def test_mongos(self): mongos = MockupDB() mongos.autoresponds("ismaster", maxWireVersion=6, ismaster=True, msg="isdbgrid") @@ -40,8 +41,7 @@ class TestMaxStalenessMongos(unittest.TestCase): # No maxStalenessSeconds. uri = "mongodb://localhost:%d/?readPreference=secondary" % mongos.port - client = MongoClient(uri) - self.addCleanup(client.close) + client = self.simple_client(uri) with going(client.db.coll.find_one) as future: request = mongos.receives() self.assertNotIn("maxStalenessSeconds", request.doc["$readPreference"]) @@ -60,8 +60,7 @@ class TestMaxStalenessMongos(unittest.TestCase): "&maxStalenessSeconds=1" % mongos.port ) - client = MongoClient(uri) - self.addCleanup(client.close) + client = self.simple_client(uri) with going(client.db.coll.find_one) as future: request = mongos.receives() self.assertEqual(1, request.doc["$readPreference"]["maxStalenessSeconds"]) diff --git a/test/mockupdb/test_mixed_version_sharded.py b/test/mockupdb/test_mixed_version_sharded.py index 72b42deac..99d428b5d 100644 --- a/test/mockupdb/test_mixed_version_sharded.py +++ b/test/mockupdb/test_mixed_version_sharded.py @@ -18,6 +18,7 @@ from __future__ import annotations import time import unittest from queue import Queue +from test import PyMongoTestCase import pytest @@ -35,7 +36,7 @@ from pymongo import MongoClient pytestmark = pytest.mark.mockupdb -class TestMixedVersionSharded(unittest.TestCase): +class TestMixedVersionSharded(PyMongoTestCase): def setup_server(self, upgrade): self.mongos_old, self.mongos_new = MockupDB(), MockupDB() @@ -62,7 +63,7 @@ class TestMixedVersionSharded(unittest.TestCase): self.mongos_new.address_string, ) - self.client = MongoClient(self.mongoses_uri) + self.client = self.simple_client(self.mongoses_uri) def tearDown(self): if hasattr(self, "client") and self.client: diff --git a/test/mockupdb/test_op_msg.py b/test/mockupdb/test_op_msg.py index 776d1644d..4b85c5a48 100644 --- a/test/mockupdb/test_op_msg.py +++ b/test/mockupdb/test_op_msg.py @@ -15,6 +15,7 @@ from __future__ import annotations import unittest from collections import namedtuple +from test import PyMongoTestCase import pytest @@ -273,7 +274,7 @@ else: operations_312 = [] -class TestOpMsg(unittest.TestCase): +class TestOpMsg(PyMongoTestCase): server: MockupDB client: MongoClient @@ -281,7 +282,7 @@ class TestOpMsg(unittest.TestCase): def setUpClass(cls): cls.server = MockupDB(auto_ismaster=True, max_wire_version=8) cls.server.run() - cls.client = MongoClient(cls.server.uri) + cls.client = cls.unmanaged_simple_client(cls.server.uri) @classmethod def tearDownClass(cls): diff --git a/test/mockupdb/test_op_msg_read_preference.py b/test/mockupdb/test_op_msg_read_preference.py index c7c7037f2..86293d0c0 100644 --- a/test/mockupdb/test_op_msg_read_preference.py +++ b/test/mockupdb/test_op_msg_read_preference.py @@ -16,6 +16,7 @@ from __future__ import annotations import copy import itertools import unittest +from test import PyMongoTestCase from typing import Any import pytest @@ -39,7 +40,7 @@ from pymongo.read_preferences import ( pytestmark = pytest.mark.mockupdb -class OpMsgReadPrefBase(unittest.TestCase): +class OpMsgReadPrefBase(PyMongoTestCase): single_mongod = False primary: MockupDB secondary: MockupDB @@ -53,8 +54,7 @@ class OpMsgReadPrefBase(unittest.TestCase): setattr(cls, test_name, test) def setup_client(self, read_preference): - client = MongoClient(self.primary.uri, read_preference=read_preference) - self.addCleanup(client.close) + client = self.simple_client(self.primary.uri, read_preference=read_preference) return client @@ -115,12 +115,13 @@ class TestOpMsgReplicaSet(OpMsgReadPrefBase): setattr(cls, test_name, test) def setup_client(self, read_preference): - client = MongoClient(self.primary.uri, replicaSet="rs", read_preference=read_preference) + client = self.simple_client( + self.primary.uri, replicaSet="rs", read_preference=read_preference + ) # Run a command on a secondary to discover the topology. This ensures # that secondaryPreferred commands will select the secondary. client.admin.command("ismaster", read_preference=ReadPreference.SECONDARY) - self.addCleanup(client.close) return client diff --git a/test/mockupdb/test_query_read_pref_sharded.py b/test/mockupdb/test_query_read_pref_sharded.py index 6276ee778..676e71b71 100644 --- a/test/mockupdb/test_query_read_pref_sharded.py +++ b/test/mockupdb/test_query_read_pref_sharded.py @@ -16,6 +16,7 @@ from __future__ import annotations import unittest +from test import PyMongoTestCase import pytest @@ -40,7 +41,7 @@ from pymongo.read_preferences import ( pytestmark = pytest.mark.mockupdb -class TestQueryAndReadModeSharded(unittest.TestCase): +class TestQueryAndReadModeSharded(PyMongoTestCase): def test_query_and_read_mode_sharded_op_msg(self): """Test OP_MSG sends non-primary $readPreference and never $query.""" server = MockupDB() @@ -50,8 +51,7 @@ class TestQueryAndReadModeSharded(unittest.TestCase): server.run() self.addCleanup(server.stop) - client = MongoClient(server.uri) - self.addCleanup(client.close) + client = self.simple_client(server.uri) read_prefs = ( Primary(), diff --git a/test/mockupdb/test_reset_and_request_check.py b/test/mockupdb/test_reset_and_request_check.py index 2919025f0..dd6ad46b1 100644 --- a/test/mockupdb/test_reset_and_request_check.py +++ b/test/mockupdb/test_reset_and_request_check.py @@ -16,6 +16,7 @@ from __future__ import annotations import itertools import time import unittest +from test import PyMongoTestCase import pytest @@ -37,7 +38,7 @@ from pymongo.server_type import SERVER_TYPE pytestmark = pytest.mark.mockupdb -class TestResetAndRequestCheck(unittest.TestCase): +class TestResetAndRequestCheck(PyMongoTestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.ismaster_time = 0.0 @@ -58,7 +59,7 @@ class TestResetAndRequestCheck(unittest.TestCase): kwargs = {"socketTimeoutMS": 100} # Disable retryable reads when pymongo supports it. kwargs["retryReads"] = False - self.client = MongoClient(self.server.uri, **kwargs) # type: ignore + self.client = self.simple_client(self.server.uri, **kwargs) # type: ignore wait_until(lambda: self.client.nodes, "connect to standalone") def tearDown(self): diff --git a/test/test_index_management.py b/test/test_index_management.py index 426d0c734..ec1e36373 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -25,11 +25,10 @@ import pytest sys.path[0:0] = [""] -from test import IntegrationTest, unittest +from test import IntegrationTest, PyMongoTestCase, unittest from test.unified_format import generate_test_classes from test.utils import AllowListEventListener, EventListener -from pymongo import MongoClient from pymongo.errors import OperationFailure from pymongo.operations import SearchIndexModel from pymongo.read_concern import ReadConcern @@ -47,8 +46,7 @@ class TestCreateSearchIndex(IntegrationTest): if not os.environ.get("TEST_INDEX_MANAGEMENT"): raise unittest.SkipTest("Skipping index management tests") listener = AllowListEventListener("createSearchIndexes") - client = MongoClient(event_listeners=[listener]) - self.addCleanup(client.close) + client = self.simple_client(event_listeners=[listener]) coll = client.test.test coll.drop() definition = dict(mappings=dict(dynamic=True)) @@ -79,7 +77,7 @@ class TestCreateSearchIndex(IntegrationTest): ) -class SearchIndexIntegrationBase(unittest.TestCase): +class SearchIndexIntegrationBase(PyMongoTestCase): db_name = "test_search_index_base" @classmethod @@ -91,7 +89,7 @@ class SearchIndexIntegrationBase(unittest.TestCase): username = os.environ["DB_USER"] password = os.environ["DB_PASSWORD"] cls.listener = listener = EventListener() - cls.client = MongoClient( + cls.client = cls.unmanaged_simple_client( url, username=username, password=password, event_listeners=[listener] ) cls.client.drop_database(_NAME) diff --git a/test/test_retryable_reads.py b/test/test_retryable_reads.py index a1c72bb7b..b4fafe465 100644 --- a/test/test_retryable_reads.py +++ b/test/test_retryable_reads.py @@ -43,7 +43,6 @@ from pymongo.monitoring import ( ConnectionCheckOutFailedReason, PoolClearedEvent, ) -from pymongo.synchronous.mongo_client import MongoClient # Location of JSON test specifications. _TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryable_reads", "legacy") @@ -51,19 +50,19 @@ _TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryabl class TestClientOptions(PyMongoTestCase): def test_default(self): - client = MongoClient(connect=False) + client = self.simple_client(connect=False) self.assertEqual(client.options.retry_reads, True) def test_kwargs(self): - client = MongoClient(retryReads=True, connect=False) + client = self.simple_client(retryReads=True, connect=False) self.assertEqual(client.options.retry_reads, True) - client = MongoClient(retryReads=False, connect=False) + client = self.simple_client(retryReads=False, connect=False) self.assertEqual(client.options.retry_reads, False) def test_uri(self): - client = MongoClient("mongodb://h/?retryReads=true", connect=False) + client = self.simple_client("mongodb://h/?retryReads=true", connect=False) self.assertEqual(client.options.retry_reads, True) - client = MongoClient("mongodb://h/?retryReads=false", connect=False) + client = self.simple_client("mongodb://h/?retryReads=false", connect=False) self.assertEqual(client.options.retry_reads, False) From 699d9627583081ff1b09b8932acbc95e09e7beb4 Mon Sep 17 00:00:00 2001 From: "mongodb-dbx-release-bot[bot]" <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:15:44 +0000 Subject: [PATCH 4/8] BUMP 4.9 Signed-off-by: mongodb-dbx-release-bot[bot] <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> --- pymongo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/_version.py b/pymongo/_version.py index 537a340cc..75ec22cc5 100644 --- a/pymongo/_version.py +++ b/pymongo/_version.py @@ -18,7 +18,7 @@ from __future__ import annotations import re from typing import List, Tuple, Union -__version__ = "4.9.0.dev0" +__version__ = "4.9" def get_version_tuple(version: str) -> Tuple[Union[int, str], ...]: From 2ddd16de5a35cb457c7600d9585bb88351fc67d6 Mon Sep 17 00:00:00 2001 From: "mongodb-dbx-release-bot[bot]" <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:33:00 +0000 Subject: [PATCH 5/8] BUMP 4.10.0.dev0 Signed-off-by: mongodb-dbx-release-bot[bot] <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> --- pymongo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/_version.py b/pymongo/_version.py index 75ec22cc5..5ff72d6cc 100644 --- a/pymongo/_version.py +++ b/pymongo/_version.py @@ -18,7 +18,7 @@ from __future__ import annotations import re from typing import List, Tuple, Union -__version__ = "4.9" +__version__ = "4.10.0.dev0" def get_version_tuple(version: str) -> Tuple[Union[int, str], ...]: From d0772f21619a44f7dc511e067c86c48e7f1f8c6f Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 18 Sep 2024 18:09:20 -0400 Subject: [PATCH 6/8] PYTHON-4773 - Async PyMongo Beta docs update (#1868) --- doc/api/pymongo/asynchronous/change_stream.rst | 5 +++++ doc/api/pymongo/asynchronous/client_session.rst | 5 +++++ doc/api/pymongo/asynchronous/collection.rst | 5 +++++ doc/api/pymongo/asynchronous/command_cursor.rst | 5 +++++ doc/api/pymongo/asynchronous/cursor.rst | 5 +++++ doc/api/pymongo/asynchronous/database.rst | 5 +++++ doc/api/pymongo/asynchronous/index.rst | 5 +++++ doc/api/pymongo/asynchronous/mongo_client.rst | 5 +++++ doc/changelog.rst | 9 ++++++++- pymongo/asynchronous/mongo_client.py | 2 ++ tools/synchro.py | 11 ++++++++++- 11 files changed, 60 insertions(+), 2 deletions(-) diff --git a/doc/api/pymongo/asynchronous/change_stream.rst b/doc/api/pymongo/asynchronous/change_stream.rst index 2ba0feb5c..df4f5dee4 100644 --- a/doc/api/pymongo/asynchronous/change_stream.rst +++ b/doc/api/pymongo/asynchronous/change_stream.rst @@ -1,5 +1,10 @@ :mod:`change_stream` -- Watch changes on a collection, database, or cluster =========================================================================== +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.change_stream :members: diff --git a/doc/api/pymongo/asynchronous/client_session.rst b/doc/api/pymongo/asynchronous/client_session.rst index 1e74e1be7..c4bbd8edd 100644 --- a/doc/api/pymongo/asynchronous/client_session.rst +++ b/doc/api/pymongo/asynchronous/client_session.rst @@ -1,5 +1,10 @@ :mod:`client_session` -- Logical sessions for sequential operations =================================================================== +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.client_session :members: diff --git a/doc/api/pymongo/asynchronous/collection.rst b/doc/api/pymongo/asynchronous/collection.rst index 779557ff6..ce1fe3ca0 100644 --- a/doc/api/pymongo/asynchronous/collection.rst +++ b/doc/api/pymongo/asynchronous/collection.rst @@ -1,6 +1,11 @@ :mod:`collection` -- Collection level operations ================================================ +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.collection :synopsis: Collection level operations diff --git a/doc/api/pymongo/asynchronous/command_cursor.rst b/doc/api/pymongo/asynchronous/command_cursor.rst index 41a8f617e..7058563ee 100644 --- a/doc/api/pymongo/asynchronous/command_cursor.rst +++ b/doc/api/pymongo/asynchronous/command_cursor.rst @@ -1,6 +1,11 @@ :mod:`command_cursor` -- Tools for iterating over MongoDB command results ========================================================================= +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.command_cursor :synopsis: Tools for iterating over MongoDB command results :members: diff --git a/doc/api/pymongo/asynchronous/cursor.rst b/doc/api/pymongo/asynchronous/cursor.rst index ff7103c01..d357b8451 100644 --- a/doc/api/pymongo/asynchronous/cursor.rst +++ b/doc/api/pymongo/asynchronous/cursor.rst @@ -1,6 +1,11 @@ :mod:`cursor` -- Tools for iterating over MongoDB query results =============================================================== +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.cursor :synopsis: Tools for iterating over MongoDB query results diff --git a/doc/api/pymongo/asynchronous/database.rst b/doc/api/pymongo/asynchronous/database.rst index afd6959c9..b45fe457e 100644 --- a/doc/api/pymongo/asynchronous/database.rst +++ b/doc/api/pymongo/asynchronous/database.rst @@ -1,6 +1,11 @@ :mod:`database` -- Database level operations ============================================ +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.database :synopsis: Database level operations diff --git a/doc/api/pymongo/asynchronous/index.rst b/doc/api/pymongo/asynchronous/index.rst index 25dfac632..1b41fb822 100644 --- a/doc/api/pymongo/asynchronous/index.rst +++ b/doc/api/pymongo/asynchronous/index.rst @@ -1,6 +1,11 @@ :mod:`pymongo async` -- Async Python driver for MongoDB ======================================================= +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous :synopsis: Asynchronous Python driver for MongoDB diff --git a/doc/api/pymongo/asynchronous/mongo_client.rst b/doc/api/pymongo/asynchronous/mongo_client.rst index 75952f1b6..d0729da78 100644 --- a/doc/api/pymongo/asynchronous/mongo_client.rst +++ b/doc/api/pymongo/asynchronous/mongo_client.rst @@ -1,6 +1,11 @@ :mod:`mongo_client` -- Tools for connecting to MongoDB ====================================================== +.. warning:: This API is currently in beta, meaning the classes, methods, + and behaviors described within may change before the full release. + If you come across any bugs during your use of this API, + please file a Jira ticket in the "Python Driver" project at https://jira.mongodb.org/browse/PYTHON. + .. automodule:: pymongo.asynchronous.mongo_client :synopsis: Tools for connecting to MongoDB diff --git a/doc/changelog.rst b/doc/changelog.rst index 69fbb6f8f..dfb3c7982 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -7,11 +7,18 @@ Changes in Version 4.9.0 .. warning:: Driver support for MongoDB 3.6 reached end of life in April 2024. PyMongo 4.9 will be the last release to support MongoDB 3.6. +.. warning:: PyMongo 4.9 refactors a large portion of internal APIs to support the new asynchronous API beta. + As a result, versions of Motor older than 3.6 are not compatible with PyMongo 4.9. + Existing users of these versions must either upgrade to Motor 3.6 and PyMongo 4.9, + or cap their PyMongo version to ``< 4.9``. + Any applications that use private APIs may also break as a result of these internal changes. + PyMongo 4.9 brings a number of improvements including: - Added support for MongoDB 8.0. - Added support for Python 3.13. -- A new asynchronous API with full asyncio support. +- A new beta asynchronous API with full asyncio support. + This new asynchronous API is a work-in-progress that may change during the beta period before the full release. - Added support for In-Use Encryption range queries with MongoDB 8.0. Added :attr:`~pymongo.encryption.Algorithm.RANGE`. ``sparsity`` and ``trim_factor`` are now optional in :class:`~pymongo.encryption_options.RangeOpts`. diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 6d0e5d528..814c60456 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -177,6 +177,8 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): For more details, see the relevant section of the PyMongo 4.x migration guide: :ref:`pymongo4-migration-direct-connection`. + .. warning:: This API is currently in beta, meaning the classes, methods, and behaviors described within may change before the full release. + The client object is thread-safe and has connection-pooling built in. If an operation fails because of a network error, :class:`~pymongo.errors.ConnectionFailure` is raised and the client diff --git a/tools/synchro.py b/tools/synchro.py index fdf3a05c9..59d6e653e 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -119,6 +119,10 @@ docstring_replacements: dict[tuple[str, str], str] = { be passed as options for the create collection command.""", } +docstring_removals: set[str] = { + ".. warning:: This API is currently in beta, meaning the classes, methods, and behaviors described within may change before the full release." +} + type_replacements = {"_Condition": "threading.Condition"} import_replacements = {"test.synchronous": "test"} @@ -322,7 +326,12 @@ def translate_docstrings(lines: list[str]) -> list[str]: docstring_replacements[k], # type: ignore[index] ) - return lines + for line in docstring_removals: + if line in lines[i]: + lines[i] = "DOCSTRING_REMOVED" + lines[i + 1] = "DOCSTRING_REMOVED" + + return [line for line in lines if line != "DOCSTRING_REMOVED"] def unasync_directory(files: list[str], src: str, dest: str, replacements: dict[str, str]) -> None: From 8b26d4bc0952c4fe4b19eaf593b9040a1d62a9e0 Mon Sep 17 00:00:00 2001 From: "mongodb-dbx-release-bot[bot]" <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 22:10:50 +0000 Subject: [PATCH 7/8] BUMP 4.9.1 Signed-off-by: mongodb-dbx-release-bot[bot] <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> --- pymongo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/_version.py b/pymongo/_version.py index 5ff72d6cc..adcb933c3 100644 --- a/pymongo/_version.py +++ b/pymongo/_version.py @@ -18,7 +18,7 @@ from __future__ import annotations import re from typing import List, Tuple, Union -__version__ = "4.10.0.dev0" +__version__ = "4.9.1" def get_version_tuple(version: str) -> Tuple[Union[int, str], ...]: From 9df635f10276d92e26b39c8ba35243c1064a29dd Mon Sep 17 00:00:00 2001 From: "mongodb-dbx-release-bot[bot]" <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 22:27:23 +0000 Subject: [PATCH 8/8] BUMP 4.10.0.dev0 Signed-off-by: mongodb-dbx-release-bot[bot] <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> --- pymongo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/_version.py b/pymongo/_version.py index adcb933c3..5ff72d6cc 100644 --- a/pymongo/_version.py +++ b/pymongo/_version.py @@ -18,7 +18,7 @@ from __future__ import annotations import re from typing import List, Tuple, Union -__version__ = "4.9.1" +__version__ = "4.10.0.dev0" def get_version_tuple(version: str) -> Tuple[Union[int, str], ...]: