From c30eff1291252a6ab3321fe0d70677adaa5e5616 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 5 May 2026 11:40:19 -0400 Subject: [PATCH 1/8] =?UTF-8?q?PYTHON-5811=20-=20Change=20stream=20events?= =?UTF-8?q?=20are=20not=20emitted=20for=20timeseries=20as=20=E2=80=A6=20(#?= =?UTF-8?q?2791)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../unified/change-streams-nsType.json | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/test/change_streams/unified/change-streams-nsType.json b/test/change_streams/unified/change-streams-nsType.json index 1861c9a5e..877ca9c00 100644 --- a/test/change_streams/unified/change-streams-nsType.json +++ b/test/change_streams/unified/change-streams-nsType.json @@ -63,47 +63,6 @@ } ] }, - { - "description": "nsType is present when creating timeseries", - "operations": [ - { - "name": "dropCollection", - "object": "database0", - "arguments": { - "collection": "foo" - } - }, - { - "name": "createChangeStream", - "object": "database0", - "arguments": { - "pipeline": [], - "showExpandedEvents": true - }, - "saveResultAsEntity": "changeStream0" - }, - { - "name": "createCollection", - "object": "database0", - "arguments": { - "collection": "foo", - "timeseries": { - "timeField": "time", - "metaField": "meta", - "granularity": "minutes" - } - } - }, - { - "name": "iterateUntilDocumentOrError", - "object": "changeStream0", - "expectResult": { - "operationType": "create", - "nsType": "timeseries" - } - } - ] - }, { "description": "nsType is present when creating views", "operations": [ From 575d75f4d3c690bb4b5562e7bd896e5547548f96 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 5 May 2026 13:41:10 -0400 Subject: [PATCH 2/8] =?UTF-8?q?PYTHON-5813=20-=20Skip=20QE=20prefixPreview?= =?UTF-8?q?=20and=20suffixPreview=20tests=20on=20server=E2=80=A6=20(#2792)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/asynchronous/test_encryption.py | 1 + .../spec/unified/QE-Text-cleanupStructuredEncryptionData.json | 1 + .../spec/unified/QE-Text-compactStructuredEncryptionData.json | 1 + .../spec/unified/QE-Text-prefixPreview.json | 1 + .../spec/unified/QE-Text-substringPreview.json | 2 +- .../spec/unified/QE-Text-suffixPreview.json | 1 + test/test_encryption.py | 1 + 7 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 9650f7043..1503f54c7 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3326,6 +3326,7 @@ class TestAutomaticDecryptionKeys(AsyncEncryptionIntegrationTest): class TestExplicitTextEncryptionProse(AsyncEncryptionIntegrationTest): @async_client_context.require_no_standalone @async_client_context.require_version_min(8, 2, -1) + @async_client_context.require_version_max(8, 99, 99) @async_client_context.require_libmongocrypt_min(1, 15, 1) @async_client_context.require_pymongocrypt_min(1, 16, 0) async def asyncSetUp(self): diff --git a/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json b/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json index 24f33ab3e..fd74573ea 100644 --- a/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json +++ b/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json @@ -4,6 +4,7 @@ "runOnRequirements": [ { "minServerVersion": "8.2.0", + "maxServerVersion": "8.99.99", "topologies": [ "replicaset", "sharded", diff --git a/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json b/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json index c7abfe2d4..a89ab96fc 100644 --- a/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json +++ b/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json @@ -4,6 +4,7 @@ "runOnRequirements": [ { "minServerVersion": "8.2.0", + "maxServerVersion": "8.99.99", "topologies": [ "replicaset", "sharded", diff --git a/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json b/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json index 727938574..c193608e8 100644 --- a/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json +++ b/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json @@ -4,6 +4,7 @@ "runOnRequirements": [ { "minServerVersion": "8.2.0", + "maxServerVersion": "8.99.99", "topologies": [ "replicaset", "sharded", diff --git a/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json b/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json index 6a8f133ea..7787194fc 100644 --- a/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json +++ b/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json @@ -126,7 +126,7 @@ ], "tests": [ { - "description": "Insert QE suffixPreview", + "description": "Insert QE substringPreview", "operations": [ { "name": "insertOne", diff --git a/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json b/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json index deec5e63b..2de5cde4a 100644 --- a/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json +++ b/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json @@ -4,6 +4,7 @@ "runOnRequirements": [ { "minServerVersion": "8.2.0", + "maxServerVersion": "8.99.99", "topologies": [ "replicaset", "sharded", diff --git a/test/test_encryption.py b/test/test_encryption.py index af9f2e3df..099336227 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3308,6 +3308,7 @@ class TestAutomaticDecryptionKeys(EncryptionIntegrationTest): class TestExplicitTextEncryptionProse(EncryptionIntegrationTest): @client_context.require_no_standalone @client_context.require_version_min(8, 2, -1) + @client_context.require_version_max(8, 99, 99) @client_context.require_libmongocrypt_min(1, 15, 1) @client_context.require_pymongocrypt_min(1, 16, 0) def setUp(self): From 900d9c79103190b414e8983fc53e195b1224dc43 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 May 2026 13:10:13 -0400 Subject: [PATCH 3/8] =?UTF-8?q?PYTHON-5436=20-=20Always=20include=20sessio?= =?UTF-8?q?n=20on=20getMores=20if=20the=20initial=20curso=E2=80=A6=20(#279?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pymongo/asynchronous/client_session.py | 6 +++- pymongo/synchronous/client_session.py | 6 +++- test/asynchronous/test_session.py | 38 +++++++++++++++++++++++--- test/test_session.py | 38 +++++++++++++++++++++++--- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/pymongo/asynchronous/client_session.py b/pymongo/asynchronous/client_session.py index 31e6ceb38..9fbd10c8e 100644 --- a/pymongo/asynchronous/client_session.py +++ b/pymongo/asynchronous/client_session.py @@ -1101,7 +1101,11 @@ class AsyncClientSession: read_preference: _ServerMode, conn: AsyncConnection, ) -> None: - if not conn.supports_sessions: + # getMores must be sent with a session if the cursor was opened with one + operation = next(iter(command)) + if not conn.supports_sessions and ( + isinstance(self._server_session, _EmptyServerSession) or operation != "getMore" + ): if not self._implicit: raise ConfigurationError("Sessions are not supported by this MongoDB deployment") return diff --git a/pymongo/synchronous/client_session.py b/pymongo/synchronous/client_session.py index 3165dd52b..756385084 100644 --- a/pymongo/synchronous/client_session.py +++ b/pymongo/synchronous/client_session.py @@ -1097,7 +1097,11 @@ class ClientSession: read_preference: _ServerMode, conn: Connection, ) -> None: - if not conn.supports_sessions: + # getMores must be sent with a session if the cursor was opened with one + operation = next(iter(command)) + if not conn.supports_sessions and ( + isinstance(self._server_session, _EmptyServerSession) or operation != "getMore" + ): if not self._implicit: raise ConfigurationError("Sessions are not supported by this MongoDB deployment") return diff --git a/test/asynchronous/test_session.py b/test/asynchronous/test_session.py index 404a69fde..13ce57867 100644 --- a/test/asynchronous/test_session.py +++ b/test/asynchronous/test_session.py @@ -15,7 +15,6 @@ """Test the client_session module.""" from __future__ import annotations -import asyncio import copy import sys import time @@ -24,8 +23,6 @@ from io import BytesIO from test.asynchronous.helpers import ExceptionCatchingTask from typing import Any, Callable, List, Set, Tuple -from pymongo.synchronous.mongo_client import MongoClient - sys.path[0:0] = [""] from test.asynchronous import ( @@ -45,7 +42,7 @@ from test.utils_shared import ( from bson import DBRef from gridfs.asynchronous.grid_file import AsyncGridFS, AsyncGridFSBucket -from pymongo import ASCENDING, AsyncMongoClient, _csot, monitoring +from pymongo import ASCENDING, AsyncMongoClient, monitoring from pymongo.asynchronous.command_cursor import AsyncCommandCursor from pymongo.asynchronous.cursor import AsyncCursor from pymongo.asynchronous.helpers import anext @@ -938,6 +935,39 @@ class TestSession(AsyncIntegrationTest): await s2.end_session() + async def test_getmore_preserves_lsid_after_session_support_lost(self): + listener = OvertCommandListener() + client = await self.async_rs_or_single_client(event_listeners=[listener], maxPoolSize=1) + coll = client.pymongo_test.test + await coll.drop() + await coll.insert_many([{"x": i} for i in range(10)]) + self.addAsyncCleanup(coll.drop) + + async with client.start_session() as s: + cursor = coll.find({}, batch_size=2, session=s) + await anext(cursor) + + find_event = next(e for e in listener.started_events if e.command_name == "find") + lsid = find_event.command["lsid"] + + # Simulate a node stepping down: mark idle connections as not supporting sessions. + for server in client._topology._servers.values(): + for conn in server.pool.conns: + conn.supports_sessions = False + + listener.reset() + await cursor.to_list() + + getmore_events = [e for e in listener.started_events if e.command_name == "getMore"] + self.assertGreater(len(getmore_events), 0, "expected at least one getMore command") + for event in getmore_events: + self.assertIn( + "lsid", event.command, "getMore must include lsid when session is materialized" + ) + self.assertEqual( + lsid, event.command["lsid"], "getMore lsid must match the session lsid from find" + ) + class TestCausalConsistency(AsyncUnitTest): listener: SessionTestListener diff --git a/test/test_session.py b/test/test_session.py index 3963f88da..cf071df49 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -15,7 +15,6 @@ """Test the client_session module.""" from __future__ import annotations -import asyncio import copy import sys import time @@ -24,8 +23,6 @@ from io import BytesIO from test.helpers import ExceptionCatchingTask from typing import Any, Callable, List, Set, Tuple -from pymongo.synchronous.mongo_client import MongoClient - sys.path[0:0] = [""] from test import ( @@ -45,7 +42,7 @@ from test.utils_shared import ( from bson import DBRef from gridfs.synchronous.grid_file import GridFS, GridFSBucket -from pymongo import ASCENDING, MongoClient, _csot, monitoring +from pymongo import ASCENDING, MongoClient, monitoring from pymongo.common import _MAX_END_SESSIONS from pymongo.errors import ConfigurationError, InvalidOperation, OperationFailure from pymongo.operations import IndexModel, InsertOne, UpdateOne @@ -938,6 +935,39 @@ class TestSession(IntegrationTest): s2.end_session() + def test_getmore_preserves_lsid_after_session_support_lost(self): + listener = OvertCommandListener() + client = self.rs_or_single_client(event_listeners=[listener], maxPoolSize=1) + coll = client.pymongo_test.test + coll.drop() + coll.insert_many([{"x": i} for i in range(10)]) + self.addCleanup(coll.drop) + + with client.start_session() as s: + cursor = coll.find({}, batch_size=2, session=s) + next(cursor) + + find_event = next(e for e in listener.started_events if e.command_name == "find") + lsid = find_event.command["lsid"] + + # Simulate a node stepping down: mark idle connections as not supporting sessions. + for server in client._topology._servers.values(): + for conn in server.pool.conns: + conn.supports_sessions = False + + listener.reset() + cursor.to_list() + + getmore_events = [e for e in listener.started_events if e.command_name == "getMore"] + self.assertGreater(len(getmore_events), 0, "expected at least one getMore command") + for event in getmore_events: + self.assertIn( + "lsid", event.command, "getMore must include lsid when session is materialized" + ) + self.assertEqual( + lsid, event.command["lsid"], "getMore lsid must match the session lsid from find" + ) + class TestCausalConsistency(UnitTest): listener: SessionTestListener From f4219bdca2cfa816f90edb9f5463dbfc2666e609 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 May 2026 13:28:36 -0400 Subject: [PATCH 4/8] PYTHON-5817 - Add "Project Structure and Asyncio Considerations" section to CONTRIBUTING.md (#2788) Co-authored-by: Jib --- CONTRIBUTING.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77888eb08..773c9ec0d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -505,13 +505,20 @@ python3 ./.evergreen/scripts/resync-all-specs.py Follow the [Python Driver Release Process Wiki](https://wiki.corp.mongodb.com/display/DRIVERS/Python+Driver+Release+Process). -## Asyncio considerations +## Project Structure and Asyncio Considerations -PyMongo adds asyncio capability by modifying the source files in `*/asynchronous` to `*/synchronous` using -[unasync](https://github.com/python-trio/unasync/) and some custom transforms. +This section describes the layout of the `pymongo/` package. -Where possible, edit the code in `*/asynchronous/*.py` and not the synchronous files. -You can run `pre-commit run --all-files synchro` before running tests if you are testing synchronous code. +Within `pymongo/`, the code is further divided into the `pymongo/asynchronous` and `pymongo/synchronous` subdirectories. +Files in `pymongo/synchronous` are generated from `pymongo/asynchronous` using the `synchro` pre-commit hook, which uses [unasync](https://github.com/python-trio/unasync/) and some custom transforms. + +As a result, **all modifications** within `pymongo` must be made in either the top-level `pymongo` directory when they have to exhibit differing behavior between sync and async contexts or the `pymongo/asynchronous` directory, not `pymongo/synchronous`. +Any changes made directly to files in the `pymongo/synchronous` directory will be overwritten by the `synchro` hook when it is run, which happens automatically on commit. + +Some top-level files (e.g. `pymongo/collection.py`) are re-export files for existing import compatibility and should not be modified directly. +The other top-level files (e.g. `pymongo/network_layer.py`, `pymongo/pool_shared.py`) contain either shared code used in both the asynchronous and synchronous APIs, or code that is very different between the two APIs and therefore cannot be generated from the async version using `synchro`. + +Run `pre-commit run --all-files synchro` before running tests to generate the latest version of the synchronous code. To prevent the `synchro` hook from accidentally overwriting code, it first checks to see whether a sync version of a file is changing and not its async counterpart, and will fail. From 8dc7efade2e5b9c7dac669bc186e74a277367b6a Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 May 2026 12:28:15 -0400 Subject: [PATCH 5/8] PYTHON-5821 - Fix ordering issue between event publish and logging for Pool monitoring tests (#2796) --- pymongo/asynchronous/pool.py | 37 ++++++++++++++++++------------------ pymongo/synchronous/pool.py | 37 ++++++++++++++++++------------------ 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 3c1a85246..a5d5b2899 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -760,11 +760,7 @@ class Pool: self._pending = 0 self._max_connecting = self.opts.max_connecting self._client_id = client_id - if self.enabled_for_cmap: - assert self.opts._event_listeners is not None - self.opts._event_listeners.publish_pool_created( - self.address, self.opts.non_default_options - ) + # Log before publishing event to prevent potential listener preemption in tests if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -774,6 +770,11 @@ class Pool: serverPort=self.address[1], **self.opts.non_default_options, ) + if self.enabled_for_cmap: + assert self.opts._event_listeners is not None + self.opts._event_listeners.publish_pool_created( + self.address, self.opts.non_default_options + ) # Similar to active_sockets but includes threads in the wait queue. self.operation_count: int = 0 # Retain references to pinned connections to prevent the CPython GC @@ -788,9 +789,6 @@ class Pool: async with self.lock: if self.state != PoolState.READY: self.state = PoolState.READY - if self.enabled_for_cmap: - assert self.opts._event_listeners is not None - self.opts._event_listeners.publish_pool_ready(self.address) if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -799,6 +797,9 @@ class Pool: serverHost=self.address[0], serverPort=self.address[1], ) + if self.enabled_for_cmap: + assert self.opts._event_listeners is not None + self.opts._event_listeners.publish_pool_ready(self.address) @property def closed(self) -> bool: @@ -859,9 +860,6 @@ class Pool: else: for conn in sockets: await conn.close_conn(ConnectionClosedReason.POOL_CLOSED) - if self.enabled_for_cmap: - assert listeners is not None - listeners.publish_pool_closed(self.address) if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -870,15 +868,11 @@ class Pool: serverHost=self.address[0], serverPort=self.address[1], ) + if self.enabled_for_cmap: + assert listeners is not None + listeners.publish_pool_closed(self.address) else: if old_state != PoolState.PAUSED: - if self.enabled_for_cmap: - assert listeners is not None - listeners.publish_pool_cleared( - self.address, - service_id=service_id, - interrupt_connections=interrupt_connections, - ) if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -888,6 +882,13 @@ class Pool: serverPort=self.address[1], serviceId=service_id, ) + if self.enabled_for_cmap: + assert listeners is not None + listeners.publish_pool_cleared( + self.address, + service_id=service_id, + interrupt_connections=interrupt_connections, + ) if not _IS_SYNC: await asyncio.gather( *[conn.close_conn(ConnectionClosedReason.STALE) for conn in sockets], # type: ignore[func-returns-value] diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index d33cb59a9..25f2d08fe 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -758,11 +758,7 @@ class Pool: self._pending = 0 self._max_connecting = self.opts.max_connecting self._client_id = client_id - if self.enabled_for_cmap: - assert self.opts._event_listeners is not None - self.opts._event_listeners.publish_pool_created( - self.address, self.opts.non_default_options - ) + # Log before publishing event to prevent potential listener preemption in tests if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -772,6 +768,11 @@ class Pool: serverPort=self.address[1], **self.opts.non_default_options, ) + if self.enabled_for_cmap: + assert self.opts._event_listeners is not None + self.opts._event_listeners.publish_pool_created( + self.address, self.opts.non_default_options + ) # Similar to active_sockets but includes threads in the wait queue. self.operation_count: int = 0 # Retain references to pinned connections to prevent the CPython GC @@ -786,9 +787,6 @@ class Pool: with self.lock: if self.state != PoolState.READY: self.state = PoolState.READY - if self.enabled_for_cmap: - assert self.opts._event_listeners is not None - self.opts._event_listeners.publish_pool_ready(self.address) if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -797,6 +795,9 @@ class Pool: serverHost=self.address[0], serverPort=self.address[1], ) + if self.enabled_for_cmap: + assert self.opts._event_listeners is not None + self.opts._event_listeners.publish_pool_ready(self.address) @property def closed(self) -> bool: @@ -857,9 +858,6 @@ class Pool: else: for conn in sockets: conn.close_conn(ConnectionClosedReason.POOL_CLOSED) - if self.enabled_for_cmap: - assert listeners is not None - listeners.publish_pool_closed(self.address) if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -868,15 +866,11 @@ class Pool: serverHost=self.address[0], serverPort=self.address[1], ) + if self.enabled_for_cmap: + assert listeners is not None + listeners.publish_pool_closed(self.address) else: if old_state != PoolState.PAUSED: - if self.enabled_for_cmap: - assert listeners is not None - listeners.publish_pool_cleared( - self.address, - service_id=service_id, - interrupt_connections=interrupt_connections, - ) if self.enabled_for_logging and _CONNECTION_LOGGER.isEnabledFor(logging.DEBUG): _debug_log( _CONNECTION_LOGGER, @@ -886,6 +880,13 @@ class Pool: serverPort=self.address[1], serviceId=service_id, ) + if self.enabled_for_cmap: + assert listeners is not None + listeners.publish_pool_cleared( + self.address, + service_id=service_id, + interrupt_connections=interrupt_connections, + ) if not _IS_SYNC: asyncio.gather( *[conn.close_conn(ConnectionClosedReason.STALE) for conn in sockets], # type: ignore[func-returns-value] From b6bac45c7e9a14e583491a3574df2dec31bb2935 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 May 2026 14:52:19 -0400 Subject: [PATCH 6/8] =?UTF-8?q?PYTHON-5032=20-=20Use=20PyErr=5FGetRaisedEx?= =?UTF-8?q?ception=20instead=20of=20deprecated=20PyEr=E2=80=A6=20(#2795)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bson/_cbsonmodule.c | 215 +++++++++++++++++++++++++++++--------------- 1 file changed, 144 insertions(+), 71 deletions(-) diff --git a/bson/_cbsonmodule.c b/bson/_cbsonmodule.c index 034490f55..330a52a73 100644 --- a/bson/_cbsonmodule.c +++ b/bson/_cbsonmodule.c @@ -109,6 +109,7 @@ struct module_state { #define DATETIME_CLAMP 2 #define DATETIME_MS 3 #define DATETIME_AUTO 4 +#define PYTHON_3_12 0x030C0000 /* Converts integer to its string representation in decimal notation. */ extern int cbson_long_long_to_str(long long num, char* str, size_t size) { @@ -249,6 +250,67 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, */ static int write_raw_doc(buffer_t buffer, PyObject* raw, PyObject* _raw); +#if PY_VERSION_HEX >= PYTHON_3_12 +/* Transfer traceback from old_exc to new_exc. + * Steals reference to old_exc. */ +static PyObject* _transfer_traceback(PyObject *old_exc, PyObject *new_exc) { + PyObject *tb = PyException_GetTraceback(old_exc); + if (tb) { + PyException_SetTraceback(new_exc, tb); + Py_DECREF(tb); + } + Py_DECREF(old_exc); + return new_exc; +} +#endif + +/* Rewrap the current exception as InvalidBSON(str(e)) if it is not already an InvalidBSON error. */ +static void _rewrap_as_invalid_bson(void) { +#if PY_VERSION_HEX >= PYTHON_3_12 + PyObject *exc = PyErr_GetRaisedException(); + if (exc && PyErr_GivenExceptionMatches(exc, PyExc_Exception)) { + PyObject *InvalidBSON = _error("InvalidBSON"); + if (InvalidBSON) { + if (!PyErr_GivenExceptionMatches(exc, InvalidBSON)) { + PyObject *err_msg = PyObject_Str(exc); + if (err_msg) { + PyObject *new_exc = PyObject_CallOneArg(InvalidBSON, err_msg); + if (new_exc) { + exc = _transfer_traceback(exc, new_exc); + } + } + Py_XDECREF(err_msg); + } + Py_DECREF(InvalidBSON); + } + } + /* Steals reference to exc. */ + PyErr_SetRaisedException(exc); +#else + PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; + PyObject *InvalidBSON = NULL; + PyErr_Fetch(&etype, &evalue, &etrace); + if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { + InvalidBSON = _error("InvalidBSON"); + if (InvalidBSON) { + if (!PyErr_GivenExceptionMatches(etype, InvalidBSON)) { + Py_DECREF(etype); + etype = InvalidBSON; + if (evalue) { + PyObject *msg = PyObject_Str(evalue); + Py_DECREF(evalue); + evalue = msg; + } + PyErr_NormalizeException(&etype, &evalue, &etrace); + } else { + Py_DECREF(InvalidBSON); + } + } + } + PyErr_Restore(etype, evalue, etrace); +#endif +} + /* Date stuff */ static PyObject* datetime_from_millis(long long millis) { /* To encode a datetime instance like datetime(9999, 12, 31, 23, 59, 59, 999999) @@ -294,34 +356,57 @@ static PyObject* datetime_from_millis(long long millis) { timeinfo.tm_sec, microseconds); if(!datetime) { - PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; + #if PY_VERSION_HEX >= PYTHON_3_12 + PyObject *exc = PyErr_GetRaisedException(); - /* - * Calling _error clears the error state, so fetch it first. - */ - PyErr_Fetch(&etype, &evalue, &etrace); - - /* Only add addition error message on ValueError exceptions. */ - if (PyErr_GivenExceptionMatches(etype, PyExc_ValueError)) { - if (evalue) { - PyObject* err_msg = PyObject_Str(evalue); + /* Only add additional error message on ValueError exceptions. */ + if (exc && PyErr_GivenExceptionMatches(exc, PyExc_ValueError)) { + PyObject* err_msg = PyObject_Str(exc); if (err_msg) { PyObject* appendage = PyUnicode_FromString(" (Consider Using CodecOptions(datetime_conversion=DATETIME_AUTO) or MongoClient(datetime_conversion='DATETIME_AUTO')). See: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes"); if (appendage) { PyObject* msg = PyUnicode_Concat(err_msg, appendage); if (msg) { - Py_DECREF(evalue); - evalue = msg; + PyObject* new_exc = PyObject_CallOneArg(PyExc_ValueError, msg); + if (new_exc) { + exc = _transfer_traceback(exc, new_exc); + } + Py_DECREF(msg); } } Py_XDECREF(appendage); } Py_XDECREF(err_msg); } - PyErr_NormalizeException(&etype, &evalue, &etrace); - } - /* Steals references to args. */ - PyErr_Restore(etype, evalue, etrace); + /* Steals reference to exc. */ + PyErr_SetRaisedException(exc); + #else + /* Calling _error clears the error state, so fetch it first.*/ + PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; + PyErr_Fetch(&etype, &evalue, &etrace); + + /* Only add additional error message on ValueError exceptions. */ + if (PyErr_GivenExceptionMatches(etype, PyExc_ValueError)) { + if (evalue) { + PyObject* err_msg = PyObject_Str(evalue); + if (err_msg) { + PyObject* appendage = PyUnicode_FromString(" (Consider Using CodecOptions(datetime_conversion=DATETIME_AUTO) or MongoClient(datetime_conversion='DATETIME_AUTO')). See: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes"); + if (appendage) { + PyObject* msg = PyUnicode_Concat(err_msg, appendage); + if (msg) { + Py_DECREF(evalue); + evalue = msg; + } + } + Py_XDECREF(appendage); + } + Py_XDECREF(err_msg); + } + PyErr_NormalizeException(&etype, &evalue, &etrace); + } + /* Steals references to args. */ + PyErr_Restore(etype, evalue, etrace); + #endif } return datetime; } @@ -1681,6 +1766,46 @@ fail: /* Update Invalid Document error to include doc as a property. */ void handle_invalid_doc_error(PyObject* dict) { +#if PY_VERSION_HEX >= PYTHON_3_12 + PyObject *exc = PyErr_GetRaisedException(); + PyObject *msg = NULL, *new_msg = NULL; + PyObject *InvalidDocument = NULL; + + if (exc == NULL) { + return; + } + + InvalidDocument = _error("InvalidDocument"); + if (InvalidDocument == NULL) { + goto cleanup; + } + + if (PyErr_GivenExceptionMatches(exc, InvalidDocument)) { + msg = PyObject_Str(exc); + if (msg) { + const char *msg_utf8 = PyUnicode_AsUTF8(msg); + if (msg_utf8 == NULL) { + goto cleanup; + } + new_msg = PyUnicode_FromFormat("Invalid document: %s", msg_utf8); + if (new_msg == NULL) { + goto cleanup; + } + /* Add doc to the error instance as a property. */ + PyObject* exc_args[2] = {new_msg, dict}; + PyObject* new_exc = PyObject_Vectorcall(InvalidDocument, exc_args, 2, NULL); + if (new_exc) { + exc = _transfer_traceback(exc, new_exc); + } + } + } +cleanup: + /* Steals reference to exc. */ + PyErr_SetRaisedException(exc); + Py_XDECREF(msg); + Py_XDECREF(InvalidDocument); + Py_XDECREF(new_msg); +#else PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; PyObject *msg = NULL, *new_msg = NULL, *new_evalue = NULL; PyErr_Fetch(&etype, &evalue, &etrace); @@ -1723,6 +1848,7 @@ cleanup: Py_XDECREF(InvalidDocument); Py_XDECREF(new_evalue); Py_XDECREF(new_msg); +#endif } @@ -2654,42 +2780,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, * Wrap any non-InvalidBSON errors in InvalidBSON. */ if (PyErr_Occurred()) { - PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; - PyObject *InvalidBSON = NULL; - - /* - * Calling _error clears the error state, so fetch it first. - */ - PyErr_Fetch(&etype, &evalue, &etrace); - - /* Dont reraise anything but PyExc_Exceptions as InvalidBSON. */ - if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { - InvalidBSON = _error("InvalidBSON"); - if (InvalidBSON) { - if (!PyErr_GivenExceptionMatches(etype, InvalidBSON)) { - /* - * Raise InvalidBSON(str(e)). - */ - Py_DECREF(etype); - etype = InvalidBSON; - - if (evalue) { - PyObject *msg = PyObject_Str(evalue); - Py_DECREF(evalue); - evalue = msg; - } - PyErr_NormalizeException(&etype, &evalue, &etrace); - } else { - /* - * The current exception matches InvalidBSON, so we don't - * need this reference after all. - */ - Py_DECREF(InvalidBSON); - } - } - } - /* Steals references to args. */ - PyErr_Restore(etype, evalue, etrace); + _rewrap_as_invalid_bson(); } else { PyObject *InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { @@ -2727,25 +2818,7 @@ static int _element_to_dict(PyObject* self, const char* string, if (!*name) { /* If NULL is returned then wrap the UnicodeDecodeError in an InvalidBSON error */ - PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; - PyObject *InvalidBSON = NULL; - - PyErr_Fetch(&etype, &evalue, &etrace); - if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { - InvalidBSON = _error("InvalidBSON"); - if (InvalidBSON) { - Py_DECREF(etype); - etype = InvalidBSON; - - if (evalue) { - PyObject *msg = PyObject_Str(evalue); - Py_DECREF(evalue); - evalue = msg; - } - PyErr_NormalizeException(&etype, &evalue, &etrace); - } - } - PyErr_Restore(etype, evalue, etrace); + _rewrap_as_invalid_bson(); return -1; } position += (unsigned)name_length + 1; From f145c7db940f72cb9ad76b99793267f619e63217 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 May 2026 15:23:00 -0400 Subject: [PATCH 7/8] PYTHON-5756 - Fix BSON Binary type length bug (#2790) --- bson/_cbsonmodule.c | 2 +- test/test_bson.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/bson/_cbsonmodule.c b/bson/_cbsonmodule.c index 330a52a73..366129936 100644 --- a/bson/_cbsonmodule.c +++ b/bson/_cbsonmodule.c @@ -2281,7 +2281,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } memcpy(&length, buffer + *position, 4); length = BSON_UINT32_FROM_LE(length); - if (max < length) { + if (max - 5 < length) { // Account for 5-byte header. max >= 5 guaranteed above goto invalid; } diff --git a/test/test_bson.py b/test/test_bson.py index ffc02965f..ae1807e5f 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -1269,6 +1269,22 @@ class TestBSON(unittest.TestCase): encode(doc) self.assertEqual(cm.exception.document, doc) + def test_binary_length_accounts_for_header(self): + size = 20 + binary_length = 12 # 5 more than the actual 7 bytes + + payload = b"" + payload += struct.pack(" Date: Thu, 7 May 2026 15:23:07 -0400 Subject: [PATCH 8/8] PYTHON-5708 - Unskip large encryption tests on mongocryptd (#2793) --- test/asynchronous/test_encryption.py | 4 ---- test/test_encryption.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 1503f54c7..455b1940c 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -876,8 +876,6 @@ class TestViews(AsyncEncryptionIntegrationTest): class TestCorpus(AsyncEncryptionIntegrationTest): - # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. - @async_client_context.require_version_max(6, 99) @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") async def asyncSetUp(self): await super().asyncSetUp() @@ -1054,8 +1052,6 @@ class TestBsonSizeBatches(AsyncEncryptionIntegrationTest): client_encrypted: AsyncMongoClient listener: OvertCommandListener - # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. - @async_client_context.require_version_max(6, 99) async def asyncSetUp(self): await super().asyncSetUp() db = async_client_context.client.db diff --git a/test/test_encryption.py b/test/test_encryption.py index 099336227..7df9e7ac3 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -872,8 +872,6 @@ class TestViews(EncryptionIntegrationTest): class TestCorpus(EncryptionIntegrationTest): - # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. - @client_context.require_version_max(6, 99) @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") def setUp(self): super().setUp() @@ -1050,8 +1048,6 @@ class TestBsonSizeBatches(EncryptionIntegrationTest): client_encrypted: MongoClient listener: OvertCommandListener - # PYTHON-5708: Encryption tests sending large payloads fail on some mongocryptd versions. - @client_context.require_version_max(6, 99) def setUp(self): super().setUp() db = client_context.client.db