From 8b2eb24c35689a6e6508cd694c2bd615d2fd20d8 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 1 Nov 2021 14:26:47 -0700 Subject: [PATCH] PYTHON-2164 Remove client max_bson_size/max_message_size/max_write_batch_size (#766) Use the hello command instead: doc = client.admin.command('hello') max_bson_size = doc['maxBsonObjectSize'] max_message_size = doc['maxMessageSizeBytes'] max_write_batch_size = doc['maxWriteBatchSize'] Also add documentation for TopologyDescription.apply_selector. --- doc/api/pymongo/mongo_client.rst | 3 --- doc/changelog.rst | 3 +++ doc/migrate-to-pymongo4.rst | 23 +++++++++++++++++++ pymongo/mongo_client.py | 33 --------------------------- pymongo/topology_description.py | 39 ++++++++++++++++++++------------ test/__init__.py | 14 +++++++++++- test/test_bulk.py | 8 +++---- test/test_client.py | 16 ++----------- test/test_collection.py | 8 +++---- 9 files changed, 74 insertions(+), 73 deletions(-) diff --git a/doc/api/pymongo/mongo_client.rst b/doc/api/pymongo/mongo_client.rst index e48af01ad..ba95e0647 100644 --- a/doc/api/pymongo/mongo_client.rst +++ b/doc/api/pymongo/mongo_client.rst @@ -26,9 +26,6 @@ .. autoattribute:: min_pool_size .. autoattribute:: max_idle_time_ms .. autoattribute:: nodes - .. autoattribute:: max_bson_size - .. autoattribute:: max_message_size - .. autoattribute:: max_write_batch_size .. autoattribute:: local_threshold_ms .. autoattribute:: server_selection_timeout .. autoattribute:: codec_options diff --git a/doc/changelog.rst b/doc/changelog.rst index acb0949d8..b5f32d21b 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -33,6 +33,9 @@ Breaking Changes in 4.0 :meth:`pymongo.mongo_client.MongoClient.unlock`, and :attr:`pymongo.mongo_client.MongoClient.is_locked`. - Removed :meth:`pymongo.mongo_client.MongoClient.database_names`. +- Removed :attr:`pymongo.mongo_client.MongoClient.max_bson_size`. +- Removed :attr:`pymongo.mongo_client.MongoClient.max_message_size`. +- Removed :attr:`pymongo.mongo_client.MongoClient.max_write_batch_size`. - Removed :meth:`pymongo.database.Database.eval`, :data:`pymongo.database.Database.system_js` and :class:`pymongo.database.SystemJS`. diff --git a/doc/migrate-to-pymongo4.rst b/doc/migrate-to-pymongo4.rst index 8a0734670..b83071805 100644 --- a/doc/migrate-to-pymongo4.rst +++ b/doc/migrate-to-pymongo4.rst @@ -173,6 +173,29 @@ can be changed to this:: names = client.list_database_names() +MongoClient.max_bson_size/max_message_size/max_write_batch_size are removed +........................................................................... + +Removed :attr:`pymongo.mongo_client.MongoClient.max_bson_size`, +:attr:`pymongo.mongo_client.MongoClient.max_message_size`, and +:attr:`pymongo.mongo_client.MongoClient.max_write_batch_size`. These helpers +were incorrect when in ``loadBalanced=true mode`` and ambiguous in clusters +with mixed versions. Use the `hello command`_ to get the authoritative +value from the remote server instead. Code like this:: + + max_bson_size = client.max_bson_size + max_message_size = client.max_message_size + max_write_batch_size = client.max_write_batch_size + +can be changed to this:: + + doc = client.admin.command('hello') + max_bson_size = doc['maxBsonObjectSize'] + max_message_size = doc['maxMessageSizeBytes'] + max_write_batch_size = doc['maxWriteBatchSize'] + +.. _hello command: https://docs.mongodb.com/manual/reference/command/hello/ + ``tz_aware`` defaults to ``False`` .................................. diff --git a/pymongo/mongo_client.py b/pymongo/mongo_client.py index 7dca8bb9c..9c0695e6c 100644 --- a/pymongo/mongo_client.py +++ b/pymongo/mongo_client.py @@ -1053,39 +1053,6 @@ class MongoClient(common.BaseObject): description = self._topology.description return frozenset(s.address for s in description.known_servers) - @property - def max_bson_size(self): - """The largest BSON object the connected server accepts in bytes. - - If the client is not connected, this will block until a connection is - established or raise ServerSelectionTimeoutError if no server is - available. - """ - return self._server_property('max_bson_size') - - @property - def max_message_size(self): - """The largest message the connected server accepts in bytes. - - If the client is not connected, this will block until a connection is - established or raise ServerSelectionTimeoutError if no server is - available. - """ - return self._server_property('max_message_size') - - @property - def max_write_batch_size(self): - """The maxWriteBatchSize reported by the server. - - If the client is not connected, this will block until a connection is - established or raise ServerSelectionTimeoutError if no server is - available. - - Returns a default value when connected to server versions prior to - MongoDB 2.6. - """ - return self._server_property('max_write_batch_size') - @property def local_threshold_ms(self): """The local threshold for this instance.""" diff --git a/pymongo/topology_description.py b/pymongo/topology_description.py index 19a1681fa..d0100ff8b 100644 --- a/pymongo/topology_description.py +++ b/pymongo/topology_description.py @@ -228,21 +228,32 @@ class TopologyDescription(object): def srv_max_hosts(self): return self._topology_settings._srv_max_hosts - def apply_selector(self, selector, address, custom_selector=None): + def _apply_local_threshold(self, selection): + if not selection: + return [] + # Round trip time in seconds. + fastest = min( + s.round_trip_time for s in selection.server_descriptions) + threshold = self._topology_settings.local_threshold_ms / 1000.0 + return [s for s in selection.server_descriptions + if (s.round_trip_time - fastest) <= threshold] - def apply_local_threshold(selection): - if not selection: - return [] + def apply_selector(self, selector, address=None, custom_selector=None): + """List of servers matching the provided selector(s). - settings = self._topology_settings - - # Round trip time in seconds. - fastest = min( - s.round_trip_time for s in selection.server_descriptions) - threshold = settings.local_threshold_ms / 1000.0 - return [s for s in selection.server_descriptions - if (s.round_trip_time - fastest) <= threshold] + :Parameters: + - `selector`: a callable that takes a Selection as input and returns + a Selection as output. For example, an instance of a read + preference from :mod:`~pymongo.read_preferences`. + - `address` (optional): A server address to select. + - `custom_selector` (optional): A callable that augments server + selection rules. Accepts a list of + :class:`~pymongo.server_description.ServerDescription` objects and + return a list of server descriptions that should be considered + suitable for the desired operation. + .. versionadded:: 3.4 + """ if getattr(selector, 'min_wire_version', 0): common_wv = self.common_wire_version if common_wv and common_wv < selector.min_wire_version: @@ -271,7 +282,7 @@ class TopologyDescription(object): if custom_selector is not None and selection: selection = selection.with_server_descriptions( custom_selector(selection.server_descriptions)) - return apply_local_threshold(selection) + return self._apply_local_threshold(selection) def has_readable_server(self, read_preference=ReadPreference.PRIMARY): """Does this topology have any readable servers available matching the @@ -288,7 +299,7 @@ class TopologyDescription(object): .. versionadded:: 3.4 """ common.validate_read_preference("read_preference", read_preference) - return any(self.apply_selector(read_preference, None)) + return any(self.apply_selector(read_preference)) def has_writable_server(self): """Does this topology have a writable server available? diff --git a/test/__init__.py b/test/__init__.py index 9e0e972d1..d9144e862 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -239,6 +239,7 @@ class ClientContext(object): self.auth_enabled = False self.test_commands_enabled = False self.server_parameters = {} + self._hello = None self.is_mongos = False self.mongoses = [] self.is_rs = False @@ -274,7 +275,9 @@ class ClientContext(object): @property def hello(self): - return self.client.admin.command(HelloCompat.LEGACY_CMD) + if not self._hello: + self._hello = self.client.admin.command(HelloCompat.LEGACY_CMD) + return self._hello def _connect(self, host, port, **kwargs): # Jython takes a long time to connect. @@ -391,6 +394,7 @@ class ClientContext(object): **self.default_client_options) # Get the authoritative hello result from the primary. + self._hello = None hello = self.hello nodes = [partition_node(node.lower()) for node in hello.get('hosts', [])] @@ -866,6 +870,14 @@ class ClientContext(object): # Changed in SERVER-39567. return self.version.at_least(4, 1, 10) + @property + def max_bson_size(self): + return self.hello['maxBsonObjectSize'] + + @property + def max_write_batch_size(self): + return self.hello['maxWriteBatchSize'] + # Reusable client context client_context = ClientContext() diff --git a/test/test_bulk.py b/test/test_bulk.py index aa2dcab92..f93cd6c76 100644 --- a/test/test_bulk.py +++ b/test/test_bulk.py @@ -283,7 +283,7 @@ class TestBulk(BulkTestBase): def test_numerous_inserts(self): # Ensure we don't exceed server's maxWriteBatchSize size limit. - n_docs = self.client.max_write_batch_size + 100 + n_docs = client_context.max_write_batch_size + 100 requests = [InsertOne({}) for _ in range(n_docs)] result = self.coll.bulk_write(requests, ordered=False) self.assertEqual(n_docs, result.inserted_count) @@ -344,7 +344,7 @@ class TestBulk(BulkTestBase): self.coll.bulk_write([{}]) def test_upsert_large(self): - big = 'a' * (client_context.client.max_bson_size - 37) + big = 'a' * (client_context.max_bson_size - 37) result = self.coll.bulk_write([ UpdateOne({'x': 1}, {'$set': {'s': big}}, upsert=True)]) self.assertEqualResponse( @@ -566,7 +566,7 @@ class TestBulk(BulkTestBase): result) def test_large_inserts_ordered(self): - big = 'x' * self.coll.database.client.max_bson_size + big = 'x' * client_context.max_bson_size requests = [ InsertOne({'b': 1, 'a': 1}), InsertOne({'big': big}), @@ -599,7 +599,7 @@ class TestBulk(BulkTestBase): self.assertEqual(6, self.coll.count_documents({})) def test_large_inserts_unordered(self): - big = 'x' * self.coll.database.client.max_bson_size + big = 'x' * client_context.max_bson_size requests = [ InsertOne({'b': 1, 'a': 1}), InsertOne({'big': big}), diff --git a/test/test_client.py b/test/test_client.py index bad4ed918..2b32dee97 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -640,15 +640,14 @@ class TestClient(IntegrationTest): c = rs_or_single_client(connect=False) self.assertEqual(c.codec_options, CodecOptions()) - self.assertIsInstance(c.max_bson_size, int) c = rs_or_single_client(connect=False) self.assertFalse(c.primary) self.assertFalse(c.secondaries) c = rs_or_single_client(connect=False) - self.assertIsInstance(c.max_write_batch_size, int) self.assertIsInstance(c.topology_description, TopologyDescription) self.assertEqual(c.topology_description, c._topology._description) - + self.assertIsNone(c.address) # PYTHON-2981 + c.admin.command('ping') # connect if client_context.is_rs: # The primary's host and port are from the replica set config. self.assertIsNotNone(c.address) @@ -1837,17 +1836,6 @@ class TestClientLazyConnect(IntegrationTest): lazy_client_trial(reset, find_one, test, self._get_client) - def test_max_bson_size(self): - c = self._get_client() - - # max_bson_size will cause the client to connect. - hello = c.db.command(HelloCompat.LEGACY_CMD) - self.assertEqual(hello['maxBsonObjectSize'], c.max_bson_size) - if 'maxMessageSizeBytes' in hello: - self.assertEqual( - hello['maxMessageSizeBytes'], - c.max_message_size) - class TestMongoClientFailover(MockClientTest): diff --git a/test/test_collection.py b/test/test_collection.py index e81776a6a..bb3795c94 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -851,7 +851,7 @@ class TestCollection(IntegrationTest): lambda: 0 == db.test.count_documents({}), 'delete 2 documents') def test_command_document_too_large(self): - large = '*' * (self.client.max_bson_size + _COMMAND_OVERHEAD) + large = '*' * (client_context.max_bson_size + _COMMAND_OVERHEAD) coll = self.db.test self.assertRaises( DocumentTooLarge, coll.insert_one, {'data': large}) @@ -862,7 +862,7 @@ class TestCollection(IntegrationTest): DocumentTooLarge, coll.delete_one, {'data': large}) def test_write_large_document(self): - max_size = self.db.client.max_bson_size + max_size = client_context.max_bson_size half_size = int(max_size / 2) max_str = "x" * max_size half_str = "x" * half_size @@ -1879,7 +1879,7 @@ class TestCollection(IntegrationTest): def test_numerous_inserts(self): # Ensure we don't exceed server's maxWriteBatchSize size limit. self.db.test.drop() - n_docs = self.client.max_write_batch_size + 100 + n_docs = client_context.max_write_batch_size + 100 self.db.test.insert_many([{} for _ in range(n_docs)]) self.assertEqual(n_docs, self.db.test.count_documents({})) self.db.test.drop() @@ -1888,7 +1888,7 @@ class TestCollection(IntegrationTest): # Tests legacy insert. db = self.client.test_insert_large_batch self.addCleanup(self.client.drop_database, 'test_insert_large_batch') - max_bson_size = self.client.max_bson_size + max_bson_size = client_context.max_bson_size # Write commands are limited to 16MB + 16k per batch big_string = 'x' * int(max_bson_size / 2)