mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-2164 Remove client max_bson_size/max_message_size/max_write_batch_size (#766)

Browse Source 
Use the hello command instead:
doc = client.admin.command('hello')
max_bson_size = doc['maxBsonObjectSize']
max_message_size = doc['maxMessageSizeBytes']
max_write_batch_size = doc['maxWriteBatchSize']

Also add documentation for TopologyDescription.apply_selector.
This commit is contained in:
Shane Harvey 2021-11-01 14:26:47 -07:00 committed by GitHub
parent 695a90e75e
commit 8b2eb24c35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 74 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/api/pymongo/mongo_client.rst
View File

@ -26,9 +26,6 @@
.. autoattribute:: min_pool_size
.. autoattribute:: max_idle_time_ms
.. autoattribute:: nodes
.. autoattribute:: max_bson_size
.. autoattribute:: max_message_size
.. autoattribute:: max_write_batch_size
.. autoattribute:: local_threshold_ms
.. autoattribute:: server_selection_timeout
.. autoattribute:: codec_options

3
doc/changelog.rst
View File

@ -33,6 +33,9 @@ Breaking Changes in 4.0
:meth:`pymongo.mongo_client.MongoClient.unlock`, and
:attr:`pymongo.mongo_client.MongoClient.is_locked`.
- Removed :meth:`pymongo.mongo_client.MongoClient.database_names`.
- Removed :attr:`pymongo.mongo_client.MongoClient.max_bson_size`.
- Removed :attr:`pymongo.mongo_client.MongoClient.max_message_size`.
- Removed :attr:`pymongo.mongo_client.MongoClient.max_write_batch_size`.
- Removed :meth:`pymongo.database.Database.eval`,
:data:`pymongo.database.Database.system_js` and
:class:`pymongo.database.SystemJS`.

23
doc/migrate-to-pymongo4.rst
View File

@ -173,6 +173,29 @@ can be changed to this::
names = client.list_database_names()
MongoClient.max_bson_size/max_message_size/max_write_batch_size are removed
...........................................................................
Removed :attr:`pymongo.mongo_client.MongoClient.max_bson_size`,
:attr:`pymongo.mongo_client.MongoClient.max_message_size`, and
:attr:`pymongo.mongo_client.MongoClient.max_write_batch_size`. These helpers
were incorrect when in ``loadBalanced=true mode`` and ambiguous in clusters
with mixed versions. Use the `hello command`_ to get the authoritative
value from the remote server instead. Code like this::
max_bson_size = client.max_bson_size
max_message_size = client.max_message_size
max_write_batch_size = client.max_write_batch_size
can be changed to this::
doc = client.admin.command('hello')
max_bson_size = doc['maxBsonObjectSize']
max_message_size = doc['maxMessageSizeBytes']
max_write_batch_size = doc['maxWriteBatchSize']
.. _hello command: https://docs.mongodb.com/manual/reference/command/hello/
``tz_aware`` defaults to ``False``
..................................

33
pymongo/mongo_client.py
View File

@ -1053,39 +1053,6 @@ class MongoClient(common.BaseObject):
description = self._topology.description
return frozenset(s.address for s in description.known_servers)
@property
def max_bson_size(self):
"""The largest BSON object the connected server accepts in bytes.
If the client is not connected, this will block until a connection is
established or raise ServerSelectionTimeoutError if no server is
available.
"""
return self._server_property('max_bson_size')
@property
def max_message_size(self):
"""The largest message the connected server accepts in bytes.
If the client is not connected, this will block until a connection is
established or raise ServerSelectionTimeoutError if no server is
available.
"""
return self._server_property('max_message_size')
@property
def max_write_batch_size(self):
"""The maxWriteBatchSize reported by the server.
If the client is not connected, this will block until a connection is
established or raise ServerSelectionTimeoutError if no server is
available.
Returns a default value when connected to server versions prior to
MongoDB 2.6.
"""
return self._server_property('max_write_batch_size')
@property
def local_threshold_ms(self):
"""The local threshold for this instance."""

39
pymongo/topology_description.py
View File

@ -228,21 +228,32 @@ class TopologyDescription(object):
def srv_max_hosts(self):
return self._topology_settings._srv_max_hosts
def apply_selector(self, selector, address, custom_selector=None):
def _apply_local_threshold(self, selection):
if not selection:
return []
# Round trip time in seconds.
fastest = min(
s.round_trip_time for s in selection.server_descriptions)
threshold = self._topology_settings.local_threshold_ms / 1000.0
return [s for s in selection.server_descriptions
if (s.round_trip_time - fastest) <= threshold]
def apply_local_threshold(selection):
if not selection:
return []
def apply_selector(self, selector, address=None, custom_selector=None):
"""List of servers matching the provided selector(s).
settings = self._topology_settings
# Round trip time in seconds.
fastest = min(
s.round_trip_time for s in selection.server_descriptions)
threshold = settings.local_threshold_ms / 1000.0
return [s for s in selection.server_descriptions
if (s.round_trip_time - fastest) <= threshold]
:Parameters:
- `selector`: a callable that takes a Selection as input and returns
a Selection as output. For example, an instance of a read
preference from :mod:`~pymongo.read_preferences`.
- `address` (optional): A server address to select.
- `custom_selector` (optional): A callable that augments server
selection rules. Accepts a list of
:class:`~pymongo.server_description.ServerDescription` objects and
return a list of server descriptions that should be considered
suitable for the desired operation.
.. versionadded:: 3.4
"""
if getattr(selector, 'min_wire_version', 0):
common_wv = self.common_wire_version
if common_wv and common_wv < selector.min_wire_version:
@ -271,7 +282,7 @@ class TopologyDescription(object):
if custom_selector is not None and selection:
selection = selection.with_server_descriptions(
custom_selector(selection.server_descriptions))
return apply_local_threshold(selection)
return self._apply_local_threshold(selection)
def has_readable_server(self, read_preference=ReadPreference.PRIMARY):
"""Does this topology have any readable servers available matching the
@ -288,7 +299,7 @@ class TopologyDescription(object):
.. versionadded:: 3.4
"""
common.validate_read_preference("read_preference", read_preference)
return any(self.apply_selector(read_preference, None))
return any(self.apply_selector(read_preference))
def has_writable_server(self):
"""Does this topology have a writable server available?

14
test/__init__.py
View File

@ -239,6 +239,7 @@ class ClientContext(object):
self.auth_enabled = False
self.test_commands_enabled = False
self.server_parameters = {}
self._hello = None
self.is_mongos = False
self.mongoses = []
self.is_rs = False
@ -274,7 +275,9 @@ class ClientContext(object):
@property
def hello(self):
return self.client.admin.command(HelloCompat.LEGACY_CMD)
if not self._hello:
self._hello = self.client.admin.command(HelloCompat.LEGACY_CMD)
return self._hello
def _connect(self, host, port, **kwargs):
# Jython takes a long time to connect.
@ -391,6 +394,7 @@ class ClientContext(object):
**self.default_client_options)
# Get the authoritative hello result from the primary.
self._hello = None
hello = self.hello
nodes = [partition_node(node.lower())
for node in hello.get('hosts', [])]
@ -866,6 +870,14 @@ class ClientContext(object):
# Changed in SERVER-39567.
return self.version.at_least(4, 1, 10)
@property
def max_bson_size(self):
return self.hello['maxBsonObjectSize']
@property
def max_write_batch_size(self):
return self.hello['maxWriteBatchSize']
# Reusable client context
client_context = ClientContext()

8
test/test_bulk.py
View File

@ -283,7 +283,7 @@ class TestBulk(BulkTestBase):
def test_numerous_inserts(self):
# Ensure we don't exceed server's maxWriteBatchSize size limit.
n_docs = self.client.max_write_batch_size + 100
n_docs = client_context.max_write_batch_size + 100
requests = [InsertOne({}) for _ in range(n_docs)]
result = self.coll.bulk_write(requests, ordered=False)
self.assertEqual(n_docs, result.inserted_count)
@ -344,7 +344,7 @@ class TestBulk(BulkTestBase):
self.coll.bulk_write([{}])
def test_upsert_large(self):
big = 'a' * (client_context.client.max_bson_size - 37)
big = 'a' * (client_context.max_bson_size - 37)
result = self.coll.bulk_write([
UpdateOne({'x': 1}, {'$set': {'s': big}}, upsert=True)])
self.assertEqualResponse(
@ -566,7 +566,7 @@ class TestBulk(BulkTestBase):
result)
def test_large_inserts_ordered(self):
big = 'x' * self.coll.database.client.max_bson_size
big = 'x' * client_context.max_bson_size
requests = [
InsertOne({'b': 1, 'a': 1}),
InsertOne({'big': big}),
@ -599,7 +599,7 @@ class TestBulk(BulkTestBase):
self.assertEqual(6, self.coll.count_documents({}))
def test_large_inserts_unordered(self):
big = 'x' * self.coll.database.client.max_bson_size
big = 'x' * client_context.max_bson_size
requests = [
InsertOne({'b': 1, 'a': 1}),
InsertOne({'big': big}),

16
test/test_client.py
View File

@ -640,15 +640,14 @@ class TestClient(IntegrationTest):
c = rs_or_single_client(connect=False)
self.assertEqual(c.codec_options, CodecOptions())
self.assertIsInstance(c.max_bson_size, int)
c = rs_or_single_client(connect=False)
self.assertFalse(c.primary)
self.assertFalse(c.secondaries)
c = rs_or_single_client(connect=False)
self.assertIsInstance(c.max_write_batch_size, int)
self.assertIsInstance(c.topology_description, TopologyDescription)
self.assertEqual(c.topology_description, c._topology._description)
self.assertIsNone(c.address) # PYTHON-2981
c.admin.command('ping') # connect
if client_context.is_rs:
# The primary's host and port are from the replica set config.
self.assertIsNotNone(c.address)
@ -1837,17 +1836,6 @@ class TestClientLazyConnect(IntegrationTest):
lazy_client_trial(reset, find_one, test, self._get_client)
def test_max_bson_size(self):
c = self._get_client()
# max_bson_size will cause the client to connect.
hello = c.db.command(HelloCompat.LEGACY_CMD)
self.assertEqual(hello['maxBsonObjectSize'], c.max_bson_size)
if 'maxMessageSizeBytes' in hello:
self.assertEqual(
hello['maxMessageSizeBytes'],
c.max_message_size)
class TestMongoClientFailover(MockClientTest):

8
test/test_collection.py
View File

@ -851,7 +851,7 @@ class TestCollection(IntegrationTest):
lambda: 0 == db.test.count_documents({}), 'delete 2 documents')
def test_command_document_too_large(self):
large = '*' * (self.client.max_bson_size + _COMMAND_OVERHEAD)
large = '*' * (client_context.max_bson_size + _COMMAND_OVERHEAD)
coll = self.db.test
self.assertRaises(
DocumentTooLarge, coll.insert_one, {'data': large})
@ -862,7 +862,7 @@ class TestCollection(IntegrationTest):
DocumentTooLarge, coll.delete_one, {'data': large})
def test_write_large_document(self):
max_size = self.db.client.max_bson_size
max_size = client_context.max_bson_size
half_size = int(max_size / 2)
max_str = "x" * max_size
half_str = "x" * half_size
@ -1879,7 +1879,7 @@ class TestCollection(IntegrationTest):
def test_numerous_inserts(self):
# Ensure we don't exceed server's maxWriteBatchSize size limit.
self.db.test.drop()
n_docs = self.client.max_write_batch_size + 100
n_docs = client_context.max_write_batch_size + 100
self.db.test.insert_many([{} for _ in range(n_docs)])
self.assertEqual(n_docs, self.db.test.count_documents({}))
self.db.test.drop()
@ -1888,7 +1888,7 @@ class TestCollection(IntegrationTest):
# Tests legacy insert.
db = self.client.test_insert_large_batch
self.addCleanup(self.client.drop_database, 'test_insert_large_batch')
max_bson_size = self.client.max_bson_size
max_bson_size = client_context.max_bson_size
# Write commands are limited to 16MB + 16k per batch
big_string = 'x' * int(max_bson_size / 2)