From e01d9a37e73a2054fd7cdcd75246debd553c3f6e Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jan 2021 17:11:15 -0800 Subject: [PATCH] PYTHON-1320 Remove legacy CRUD methods (#556) Remove save, insert, update, remove, and find_and_modify. Remove tools/benchmark.py --- bson/regex.py | 2 +- doc/api/pymongo/collection.rst | 5 - doc/changelog.rst | 5 + doc/compatibility-policy.rst | 2 +- doc/faq.rst | 7 +- doc/migrate-to-pymongo3.rst | 4 +- doc/migrate-to-pymongo4.rst | 105 +++- pymongo/collection.py | 239 +------- pymongo/helpers.py | 11 - test/test_bulk.py | 4 +- test/test_collation.py | 27 - test/test_collection.py | 133 +++-- test/test_custom_types.py | 18 +- test/test_legacy_api.py | 1009 -------------------------------- test/test_monitoring.py | 310 ++-------- test/test_retryable_writes.py | 46 +- test/test_write_concern.py | 7 + tools/benchmark.py | 165 ------ 18 files changed, 273 insertions(+), 1826 deletions(-) delete mode 100644 tools/benchmark.py diff --git a/bson/regex.py b/bson/regex.py index 0624f9ab3..3a9042500 100644 --- a/bson/regex.py +++ b/bson/regex.py @@ -53,7 +53,7 @@ class Regex(object): >>> pattern = re.compile('.*') >>> regex = Regex.from_native(pattern) >>> regex.flags ^= re.UNICODE - >>> db.collection.insert({'pattern': regex}) + >>> db.collection.insert_one({'pattern': regex}) :Parameters: - `regex`: A regular expression object from ``re.compile()``. diff --git a/doc/api/pymongo/collection.rst b/doc/api/pymongo/collection.rst index 98df4432e..ada01352e 100644 --- a/doc/api/pymongo/collection.rst +++ b/doc/api/pymongo/collection.rst @@ -71,8 +71,3 @@ .. automethod:: initialize_ordered_bulk_op .. automethod:: group .. automethod:: count - .. automethod:: insert(doc_or_docs, manipulate=True, check_keys=True, continue_on_error=False, **kwargs) - .. automethod:: save(to_save, manipulate=True, check_keys=True, **kwargs) - .. automethod:: update(spec, document, upsert=False, manipulate=False, multi=False, check_keys=True, **kwargs) - .. automethod:: remove(spec_or_id=None, multi=True, **kwargs) - .. automethod:: find_and_modify diff --git a/doc/changelog.rst b/doc/changelog.rst index 248372e15..db50129f3 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -30,6 +30,11 @@ Breaking Changes in 4.0 - Removed :meth:`pymongo.collection.Collection.parallel_scan`. - Removed :meth:`pymongo.collection.Collection.ensure_index`. - Removed :meth:`pymongo.collection.Collection.reindex`. +- Removed :meth:`pymongo.collection.Collection.save` +- Removed :meth:`pymongo.collection.Collection.insert` +- Removed :meth:`pymongo.collection.Collection.update` +- Removed :meth:`pymongo.collection.Collection.remove` +- Removed :meth:`pymongo.collection.Collection.find_and_modify` - Removed :meth:`pymongo.mongo_client.MongoClient.close_cursor`. Use :meth:`pymongo.cursor.Cursor.close` instead. - Removed :meth:`pymongo.mongo_client.MongoClient.kill_cursors`. diff --git a/doc/compatibility-policy.rst b/doc/compatibility-policy.rst index ed22c9735..a20b9681e 100644 --- a/doc/compatibility-policy.rst +++ b/doc/compatibility-policy.rst @@ -24,7 +24,7 @@ warning: .. code-block:: python - # "insert.py" + # "insert.py" (with PyMongo 3.X) from pymongo import MongoClient client = MongoClient() diff --git a/doc/faq.rst b/doc/faq.rst index 99d8f1db0..5efb38079 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -202,6 +202,9 @@ documents that already have an ``_id`` field, added by your application. Key order in subdocuments -- why does my query work in the shell but not PyMongo? --------------------------------------------------------------------------------- +.. + Note: We should rework this section now that Python 3.6+ has ordered dict. + .. testsetup:: key-order from bson.son import SON @@ -220,9 +223,9 @@ is displayed: .. code-block:: javascript > // mongo shell. - > db.collection.insert( { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } ) + > db.collection.insertOne( { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } ) WriteResult({ "nInserted" : 1 }) - > db.collection.find() + > db.collection.findOne() { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } PyMongo represents BSON documents as Python dicts by default, and the order diff --git a/doc/migrate-to-pymongo3.rst b/doc/migrate-to-pymongo3.rst index bb396ddee..9b30fc36f 100644 --- a/doc/migrate-to-pymongo3.rst +++ b/doc/migrate-to-pymongo3.rst @@ -340,14 +340,14 @@ this:: >>> oid = collection.insert({"a": 2}, w="majority") -can be changed to this with PyMongo 2.9 or later: +can be changed to this with PyMongo 3 or later: .. doctest:: >>> from pymongo import WriteConcern >>> coll2 = collection.with_options( ... write_concern=WriteConcern(w="majority")) - >>> oid = coll2.insert({"a": 2}) + >>> oid = coll2.insert_one({"a": 2}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` diff --git a/doc/migrate-to-pymongo4.rst b/doc/migrate-to-pymongo4.rst index 68788c05b..cf05114ca 100644 --- a/doc/migrate-to-pymongo4.rst +++ b/doc/migrate-to-pymongo4.rst @@ -140,6 +140,103 @@ can be changed to this:: Collection ---------- +Collection.insert is removed +............................ + +Removed :meth:`pymongo.collection.Collection.insert`. Use +:meth:`~pymongo.collection.Collection.insert_one` or +:meth:`~pymongo.collection.Collection.insert_many` instead. + +Code like this:: + + collection.insert({'doc': 1}) + collection.insert([{'doc': 2}, {'doc': 3}]) + +Can be changed to this:: + + collection.insert_one({'my': 'document'}) + collection.insert_many([{'doc': 2}, {'doc': 3}]) + +Collection.save is removed +.......................... + +Removed :meth:`pymongo.collection.Collection.save`. Applications will +get better performance using :meth:`~pymongo.collection.Collection.insert_one` +to insert a new document and :meth:`~pymongo.collection.Collection.update_one` +to update an existing document. Code like this:: + + doc = collection.find_one({"_id": "some id"}) + doc["some field"] = + db.collection.save(doc) + +Can be changed to this:: + + result = collection.update_one({"_id": "some id"}, {"$set": {"some field": }}) + +If performance is not a concern and refactoring is untenable, ``save`` can be +implemented like so:: + + def save(doc): + if '_id' in doc: + collection.replace_one({'_id': doc['_id']}, doc, upsert=True) + return doc['_id'] + else: + res = collection.insert_one(doc) + return res.inserted_id + +Collection.update is removed +............................ + +Removed :meth:`pymongo.collection.Collection.update`. Use +:meth:`~pymongo.collection.Collection.update_one` +to update a single document or +:meth:`~pymongo.collection.Collection.update_many` to update multiple +documents. Code like this:: + + collection.update({}, {'$set': {'a': 1}}) + collection.update({}, {'$set': {'b': 1}}, multi=True) + +Can be changed to this:: + + collection.update_one({}, {'$set': {'a': 1}}) + collection.update_many({}, {'$set': {'b': 1}}) + +Collection.remove is removed +............................ + +Removed :meth:`pymongo.collection.Collection.remove`. Use +:meth:`~pymongo.collection.Collection.delete_one` +to delete a single document or +:meth:`~pymongo.collection.Collection.delete_many` to delete multiple +documents. Code like this:: + + collection.remove({'a': 1}, multi=False) + collection.remove({'b': 1}) + +Can be changed to this:: + + collection.delete_one({'a': 1}) + collection.delete_many({'b': 1}) + +Collection.find_and_modify is removed +..................................... + +Removed :meth:`pymongo.collection.Collection.find_and_modify`. Use +:meth:`~pymongo.collection.Collection.find_one_and_update`, +:meth:`~pymongo.collection.Collection.find_one_and_replace`, or +:meth:`~pymongo.collection.Collection.find_one_and_delete` instead. +Code like this:: + + updated_doc = collection.find_and_modify({'a': 1}, {'$set': {'b': 1}}) + replaced_doc = collection.find_and_modify({'b': 1}, {'c': 1}) + deleted_doc = collection.find_and_modify({'c': 1}, remove=True) + +Can be changed to this:: + + updated_doc = collection.find_one_and_update({'a': 1}, {'$set': {'b': 1}}) + replaced_doc = collection.find_one_and_replace({'b': 1}, {'c': 1}) + deleted_doc = collection.find_one_and_delete({'c': 1}) + Collection.ensure_index is removed .................................. @@ -152,16 +249,16 @@ to :meth:`~pymongo.collection.Collection.create_index` or :meth:`~pymongo.collection.Collection.create_indexes`. Code like this:: def persist(self, document): - my_collection.ensure_index('a', unique=True) - my_collection.insert_one(document) + collection.ensure_index('a', unique=True) + collection.insert_one(document) Can be changed to this:: def persist(self, document): if not self.created_index: - my_collection.create_index('a', unique=True) + collection.create_index('a', unique=True) self.created_index = True - my_collection.insert_one(document) + collection.insert_one(document) Collection.reindex is removed ............................. diff --git a/pymongo/collection.py b/pymongo/collection.py index 97787f3d6..d600f2707 100644 --- a/pymongo/collection.py +++ b/pymongo/collection.py @@ -31,17 +31,14 @@ from pymongo.aggregation import (_CollectionAggregationCommand, _CollectionRawAggregationCommand) from pymongo.bulk import BulkOperationBuilder, _Bulk from pymongo.command_cursor import CommandCursor, RawBatchCommandCursor -from pymongo.common import ORDERED_TYPES from pymongo.collation import validate_collation_or_none from pymongo.change_stream import CollectionChangeStream from pymongo.cursor import Cursor, RawBatchCursor -from pymongo.errors import (BulkWriteError, - ConfigurationError, +from pymongo.errors import (ConfigurationError, InvalidName, InvalidOperation, OperationFailure) -from pymongo.helpers import (_check_write_command_response, - _raise_last_error) +from pymongo.helpers import _check_write_command_response from pymongo.message import _UNICODE_REPLACE_CODEC_OPTIONS from pymongo.operations import IndexModel from pymongo.read_preferences import ReadPreference @@ -604,52 +601,6 @@ class Collection(common.BaseObject): if not isinstance(doc, RawBSONDocument): return doc.get('_id') - def _insert(self, docs, ordered=True, check_keys=True, - manipulate=False, write_concern=None, op_id=None, - bypass_doc_val=False, session=None): - """Internal insert helper.""" - if isinstance(docs, abc.Mapping): - return self._insert_one( - docs, ordered, check_keys, manipulate, write_concern, op_id, - bypass_doc_val, session) - - ids = [] - - if manipulate: - def gen(): - """Generator that applies SON manipulators to each document - and adds _id if necessary. - """ - _db = self.__database - for doc in docs: - # Apply user-configured SON manipulators. This order of - # operations is required for backwards compatibility, - # see PYTHON-709. - doc = _db._apply_incoming_manipulators(doc, self) - if not (isinstance(doc, RawBSONDocument) or '_id' in doc): - doc['_id'] = ObjectId() - - doc = _db._apply_incoming_copying_manipulators(doc, self) - ids.append(doc['_id']) - yield doc - else: - def gen(): - """Generator that only tracks existing _ids.""" - for doc in docs: - # Don't inflate RawBSONDocument by touching fields. - if not isinstance(doc, RawBSONDocument): - ids.append(doc.get('_id')) - yield doc - - write_concern = write_concern or self._write_concern_for(session) - blk = _Bulk(self, ordered, bypass_doc_val) - blk.ops = [(message._INSERT, doc) for doc in gen()] - try: - blk.execute(write_concern, session=session) - except BulkWriteError as bwe: - _raise_last_error(bwe.details) - return ids - def insert_one(self, document, bypass_document_validation=False, session=None): """Insert a single document. @@ -694,10 +645,10 @@ class Collection(common.BaseObject): write_concern = self._write_concern_for(session) return InsertOneResult( - self._insert(document, - write_concern=write_concern, - bypass_doc_val=bypass_document_validation, - session=session), + self._insert_one( + document, ordered=True, check_keys=True, manipulate=False, + write_concern=write_concern, op_id=None, + bypass_doc_val=bypass_document_validation, session=session), write_concern.acknowledged) def insert_many(self, documents, ordered=True, @@ -3068,184 +3019,6 @@ class Collection(common.BaseObject): array_filters, hint=hint, session=session, **kwargs) - def save(self, to_save, manipulate=True, check_keys=True, **kwargs): - """Save a document in this collection. - - **DEPRECATED** - Use :meth:`insert_one` or :meth:`replace_one` instead. - - .. versionchanged:: 3.0 - Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write - operations. - """ - warnings.warn("save is deprecated. Use insert_one or replace_one " - "instead", DeprecationWarning, stacklevel=2) - common.validate_is_document_type("to_save", to_save) - - write_concern = None - collation = validate_collation_or_none(kwargs.pop('collation', None)) - if kwargs: - write_concern = WriteConcern(**kwargs) - - if not (isinstance(to_save, RawBSONDocument) or "_id" in to_save): - return self._insert( - to_save, True, check_keys, manipulate, write_concern) - else: - self._update_retryable( - {"_id": to_save["_id"]}, to_save, True, - check_keys, False, manipulate, write_concern, - collation=collation) - return to_save.get("_id") - - def insert(self, doc_or_docs, manipulate=True, - check_keys=True, continue_on_error=False, **kwargs): - """Insert a document(s) into this collection. - - **DEPRECATED** - Use :meth:`insert_one` or :meth:`insert_many` instead. - - .. versionchanged:: 3.0 - Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write - operations. - """ - warnings.warn("insert is deprecated. Use insert_one or insert_many " - "instead.", DeprecationWarning, stacklevel=2) - write_concern = None - if kwargs: - write_concern = WriteConcern(**kwargs) - return self._insert(doc_or_docs, not continue_on_error, - check_keys, manipulate, write_concern) - - def update(self, spec, document, upsert=False, manipulate=False, - multi=False, check_keys=True, **kwargs): - """Update a document(s) in this collection. - - **DEPRECATED** - Use :meth:`replace_one`, :meth:`update_one`, or - :meth:`update_many` instead. - - .. versionchanged:: 3.0 - Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write - operations. - """ - warnings.warn("update is deprecated. Use replace_one, update_one or " - "update_many instead.", DeprecationWarning, stacklevel=2) - common.validate_is_mapping("spec", spec) - common.validate_is_mapping("document", document) - if document: - # If a top level key begins with '$' this is a modify operation - # and we should skip key validation. It doesn't matter which key - # we check here. Passing a document with a mix of top level keys - # starting with and without a '$' is invalid and the server will - # raise an appropriate exception. - first = next(iter(document)) - if first.startswith('$'): - check_keys = False - - write_concern = None - collation = validate_collation_or_none(kwargs.pop('collation', None)) - if kwargs: - write_concern = WriteConcern(**kwargs) - return self._update_retryable( - spec, document, upsert, check_keys, multi, manipulate, - write_concern, collation=collation) - - def remove(self, spec_or_id=None, multi=True, **kwargs): - """Remove a document(s) from this collection. - - **DEPRECATED** - Use :meth:`delete_one` or :meth:`delete_many` instead. - - .. versionchanged:: 3.0 - Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write - operations. - """ - warnings.warn("remove is deprecated. Use delete_one or delete_many " - "instead.", DeprecationWarning, stacklevel=2) - if spec_or_id is None: - spec_or_id = {} - if not isinstance(spec_or_id, abc.Mapping): - spec_or_id = {"_id": spec_or_id} - write_concern = None - collation = validate_collation_or_none(kwargs.pop('collation', None)) - if kwargs: - write_concern = WriteConcern(**kwargs) - return self._delete_retryable( - spec_or_id, multi, write_concern, collation=collation) - - def find_and_modify(self, query=None, update=None, - upsert=False, sort=None, full_response=False, - manipulate=False, **kwargs): - """Update and return an object. - - **DEPRECATED** - Use :meth:`find_one_and_delete`, - :meth:`find_one_and_replace`, or :meth:`find_one_and_update` instead. - """ - warnings.warn("find_and_modify is deprecated, use find_one_and_delete" - ", find_one_and_replace, or find_one_and_update instead", - DeprecationWarning, stacklevel=2) - - if not update and not kwargs.get('remove', None): - raise ValueError("Must either update or remove") - - if update and kwargs.get('remove', None): - raise ValueError("Can't do both update and remove") - - # No need to include empty args - if query: - kwargs['query'] = query - if update: - kwargs['update'] = update - if upsert: - kwargs['upsert'] = upsert - if sort: - # Accept a list of tuples to match Cursor's sort parameter. - if isinstance(sort, list): - kwargs['sort'] = helpers._index_document(sort) - # Accept OrderedDict, SON, and dict with len == 1 so we - # don't break existing code already using find_and_modify. - elif (isinstance(sort, ORDERED_TYPES) or - isinstance(sort, dict) and len(sort) == 1): - warnings.warn("Passing mapping types for `sort` is deprecated," - " use a list of (key, direction) pairs instead", - DeprecationWarning, stacklevel=2) - kwargs['sort'] = sort - else: - raise TypeError("sort must be a list of (key, direction) " - "pairs, a dict of len 1, or an instance of " - "SON or OrderedDict") - - fields = kwargs.pop("fields", None) - if fields is not None: - kwargs["fields"] = helpers._fields_list_to_dict(fields, "fields") - - collation = validate_collation_or_none(kwargs.pop('collation', None)) - - cmd = SON([("findAndModify", self.__name)]) - cmd.update(kwargs) - - write_concern = self._write_concern_for_cmd(cmd, None) - - def _find_and_modify(session, sock_info, retryable_write): - if (sock_info.max_wire_version >= 4 and - not write_concern.is_server_default): - cmd['writeConcern'] = write_concern.document - result = self._command( - sock_info, cmd, read_preference=ReadPreference.PRIMARY, - collation=collation, - session=session, retryable_write=retryable_write, - user_fields=_FIND_AND_MODIFY_DOC_FIELDS) - - _check_write_command_response(result) - return result - - out = self.__database.client._retryable_write( - write_concern.acknowledged, _find_and_modify, None) - - if full_response: - return out - else: - document = out.get('value') - if manipulate: - document = self.__database._fix_outgoing(document, self) - return document - def __iter__(self): return self diff --git a/pymongo/helpers.py b/pymongo/helpers.py index 4d306c6ba..107e061ac 100644 --- a/pymongo/helpers.py +++ b/pymongo/helpers.py @@ -234,17 +234,6 @@ def _check_write_command_response(result): _raise_write_concern_error(error) -def _raise_last_error(bulk_write_result): - """Backward compatibility helper for insert error handling. - """ - # Prefer write errors over write concern errors - write_errors = bulk_write_result.get("writeErrors") - if write_errors: - _raise_last_write_error(write_errors) - - _raise_write_concern_error(bulk_write_result["writeConcernErrors"][-1]) - - def _fields_list_to_dict(fields, option_name): """Takes a sequence of field names and returns a matching dictionary. diff --git a/test/test_bulk.py b/test/test_bulk.py index 22e971d57..7d8176e57 100644 --- a/test/test_bulk.py +++ b/test/test_bulk.py @@ -295,8 +295,8 @@ class TestBulk(BulkTestBase): self.assertEqual(self.coll.count_documents({'foo': 'bar'}), 1) def test_numerous_inserts(self): - # Ensure we don't exceed server's 1000-document batch size limit. - n_docs = 2100 + # Ensure we don't exceed server's maxWriteBatchSize size limit. + n_docs = self.client.max_write_batch_size + 100 requests = [InsertOne({}) for _ in range(n_docs)] result = self.coll.bulk_write(requests, ordered=False) self.assertEqual(n_docs, result.inserted_count) diff --git a/test/test_collation.py b/test/test_collation.py index c8d6c15e1..c6d9baa46 100644 --- a/test/test_collation.py +++ b/test/test_collation.py @@ -217,30 +217,8 @@ class TestCollation(unittest.TestCase): self.collation.document, command['deletes'][0]['collation']) - self.listener.results.clear() - self.db.test.remove({'foo': 42}, collation=self.collation) - command = self.listener.results['started'][0].command - self.assertEqual( - self.collation.document, - command['deletes'][0]['collation']) - @raisesConfigurationErrorForOldMongoDB def test_update(self): - self.db.test.update({'foo': 42}, {'$set': {'foo': 'bar'}}, - collation=self.collation) - command = self.listener.results['started'][0].command - self.assertEqual( - self.collation.document, - command['updates'][0]['collation']) - - self.listener.results.clear() - self.db.test.save({'_id': 12345}, collation=self.collation) - command = self.listener.results['started'][0].command - self.assertEqual( - self.collation.document, - command['updates'][0]['collation']) - - self.listener.results.clear() self.db.test.replace_one({'foo': 42}, {'foo': 43}, collation=self.collation) command = self.listener.results['started'][0].command @@ -266,11 +244,6 @@ class TestCollation(unittest.TestCase): @raisesConfigurationErrorForOldMongoDB def test_find_and(self): - self.db.test.find_and_modify({'foo': 42}, {'$set': {'foo': 43}}, - collation=self.collation) - self.assertCollationInLastCommand() - - self.listener.results.clear() self.db.test.find_one_and_delete({'foo': 42}, collation=self.collation) self.assertCollationInLastCommand() diff --git a/test/test_collection.py b/test/test_collection.py index 3600954c0..0ac4aed2a 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -847,6 +847,36 @@ class TestCollection(IntegrationTest): self.assertRaises( DocumentTooLarge, coll.delete_one, {'data': large}) + def test_write_large_document(self): + max_size = self.db.client.max_bson_size + half_size = int(max_size / 2) + max_str = "x" * max_size + half_str = "x" * half_size + self.assertEqual(max_size, 16777216) + + self.assertRaises(OperationFailure, self.db.test.insert_one, + {"foo": max_str}) + self.assertRaises(OperationFailure, self.db.test.replace_one, + {}, {"foo": max_str}, upsert=True) + self.assertRaises(OperationFailure, self.db.test.insert_many, + [{"x": 1}, {"foo": max_str}]) + self.db.test.insert_many([{"foo": half_str}, {"foo": half_str}]) + + self.db.test.insert_one({"bar": "x"}) + # Use w=0 here to test legacy doc size checking in all server versions + unack_coll = self.db.test.with_options(write_concern=WriteConcern(w=0)) + self.assertRaises(DocumentTooLarge, unack_coll.replace_one, + {"bar": "x"}, {"bar": "x" * (max_size - 14)}) + # This will pass with OP_UPDATE or the update command. + self.db.test.replace_one({"bar": "x"}, {"bar": "x" * (max_size - 32)}) + + def test_bad_dbref(self): + # Incomplete DBRefs. + ref_only = {'ref': {'$ref': 'collection'}} + id_only = {'ref': {'$id': ObjectId()}} + self.assertRaises(InvalidDocument, self.db.test.insert_one, ref_only) + self.assertRaises(InvalidDocument, self.db.test.insert_one, id_only) + @client_context.require_version_min(3, 1, 9, -1) def test_insert_bypass_document_validation(self): db = self.db @@ -1269,26 +1299,12 @@ class TestCollection(IntegrationTest): db.test.insert_one, {"text": text}) - self.assertRaises(DuplicateKeyError, - db.test.insert, - {"text": text}) - - self.assertRaises(DuplicateKeyError, - db.test.insert, - [{"text": text}]) - self.assertRaises(DuplicateKeyError, db.test.replace_one, {"_id": ObjectId()}, {"text": text}, upsert=True) - self.assertRaises(DuplicateKeyError, - db.test.update, - {"_id": ObjectId()}, - {"text": text}, - upsert=True) - # Should raise BulkWriteError, not InvalidBSON self.assertRaises(BulkWriteError, db.test.insert_many, @@ -1941,13 +1957,79 @@ class TestCollection(IntegrationTest): self.assertEqual(2, docs[0]["x"]) def test_numerous_inserts(self): - # Ensure we don't exceed server's 1000-document batch size limit. + # Ensure we don't exceed server's maxWriteBatchSize size limit. self.db.test.drop() - n_docs = 2100 + n_docs = self.client.max_write_batch_size + 100 self.db.test.insert_many([{} for _ in range(n_docs)]) self.assertEqual(n_docs, self.db.test.count_documents({})) self.db.test.drop() + def test_insert_many_large_batch(self): + # Tests legacy insert. + db = self.client.test_insert_large_batch + self.addCleanup(self.client.drop_database, 'test_insert_large_batch') + max_bson_size = self.client.max_bson_size + # Write commands are limited to 16MB + 16k per batch + big_string = 'x' * int(max_bson_size / 2) + + # Batch insert that requires 2 batches. + successful_insert = [{'x': big_string}, {'x': big_string}, + {'x': big_string}, {'x': big_string}] + db.collection_0.insert_many(successful_insert) + self.assertEqual(4, db.collection_0.count_documents({})) + + db.collection_0.drop() + + # Test that inserts fail after first error. + insert_second_fails = [{'_id': 'id0', 'x': big_string}, + {'_id': 'id0', 'x': big_string}, + {'_id': 'id1', 'x': big_string}, + {'_id': 'id2', 'x': big_string}] + + with self.assertRaises(BulkWriteError): + db.collection_1.insert_many(insert_second_fails) + + self.assertEqual(1, db.collection_1.count_documents({})) + + db.collection_1.drop() + + # 2 batches, 2nd insert fails, unacknowledged, ordered. + unack_coll = db.collection_2.with_options( + write_concern=WriteConcern(w=0)) + unack_coll.insert_many(insert_second_fails) + wait_until(lambda: 1 == db.collection_2.count_documents({}), + 'insert 1 document', timeout=60) + + db.collection_2.drop() + + # 2 batches, ids of docs 0 and 1 are dupes, ids of docs 2 and 3 are + # dupes. Acknowledged, unordered. + insert_two_failures = [{'_id': 'id0', 'x': big_string}, + {'_id': 'id0', 'x': big_string}, + {'_id': 'id1', 'x': big_string}, + {'_id': 'id1', 'x': big_string}] + + with self.assertRaises(OperationFailure) as context: + db.collection_3.insert_many(insert_two_failures, ordered=False) + + self.assertIn('id1', str(context.exception)) + + # Only the first and third documents should be inserted. + self.assertEqual(2, db.collection_3.count_documents({})) + + db.collection_3.drop() + + # 2 batches, 2 errors, unacknowledged, unordered. + unack_coll = db.collection_4.with_options( + write_concern=WriteConcern(w=0)) + unack_coll.insert_many(insert_two_failures, ordered=False) + + # Only the first and third documents are inserted. + wait_until(lambda: 2 == db.collection_4.count_documents({}), + 'insert 2 documents', timeout=60) + + db.collection_4.drop() + def test_map_reduce(self): db = self.db db.drop_collection("test") @@ -2168,12 +2250,6 @@ class TestCollection(IntegrationTest): db.command('ismaster') results.clear() if client_context.version.at_least(3, 1, 9, -1): - c_w0.find_and_modify( - {'_id': 1}, {'$set': {'foo': 'bar'}}) - self.assertEqual( - {'w': 0}, results['started'][0].command['writeConcern']) - results.clear() - c_w0.find_one_and_update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual( @@ -2196,10 +2272,6 @@ class TestCollection(IntegrationTest): 'test', write_concern=WriteConcern( w=len(client_context.nodes) + 1)) - self.assertRaises( - WriteConcernError, - c_wc_error.find_and_modify, - {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_update, @@ -2214,11 +2286,6 @@ class TestCollection(IntegrationTest): {'w': 0}, results['started'][0].command['writeConcern']) results.clear() else: - c_w0.find_and_modify( - {'_id': 1}, {'$set': {'foo': 'bar'}}) - self.assertNotIn('writeConcern', results['started'][0].command) - results.clear() - c_w0.find_one_and_update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) @@ -2232,10 +2299,6 @@ class TestCollection(IntegrationTest): self.assertNotIn('writeConcern', results['started'][0].command) results.clear() - c_default.find_and_modify({'_id': 1}, {'$set': {'foo': 'bar'}}) - self.assertNotIn('writeConcern', results['started'][0].command) - results.clear() - c_default.find_one_and_update({'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() diff --git a/test/test_custom_types.py b/test/test_custom_types.py index e48acedff..c74da2479 100644 --- a/test/test_custom_types.py +++ b/test/test_custom_types.py @@ -49,7 +49,7 @@ from pymongo.message import _CursorAddress from test import client_context, unittest from test.test_client import IntegrationTest -from test.utils import ignore_deprecations, rs_client +from test.utils import rs_client class DecimalEncoder(TypeEncoder): @@ -692,22 +692,6 @@ class TestCollectionWCustomType(IntegrationTest): self.assertEqual(doc['x'].value, 3) self.assertIsNone(c.find_one()) - @ignore_deprecations - def test_find_and_modify_w_custom_type_decoder(self): - db = self.db - c = db.get_collection('test', codec_options=UNINT_DECODER_CODECOPTS) - c.insert_one({'_id': 1, 'x': Int64(1)}) - - doc = c.find_and_modify({'_id': 1}, {'$inc': {'x': Int64(10)}}) - self.assertEqual(doc['_id'], 1) - self.assertIsInstance(doc['x'], UndecipherableInt64Type) - self.assertEqual(doc['x'].value, 1) - - doc = c.find_one() - self.assertEqual(doc['_id'], 1) - self.assertIsInstance(doc['x'], UndecipherableInt64Type) - self.assertEqual(doc['x'].value, 11) - class TestGridFileCustomType(IntegrationTest): def setUp(self): diff --git a/test/test_legacy_api.py b/test/test_legacy_api.py index 6b4e7209e..b4e65d128 100644 --- a/test/test_legacy_api.py +++ b/test/test_legacy_api.py @@ -31,7 +31,6 @@ from pymongo import ASCENDING, DESCENDING, GEOHAYSTACK from pymongo.common import partition_node from pymongo.errors import (BulkWriteError, ConfigurationError, - DocumentTooLarge, DuplicateKeyError, InvalidDocument, InvalidOperation, @@ -48,9 +47,7 @@ from test import client_context, qcheck, unittest, SkipTest from test.test_client import IntegrationTest from test.test_bulk import BulkTestBase, BulkAuthorizationTestBase from test.utils import (DeprecationFilter, - joinall, oid_generated_on_process, - rs_or_single_client, rs_or_single_client_noauth, single_client, wait_until) @@ -67,27 +64,6 @@ class TestDeprecations(IntegrationTest): def tearDownClass(cls): cls.deprecation_filter.stop() - def test_save_deprecation(self): - self.assertRaises( - DeprecationWarning, lambda: self.db.test.save({})) - - def test_insert_deprecation(self): - self.assertRaises( - DeprecationWarning, lambda: self.db.test.insert({})) - - def test_update_deprecation(self): - self.assertRaises( - DeprecationWarning, lambda: self.db.test.update({}, {})) - - def test_remove_deprecation(self): - self.assertRaises( - DeprecationWarning, lambda: self.db.test.remove({})) - - def test_find_and_modify_deprecation(self): - self.assertRaises( - DeprecationWarning, - lambda: self.db.test.find_and_modify({'i': 5}, {})) - def test_add_son_manipulator_deprecation(self): db = self.client.pymongo_test self.assertRaises(DeprecationWarning, @@ -115,795 +91,6 @@ class TestLegacy(IntegrationTest): def tearDownClass(cls): cls.deprecation_filter.stop() - def test_insert_find_one(self): - # Tests legacy insert. - db = self.db - db.test.drop() - self.assertEqual(0, len(list(db.test.find()))) - doc = {"hello": "world"} - _id = db.test.insert(doc) - self.assertEqual(1, len(list(db.test.find()))) - self.assertEqual(doc, db.test.find_one()) - self.assertEqual(doc["_id"], _id) - self.assertTrue(isinstance(_id, ObjectId)) - - db = self.client.get_database( - db.name, codec_options=CodecOptions(document_class=dict)) - - def remove_insert_find_one(doc): - db.test.remove({}) - db.test.insert(doc) - # SON equality is order sensitive. - return db.test.find_one() == doc.to_dict() - - qcheck.check_unittest(self, remove_insert_find_one, - qcheck.gen_mongo_dict(3)) - - def test_generator_insert(self): - # Only legacy insert currently supports insert from a generator. - db = self.db - db.test.remove({}) - self.assertEqual(db.test.find().count(), 0) - db.test.insert(({'a': i} for i in range(5)), manipulate=False) - self.assertEqual(5, db.test.count()) - db.test.remove({}) - - db.test.insert(({'a': i} for i in range(5)), manipulate=True) - self.assertEqual(5, db.test.count()) - db.test.remove({}) - - def test_insert_multiple(self): - # Tests legacy insert. - db = self.db - db.drop_collection("test") - doc1 = {"hello": "world"} - doc2 = {"hello": "mike"} - self.assertEqual(db.test.find().count(), 0) - ids = db.test.insert([doc1, doc2]) - self.assertEqual(db.test.find().count(), 2) - self.assertEqual(doc1, db.test.find_one({"hello": "world"})) - self.assertEqual(doc2, db.test.find_one({"hello": "mike"})) - - self.assertEqual(2, len(ids)) - self.assertEqual(doc1["_id"], ids[0]) - self.assertEqual(doc2["_id"], ids[1]) - - ids = db.test.insert([{"hello": 1}]) - self.assertTrue(isinstance(ids, list)) - self.assertEqual(1, len(ids)) - - self.assertRaises(InvalidOperation, db.test.insert, []) - - # Generator that raises StopIteration on first call to next(). - self.assertRaises(InvalidOperation, db.test.insert, (i for i in [])) - - def test_insert_multiple_with_duplicate(self): - # Tests legacy insert. - db = self.db - db.drop_collection("test_insert_multiple_with_duplicate") - collection = db.test_insert_multiple_with_duplicate - collection.create_index([('i', ASCENDING)], unique=True) - - # No error - collection.insert([{'i': i} for i in range(5, 10)], w=0) - wait_until(lambda: 5 == collection.count(), 'insert 5 documents') - - db.drop_collection("test_insert_multiple_with_duplicate") - collection.create_index([('i', ASCENDING)], unique=True) - - # No error - collection.insert([{'i': 1}] * 2, w=0) - wait_until(lambda: 1 == collection.count(), 'insert 1 document') - - self.assertRaises( - DuplicateKeyError, - lambda: collection.insert([{'i': 2}] * 2), - ) - - db.drop_collection("test_insert_multiple_with_duplicate") - db = self.client.get_database( - db.name, write_concern=WriteConcern(w=0)) - - collection = db.test_insert_multiple_with_duplicate - collection.create_index([('i', ASCENDING)], unique=True) - - # No error. - collection.insert([{'i': 1}] * 2) - wait_until(lambda: 1 == collection.count(), 'insert 1 document') - - # Implied acknowledged. - self.assertRaises( - DuplicateKeyError, - lambda: collection.insert([{'i': 2}] * 2, fsync=True), - ) - - # Explicit acknowledged. - self.assertRaises( - DuplicateKeyError, - lambda: collection.insert([{'i': 2}] * 2, w=1)) - - db.drop_collection("test_insert_multiple_with_duplicate") - - @client_context.require_replica_set - def test_insert_prefers_write_errors(self): - # Tests legacy insert. - collection = self.db.test_insert_prefers_write_errors - self.db.drop_collection(collection.name) - collection.insert_one({'_id': 1}) - large = 's' * 1024 * 1024 * 15 - with self.assertRaises(DuplicateKeyError): - collection.insert( - [{'_id': 1, 's': large}, {'_id': 2, 's': large}]) - self.assertEqual(1, collection.count()) - - with self.assertRaises(DuplicateKeyError): - collection.insert( - [{'_id': 1, 's': large}, {'_id': 2, 's': large}], - continue_on_error=True) - self.assertEqual(2, collection.count()) - collection.delete_one({'_id': 2}) - - # A writeError followed by a writeConcernError should prefer to raise - # the writeError. - with self.assertRaises(DuplicateKeyError): - collection.insert( - [{'_id': 1, 's': large}, {'_id': 2, 's': large}], - continue_on_error=True, - w=len(client_context.nodes) + 10, wtimeout=1) - self.assertEqual(2, collection.count()) - collection.delete_many({}) - - with self.assertRaises(WriteConcernError): - collection.insert( - [{'_id': 1, 's': large}, {'_id': 2, 's': large}], - continue_on_error=True, - w=len(client_context.nodes) + 10, wtimeout=1) - self.assertEqual(2, collection.count()) - - def test_insert_iterables(self): - # Tests legacy insert. - db = self.db - - self.assertRaises(TypeError, db.test.insert, 4) - self.assertRaises(TypeError, db.test.insert, None) - self.assertRaises(TypeError, db.test.insert, True) - - db.drop_collection("test") - self.assertEqual(db.test.find().count(), 0) - db.test.insert(({"hello": "world"}, {"hello": "world"})) - self.assertEqual(db.test.find().count(), 2) - - db.drop_collection("test") - self.assertEqual(db.test.find().count(), 0) - db.test.insert(map(lambda x: {"hello": "world"}, - itertools.repeat(None, 10))) - self.assertEqual(db.test.find().count(), 10) - - def test_insert_manipulate_false(self): - # Test two aspects of legacy insert with manipulate=False: - # 1. The return value is None or [None] as appropriate. - # 2. _id is not set on the passed-in document object. - collection = self.db.test_insert_manipulate_false - collection.drop() - oid = ObjectId() - doc = {'a': oid} - - try: - # The return value is None. - self.assertTrue(collection.insert(doc, manipulate=False) is None) - # insert() shouldn't set _id on the passed-in document object. - self.assertEqual({'a': oid}, doc) - - # Bulk insert. The return value is a list of None. - self.assertEqual([None], collection.insert([{}], manipulate=False)) - - docs = [{}, {}] - ids = collection.insert(docs, manipulate=False) - self.assertEqual([None, None], ids) - self.assertEqual([{}, {}], docs) - finally: - collection.drop() - - def test_continue_on_error(self): - # Tests legacy insert. - db = self.db - db.drop_collection("test_continue_on_error") - collection = db.test_continue_on_error - oid = collection.insert({"one": 1}) - self.assertEqual(1, collection.count()) - - docs = [] - docs.append({"_id": oid, "two": 2}) # Duplicate _id. - docs.append({"three": 3}) - docs.append({"four": 4}) - docs.append({"five": 5}) - - with self.assertRaises(DuplicateKeyError): - collection.insert(docs, manipulate=False) - - self.assertEqual(1, collection.count()) - - with self.assertRaises(DuplicateKeyError): - collection.insert(docs, manipulate=False, continue_on_error=True) - - self.assertEqual(4, collection.count()) - - collection.remove({}, w=client_context.w) - - oid = collection.insert({"_id": oid, "one": 1}, w=0) - wait_until(lambda: 1 == collection.count(), 'insert 1 document') - - docs[0].pop("_id") - docs[2]["_id"] = oid - - with self.assertRaises(DuplicateKeyError): - collection.insert(docs, manipulate=False) - - self.assertEqual(3, collection.count()) - collection.insert(docs, manipulate=False, continue_on_error=True, w=0) - wait_until(lambda: 6 == collection.count(), 'insert 3 documents') - - def test_acknowledged_insert(self): - # Tests legacy insert. - db = self.db - db.drop_collection("test_acknowledged_insert") - collection = db.test_acknowledged_insert - - a = {"hello": "world"} - collection.insert(a) - collection.insert(a, w=0) - self.assertRaises(OperationFailure, - collection.insert, a) - - def test_insert_adds_id(self): - # Tests legacy insert. - doc = {"hello": "world"} - self.db.test.insert(doc) - self.assertTrue("_id" in doc) - - docs = [{"hello": "world"}, {"hello": "world"}] - self.db.test.insert(docs) - for doc in docs: - self.assertTrue("_id" in doc) - - def test_insert_large_batch(self): - # Tests legacy insert. - db = self.client.test_insert_large_batch - self.addCleanup(self.client.drop_database, 'test_insert_large_batch') - max_bson_size = self.client.max_bson_size - # Write commands are limited to 16MB + 16k per batch - big_string = 'x' * int(max_bson_size / 2) - - # Batch insert that requires 2 batches. - successful_insert = [{'x': big_string}, {'x': big_string}, - {'x': big_string}, {'x': big_string}] - db.collection_0.insert(successful_insert, w=1) - self.assertEqual(4, db.collection_0.count()) - - db.collection_0.drop() - - # Test that inserts fail after first error. - insert_second_fails = [{'_id': 'id0', 'x': big_string}, - {'_id': 'id0', 'x': big_string}, - {'_id': 'id1', 'x': big_string}, - {'_id': 'id2', 'x': big_string}] - - with self.assertRaises(DuplicateKeyError): - db.collection_1.insert(insert_second_fails) - - self.assertEqual(1, db.collection_1.count()) - - db.collection_1.drop() - - # 2 batches, 2nd insert fails, don't continue on error. - self.assertTrue(db.collection_2.insert(insert_second_fails, w=0)) - wait_until(lambda: 1 == db.collection_2.count(), - 'insert 1 document', timeout=60) - - db.collection_2.drop() - - # 2 batches, ids of docs 0 and 1 are dupes, ids of docs 2 and 3 are - # dupes. Acknowledged, continue on error. - insert_two_failures = [{'_id': 'id0', 'x': big_string}, - {'_id': 'id0', 'x': big_string}, - {'_id': 'id1', 'x': big_string}, - {'_id': 'id1', 'x': big_string}] - - with self.assertRaises(OperationFailure) as context: - db.collection_3.insert(insert_two_failures, - continue_on_error=True, w=1) - - self.assertIn('id1', str(context.exception)) - - # Only the first and third documents should be inserted. - self.assertEqual(2, db.collection_3.count()) - - db.collection_3.drop() - - # 2 batches, 2 errors, unacknowledged, continue on error. - db.collection_4.insert(insert_two_failures, continue_on_error=True, w=0) - - # Only the first and third documents are inserted. - wait_until(lambda: 2 == db.collection_4.count(), - 'insert 2 documents', timeout=60) - - db.collection_4.drop() - - def test_bad_dbref(self): - # Requires the legacy API to test. - c = self.db.test - c.drop() - - # Incomplete DBRefs. - self.assertRaises( - InvalidDocument, - c.insert_one, {'ref': {'$ref': 'collection'}}) - - self.assertRaises( - InvalidDocument, - c.insert_one, {'ref': {'$id': ObjectId()}}) - - ref_only = {'ref': {'$ref': 'collection'}} - id_only = {'ref': {'$id': ObjectId()}} - - - def test_update(self): - # Tests legacy update. - db = self.db - db.drop_collection("test") - - id1 = db.test.save({"x": 5}) - db.test.update({}, {"$inc": {"x": 1}}) - self.assertEqual(db.test.find_one(id1)["x"], 6) - - id2 = db.test.save({"x": 1}) - db.test.update({"x": 6}, {"$inc": {"x": 1}}) - self.assertEqual(db.test.find_one(id1)["x"], 7) - self.assertEqual(db.test.find_one(id2)["x"], 1) - - def test_update_manipulate(self): - # Tests legacy update. - db = self.db - db.drop_collection("test") - db.test.insert({'_id': 1}) - db.test.update({'_id': 1}, {'a': 1}, manipulate=True) - self.assertEqual( - {'_id': 1, 'a': 1}, - db.test.find_one()) - - class AddField(SONManipulator): - def transform_incoming(self, son, dummy): - son['field'] = 'value' - return son - - db.add_son_manipulator(AddField()) - db.test.update({'_id': 1}, {'a': 2}, manipulate=False) - self.assertEqual( - {'_id': 1, 'a': 2}, - db.test.find_one()) - - db.test.update({'_id': 1}, {'a': 3}, manipulate=True) - self.assertEqual( - {'_id': 1, 'a': 3, 'field': 'value'}, - db.test.find_one()) - - def test_update_nmodified(self): - # Tests legacy update. - db = self.db - db.drop_collection("test") - ismaster = self.client.admin.command('ismaster') - used_write_commands = (ismaster.get("maxWireVersion", 0) > 1) - - db.test.insert({'_id': 1}) - result = db.test.update({'_id': 1}, {'$set': {'x': 1}}) - if used_write_commands: - self.assertEqual(1, result['nModified']) - else: - self.assertFalse('nModified' in result) - - # x is already 1. - result = db.test.update({'_id': 1}, {'$set': {'x': 1}}) - if used_write_commands: - self.assertEqual(0, result['nModified']) - else: - self.assertFalse('nModified' in result) - - def test_multi_update(self): - # Tests legacy update. - db = self.db - db.drop_collection("test") - - db.test.save({"x": 4, "y": 3}) - db.test.save({"x": 5, "y": 5}) - db.test.save({"x": 4, "y": 4}) - - db.test.update({"x": 4}, {"$set": {"y": 5}}, multi=True) - - self.assertEqual(3, db.test.count()) - for doc in db.test.find(): - self.assertEqual(5, doc["y"]) - - self.assertEqual(2, db.test.update({"x": 4}, {"$set": {"y": 6}}, - multi=True)["n"]) - - def test_upsert(self): - # Tests legacy update. - db = self.db - db.drop_collection("test") - - db.test.update({"page": "/"}, {"$inc": {"count": 1}}, upsert=True) - db.test.update({"page": "/"}, {"$inc": {"count": 1}}, upsert=True) - - self.assertEqual(1, db.test.count()) - self.assertEqual(2, db.test.find_one()["count"]) - - def test_acknowledged_update(self): - # Tests legacy update. - db = self.db - db.drop_collection("test_acknowledged_update") - collection = db.test_acknowledged_update - collection.create_index("x", unique=True) - - collection.insert({"x": 5}) - _id = collection.insert({"x": 4}) - - self.assertEqual( - None, collection.update({"_id": _id}, {"$inc": {"x": 1}}, w=0)) - - self.assertRaises(DuplicateKeyError, collection.update, - {"_id": _id}, {"$inc": {"x": 1}}) - - self.assertEqual(1, collection.update({"_id": _id}, - {"$inc": {"x": 2}})["n"]) - - self.assertEqual(0, collection.update({"_id": "foo"}, - {"$inc": {"x": 2}})["n"]) - db.drop_collection("test_acknowledged_update") - - def test_update_backward_compat(self): - # MongoDB versions >= 2.6.0 don't return the updatedExisting field - # and return upsert _id in an array subdocument. This test should - # pass regardless of server version or type (mongod/s). - # Tests legacy update. - c = self.db.test - c.drop() - oid = ObjectId() - res = c.update({'_id': oid}, {'$set': {'a': 'a'}}, upsert=True) - self.assertFalse(res.get('updatedExisting')) - self.assertEqual(oid, res.get('upserted')) - - res = c.update({'_id': oid}, {'$set': {'b': 'b'}}) - self.assertTrue(res.get('updatedExisting')) - - def test_save(self): - # Tests legacy save. - self.db.drop_collection("test_save") - collection = self.db.test_save - - # Save a doc with autogenerated id - _id = collection.save({"hello": "world"}) - self.assertEqual(collection.find_one()["_id"], _id) - self.assertTrue(isinstance(_id, ObjectId)) - - # Save a doc with explicit id - collection.save({"_id": "explicit_id", "hello": "bar"}) - doc = collection.find_one({"_id": "explicit_id"}) - self.assertEqual(doc['_id'], 'explicit_id') - self.assertEqual(doc['hello'], 'bar') - - # Save docs with _id field already present (shouldn't create new docs) - self.assertEqual(2, collection.count()) - collection.save({'_id': _id, 'hello': 'world'}) - self.assertEqual(2, collection.count()) - collection.save({'_id': 'explicit_id', 'hello': 'baz'}) - self.assertEqual(2, collection.count()) - self.assertEqual( - 'baz', - collection.find_one({'_id': 'explicit_id'})['hello'] - ) - - # Acknowledged mode. - collection.create_index("hello", unique=True) - # No exception, even though we duplicate the first doc's "hello" value - collection.save({'_id': 'explicit_id', 'hello': 'world'}, w=0) - - self.assertRaises( - DuplicateKeyError, - collection.save, - {'_id': 'explicit_id', 'hello': 'world'}) - self.db.drop_collection("test") - - def test_save_with_invalid_key(self): - if client_context.version.at_least(3, 5, 8): - raise SkipTest("MongoDB >= 3.5.8 allows dotted fields in updates") - # Tests legacy save. - self.db.drop_collection("test") - self.assertTrue(self.db.test.insert({"hello": "world"})) - doc = self.db.test.find_one() - doc['a.b'] = 'c' - self.assertRaises(OperationFailure, self.db.test.save, doc) - - def test_acknowledged_save(self): - # Tests legacy save. - db = self.db - db.drop_collection("test_acknowledged_save") - collection = db.test_acknowledged_save - collection.create_index("hello", unique=True) - - collection.save({"hello": "world"}) - collection.save({"hello": "world"}, w=0) - self.assertRaises(DuplicateKeyError, collection.save, - {"hello": "world"}) - db.drop_collection("test_acknowledged_save") - - def test_save_adds_id(self): - # Tests legacy save. - doc = {"hello": "jesse"} - self.db.test.save(doc) - self.assertTrue("_id" in doc) - - def test_save_returns_id(self): - doc = {"hello": "jesse"} - _id = self.db.test.save(doc) - self.assertTrue(isinstance(_id, ObjectId)) - self.assertEqual(_id, doc["_id"]) - doc["hi"] = "bernie" - _id = self.db.test.save(doc) - self.assertTrue(isinstance(_id, ObjectId)) - self.assertEqual(_id, doc["_id"]) - - def test_remove_one(self): - # Tests legacy remove. - self.db.test.remove() - self.assertEqual(0, self.db.test.count()) - - self.db.test.insert({"x": 1}) - self.db.test.insert({"y": 1}) - self.db.test.insert({"z": 1}) - self.assertEqual(3, self.db.test.count()) - - self.db.test.remove(multi=False) - self.assertEqual(2, self.db.test.count()) - self.db.test.remove() - self.assertEqual(0, self.db.test.count()) - - def test_remove_all(self): - # Tests legacy remove. - self.db.test.remove() - self.assertEqual(0, self.db.test.count()) - - self.db.test.insert({"x": 1}) - self.db.test.insert({"y": 1}) - self.assertEqual(2, self.db.test.count()) - - self.db.test.remove() - self.assertEqual(0, self.db.test.count()) - - def test_remove_non_objectid(self): - # Tests legacy remove. - db = self.db - db.drop_collection("test") - - db.test.insert_one({"_id": 5}) - - self.assertEqual(1, db.test.count()) - db.test.remove(5) - self.assertEqual(0, db.test.count()) - - def test_write_large_document(self): - # Tests legacy insert, save, and update. - max_size = self.db.client.max_bson_size - half_size = int(max_size / 2) - self.assertEqual(max_size, 16777216) - - self.assertRaises(OperationFailure, self.db.test.insert, - {"foo": "x" * max_size}) - self.assertRaises(OperationFailure, self.db.test.save, - {"foo": "x" * max_size}) - self.assertRaises(OperationFailure, self.db.test.insert, - [{"x": 1}, {"foo": "x" * max_size}]) - self.db.test.insert([{"foo": "x" * half_size}, - {"foo": "x" * half_size}]) - - self.db.test.insert({"bar": "x"}) - # Use w=0 here to test legacy doc size checking in all server versions - self.assertRaises(DocumentTooLarge, self.db.test.update, - {"bar": "x"}, {"bar": "x" * (max_size - 14)}, w=0) - # This will pass with OP_UPDATE or the update command. - self.db.test.update({"bar": "x"}, {"bar": "x" * (max_size - 32)}) - - def test_last_error_options(self): - # Tests legacy write methods. - self.db.test.save({"x": 1}, w=1, wtimeout=1) - self.db.test.insert({"x": 1}, w=1, wtimeout=1) - self.db.test.remove({"x": 1}, w=1, wtimeout=1) - self.db.test.update({"x": 1}, {"y": 2}, w=1, wtimeout=1) - - if client_context.replica_set_name: - # client_context.w is the number of hosts in the replica set - w = client_context.w + 1 - - # MongoDB 2.8+ raises error code 100, CannotSatisfyWriteConcern, - # if w > number of members. Older versions just time out after 1 ms - # as if they had enough secondaries but some are lagging. They - # return an error with 'wtimeout': True and no code. - def wtimeout_err(f, *args, **kwargs): - try: - f(*args, **kwargs) - except WTimeoutError as exc: - self.assertIsNotNone(exc.details) - except OperationFailure as exc: - self.assertIsNotNone(exc.details) - self.assertEqual(100, exc.code, - "Unexpected error: %r" % exc) - else: - self.fail("%s should have failed" % f) - - coll = self.db.test - wtimeout_err(coll.save, {"x": 1}, w=w, wtimeout=1) - wtimeout_err(coll.insert, {"x": 1}, w=w, wtimeout=1) - wtimeout_err(coll.update, {"x": 1}, {"y": 2}, w=w, wtimeout=1) - wtimeout_err(coll.remove, {"x": 1}, w=w, wtimeout=1) - - # can't use fsync and j options together - self.assertRaises(ConfigurationError, self.db.test.insert, - {"_id": 1}, j=True, fsync=True) - - def test_find_and_modify(self): - c = self.db.test - c.drop() - c.insert({'_id': 1, 'i': 1}) - - # Test that we raise DuplicateKeyError when appropriate. - c.create_index('i', unique=True) - self.assertRaises(DuplicateKeyError, - c.find_and_modify, query={'i': 1, 'j': 1}, - update={'$set': {'k': 1}}, upsert=True) - c.drop_indexes() - - # Test correct findAndModify - self.assertEqual({'_id': 1, 'i': 1}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) - self.assertEqual({'_id': 1, 'i': 3}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True)) - - self.assertEqual({'_id': 1, 'i': 3}, - c.find_and_modify({'_id': 1}, remove=True)) - - self.assertEqual(None, c.find_one({'_id': 1})) - - self.assertEqual(None, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) - self.assertEqual(None, c.find_and_modify({'_id': 1}, - {'$inc': {'i': 1}}, - upsert=True)) - self.assertEqual({'_id': 1, 'i': 2}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - upsert=True, new=True)) - - self.assertEqual({'_id': 1, 'i': 2}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - fields=['i'])) - self.assertEqual({'_id': 1, 'i': 4}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True, fields={'i': 1})) - - # Test with full_response=True. - result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True, upsert=True, - full_response=True, - fields={'i': 1}) - self.assertEqual({'_id': 1, 'i': 5}, result["value"]) - self.assertEqual(True, - result["lastErrorObject"]["updatedExisting"]) - - result = c.find_and_modify({'_id': 2}, {'$inc': {'i': 1}}, - new=True, upsert=True, - full_response=True, - fields={'i': 1}) - self.assertEqual({'_id': 2, 'i': 1}, result["value"]) - self.assertEqual(False, - result["lastErrorObject"]["updatedExisting"]) - - class ExtendedDict(dict): - pass - - result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True, fields={'i': 1}) - self.assertFalse(isinstance(result, ExtendedDict)) - c = self.db.get_collection( - "test", codec_options=CodecOptions(document_class=ExtendedDict)) - result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True, fields={'i': 1}) - self.assertTrue(isinstance(result, ExtendedDict)) - - def test_find_and_modify_with_sort(self): - c = self.db.test - c.drop() - for j in range(5): - c.insert({'j': j, 'i': 0}) - - sort = {'j': DESCENDING} - self.assertEqual(4, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - sort = {'j': ASCENDING} - self.assertEqual(0, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - sort = [('j', DESCENDING)] - self.assertEqual(4, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - sort = [('j', ASCENDING)] - self.assertEqual(0, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - sort = SON([('j', DESCENDING)]) - self.assertEqual(4, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - sort = SON([('j', ASCENDING)]) - self.assertEqual(0, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - - try: - from collections import OrderedDict - sort = OrderedDict([('j', DESCENDING)]) - self.assertEqual(4, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - sort = OrderedDict([('j', ASCENDING)]) - self.assertEqual(0, c.find_and_modify({}, - {'$inc': {'i': 1}}, - sort=sort)['j']) - except ImportError: - pass - # Test that a standard dict with two keys is rejected. - sort = {'j': DESCENDING, 'foo': DESCENDING} - self.assertRaises(TypeError, c.find_and_modify, - {}, {'$inc': {'i': 1}}, sort=sort) - - def test_find_and_modify_with_manipulator(self): - class AddCollectionNameManipulator(SONManipulator): - def will_copy(self): - return True - - def transform_incoming(self, son, dummy): - copy = SON(son) - if 'collection' in copy: - del copy['collection'] - return copy - - def transform_outgoing(self, son, collection): - copy = SON(son) - copy['collection'] = collection.name - return copy - - db = self.client.pymongo_test - db.add_son_manipulator(AddCollectionNameManipulator()) - - c = db.test - c.drop() - c.insert({'_id': 1, 'i': 1}) - - # Test correct findAndModify - # With manipulators - self.assertEqual({'_id': 1, 'i': 1, 'collection': 'test'}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - manipulate=True)) - self.assertEqual({'_id': 1, 'i': 3, 'collection': 'test'}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True, manipulate=True)) - # With out manipulators - self.assertEqual({'_id': 1, 'i': 3}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) - self.assertEqual({'_id': 1, 'i': 5}, - c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, - new=True)) - @client_context.require_version_max(4, 1, 0, -1) def test_group(self): db = self.db @@ -1022,167 +209,6 @@ class TestLegacy(IntegrationTest): coll.group([], {"_id": uu}, {"count": 0}, reduce)) - def test_auto_ref_and_deref(self): - # Legacy API. - db = self.client.pymongo_test - db.add_son_manipulator(AutoReference(db)) - db.add_son_manipulator(NamespaceInjector()) - - db.test.a.remove({}) - db.test.b.remove({}) - db.test.c.remove({}) - - a = {"hello": "world"} - db.test.a.save(a) - - b = {"test": a} - db.test.b.save(b) - - c = {"another test": b} - db.test.c.save(c) - - a["hello"] = "mike" - db.test.a.save(a) - - self.assertEqual(db.test.a.find_one(), a) - self.assertEqual(db.test.b.find_one()["test"], a) - self.assertEqual(db.test.c.find_one()["another test"]["test"], a) - self.assertEqual(db.test.b.find_one(), b) - self.assertEqual(db.test.c.find_one()["another test"], b) - self.assertEqual(db.test.c.find_one(), c) - - def test_auto_ref_and_deref_list(self): - # Legacy API. - db = self.client.pymongo_test - db.add_son_manipulator(AutoReference(db)) - db.add_son_manipulator(NamespaceInjector()) - - db.drop_collection("users") - db.drop_collection("messages") - - message_1 = {"title": "foo"} - db.messages.save(message_1) - message_2 = {"title": "bar"} - db.messages.save(message_2) - - user = {"messages": [message_1, message_2]} - db.users.save(user) - db.messages.update(message_1, {"title": "buzz"}) - - self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"]) - self.assertEqual("bar", db.users.find_one()["messages"][1]["title"]) - - def test_object_to_dict_transformer(self): - # PYTHON-709: Some users rely on their custom SONManipulators to run - # before any other checks, so they can insert non-dict objects and - # have them dictified before the _id is inserted or any other - # processing. - # Tests legacy API elements. - class Thing(object): - def __init__(self, value): - self.value = value - - class ThingTransformer(SONManipulator): - def transform_incoming(self, thing, dummy): - return {'value': thing.value} - - db = self.client.foo - db.add_son_manipulator(ThingTransformer()) - t = Thing('value') - - db.test.remove() - db.test.insert([t]) - out = db.test.find_one() - self.assertEqual('value', out.get('value')) - - def test_son_manipulator_outgoing(self): - class Thing(object): - def __init__(self, value): - self.value = value - - class ThingTransformer(SONManipulator): - def transform_outgoing(self, doc, collection): - # We don't want this applied to the command return - # value in pymongo.cursor.Cursor. - if 'value' in doc: - return Thing(doc['value']) - return doc - - db = self.client.foo - db.add_son_manipulator(ThingTransformer()) - - db.test.delete_many({}) - db.test.insert_one({'value': 'value'}) - out = db.test.find_one() - self.assertTrue(isinstance(out, Thing)) - self.assertEqual('value', out.value) - - out = next(db.test.aggregate([], cursor={})) - self.assertTrue(isinstance(out, Thing)) - self.assertEqual('value', out.value) - - def test_son_manipulator_inheritance(self): - # Tests legacy API elements. - class Thing(object): - def __init__(self, value): - self.value = value - - class ThingTransformer(SONManipulator): - def transform_incoming(self, thing, dummy): - return {'value': thing.value} - - def transform_outgoing(self, son, dummy): - return Thing(son['value']) - - class Child(ThingTransformer): - pass - - db = self.client.foo - db.add_son_manipulator(Child()) - t = Thing('value') - - db.test.remove() - db.test.insert([t]) - out = db.test.find_one() - self.assertTrue(isinstance(out, Thing)) - self.assertEqual('value', out.value) - - def test_disabling_manipulators(self): - - class IncByTwo(SONManipulator): - def transform_outgoing(self, son, collection): - if 'foo' in son: - son['foo'] += 2 - return son - - db = self.client.pymongo_test - db.add_son_manipulator(IncByTwo()) - c = db.test - c.drop() - c.insert({'foo': 0}) - self.assertEqual(2, c.find_one()['foo']) - self.assertEqual(0, c.find_one(manipulate=False)['foo']) - - self.assertEqual(2, c.find_one(manipulate=True)['foo']) - c.drop() - - def test_manipulator_properties(self): - db = self.client.foo - self.assertEqual([], db.incoming_manipulators) - self.assertEqual([], db.incoming_copying_manipulators) - self.assertEqual([], db.outgoing_manipulators) - self.assertEqual([], db.outgoing_copying_manipulators) - db.add_son_manipulator(AutoReference(db)) - db.add_son_manipulator(NamespaceInjector()) - db.add_son_manipulator(ObjectIdShuffler()) - self.assertEqual(1, len(db.incoming_manipulators)) - self.assertEqual(db.incoming_manipulators, ['NamespaceInjector']) - self.assertEqual(2, len(db.incoming_copying_manipulators)) - for name in db.incoming_copying_manipulators: - self.assertTrue(name in ('ObjectIdShuffler', 'AutoReference')) - self.assertEqual([], db.outgoing_manipulators) - self.assertEqual(['AutoReference'], - db.outgoing_copying_manipulators) class TestLegacyBulk(BulkTestBase): @@ -1207,41 +233,6 @@ class TestLegacyBulk(BulkTestBase): # No error. bulk.find({}) - @client_context.require_version_min(3, 1, 9, -1) - def test_bypass_document_validation_bulk_op(self): - - # Test insert - self.coll.insert_one({"z": 0}) - self.db.command(SON([("collMod", "test"), - ("validator", {"z": {"$gte": 0}})])) - bulk = self.coll.initialize_ordered_bulk_op( - bypass_document_validation=False) - bulk.insert({"z": -1}) # error - self.assertRaises(BulkWriteError, bulk.execute) - self.assertEqual(0, self.coll.count({"z": -1})) - - bulk = self.coll.initialize_ordered_bulk_op( - bypass_document_validation=True) - bulk.insert({"z": -1}) - bulk.execute() - self.assertEqual(1, self.coll.count({"z": -1})) - - self.coll.insert_one({"z": 0}) - self.db.command(SON([("collMod", "test"), - ("validator", {"z": {"$gte": 0}})])) - bulk = self.coll.initialize_unordered_bulk_op( - bypass_document_validation=False) - bulk.insert({"z": -1}) # error - self.assertRaises(BulkWriteError, bulk.execute) - self.assertEqual(1, self.coll.count({"z": -1})) - - bulk = self.coll.initialize_unordered_bulk_op( - bypass_document_validation=True) - bulk.insert({"z": -1}) - bulk.execute() - self.assertEqual(2, self.coll.count({"z": -1})) - self.coll.drop() - def test_insert(self): expected = { 'nMatched': 0, diff --git a/test/test_monitoring.py b/test/test_monitoring.py index f46ebe69b..46cfe87c4 100644 --- a/test/test_monitoring.py +++ b/test/test_monitoring.py @@ -37,7 +37,8 @@ from test import (client_context, from test.utils import (EventListener, get_pool, rs_or_single_client, - single_client) + single_client, + wait_until) class TestCommandMonitoring(PyMongoTestCase): @@ -820,230 +821,6 @@ class TestCommandMonitoring(PyMongoTestCase): self.assertIsInstance(error.get('code'), int) self.assertIsInstance(error.get('errmsg'), str) - def test_legacy_writes(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - - coll = self.client.pymongo_test.test - coll.drop() - self.listener.results.clear() - - # Implied write concern insert - _id = coll.insert({'x': 1}) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('insert', coll.name), - ('ordered', True), - ('documents', [{'_id': _id, 'x': 1}])]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('insert', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(1, reply.get('n')) - - # Unacknowledged insert - self.listener.results.clear() - _id = coll.insert({'x': 1}, w=0) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('insert', coll.name), - ('ordered', True), - ('documents', [{'_id': _id, 'x': 1}]), - ('writeConcern', {'w': 0})]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('insert', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - self.assertEqual(succeeded.reply, {'ok': 1}) - - # Explicit write concern insert - self.listener.results.clear() - _id = coll.insert({'x': 1}, w=1) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('insert', coll.name), - ('ordered', True), - ('documents', [{'_id': _id, 'x': 1}]), - ('writeConcern', {'w': 1})]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('insert', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(1, reply.get('n')) - - # remove all - self.listener.results.clear() - res = coll.remove({'x': 1}, w=1) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('delete', coll.name), - ('ordered', True), - ('deletes', [SON([('q', {'x': 1}), - ('limit', 0)])]), - ('writeConcern', {'w': 1})]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('delete', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(res['n'], reply.get('n')) - - # upsert - self.listener.results.clear() - oid = ObjectId() - coll.update({'_id': oid}, {'_id': oid, 'x': 1}, upsert=True, w=1) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('update', coll.name), - ('ordered', True), - ('updates', [SON([('q', {'_id': oid}), - ('u', {'_id': oid, 'x': 1}), - ('multi', False), - ('upsert', True)])]), - ('writeConcern', {'w': 1})]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('update', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(1, reply.get('n')) - self.assertEqual([{'index': 0, '_id': oid}], reply.get('upserted')) - - # update one - self.listener.results.clear() - coll.update({'x': 1}, {'$inc': {'x': 1}}) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('update', coll.name), - ('ordered', True), - ('updates', [SON([('q', {'x': 1}), - ('u', {'$inc': {'x': 1}}), - ('multi', False), - ('upsert', False)])])]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('update', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(1, reply.get('n')) - - # update many - self.listener.results.clear() - coll.update({'x': 2}, {'$inc': {'x': 1}}, multi=True) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('update', coll.name), - ('ordered', True), - ('updates', [SON([('q', {'x': 2}), - ('u', {'$inc': {'x': 1}}), - ('multi', True), - ('upsert', False)])])]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('update', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(1, reply.get('n')) - - # remove one - self.listener.results.clear() - coll.remove({'x': 3}, multi=False) - results = self.listener.results - started = results['started'][0] - succeeded = results['succeeded'][0] - self.assertEqual(0, len(results['failed'])) - self.assertIsInstance(started, monitoring.CommandStartedEvent) - expected = SON([('delete', coll.name), - ('ordered', True), - ('deletes', [SON([('q', {'x': 3}), - ('limit', 1)])])]) - self.assertEqualCommand(expected, started.command) - self.assertEqual('pymongo_test', started.database_name) - self.assertEqual('delete', started.command_name) - self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) - self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeeded.duration_micros, int) - self.assertEqual(started.command_name, succeeded.command_name) - self.assertEqual(started.request_id, succeeded.request_id) - self.assertEqual(started.connection_id, succeeded.connection_id) - reply = succeeded.reply - self.assertEqual(1, reply.get('ok')) - self.assertEqual(1, reply.get('n')) - - self.assertEqual(0, coll.count_documents({})) - def test_insert_many(self): # This always uses the bulk API. coll = self.client.pymongo_test.test @@ -1086,51 +863,48 @@ class TestCommandMonitoring(PyMongoTestCase): self.assertEqual(documents, docs) self.assertEqual(6, count) - def test_legacy_insert_many(self): + def test_insert_many_unacknowledged(self): # On legacy servers this uses bulk OP_INSERT. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + coll = self.client.pymongo_test.test + coll.drop() + unack_coll = coll.with_options(write_concern=WriteConcern(w=0)) + self.listener.results.clear() - coll = self.client.pymongo_test.test - coll.drop() - self.listener.results.clear() - - # Force two batches on legacy servers. - big = 'x' * (1024 * 1024 * 12) - docs = [{'_id': i, 'big': big} for i in range(6)] - coll.insert(docs) - results = self.listener.results - started = results['started'] - succeeded = results['succeeded'] - self.assertEqual(0, len(results['failed'])) - documents = [] - count = 0 - operation_id = started[0].operation_id - self.assertIsInstance(operation_id, int) - for start, succeed in zip(started, succeeded): - self.assertIsInstance(start, monitoring.CommandStartedEvent) - cmd = sanitize_cmd(start.command) - self.assertEqual(['insert', 'ordered', 'documents'], - list(cmd.keys())) - self.assertEqual(coll.name, cmd['insert']) - self.assertIs(True, cmd['ordered']) - documents.extend(cmd['documents']) - self.assertEqual('pymongo_test', start.database_name) - self.assertEqual('insert', start.command_name) - self.assertIsInstance(start.request_id, int) - self.assertEqual(self.client.address, start.connection_id) - self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) - self.assertIsInstance(succeed.duration_micros, int) - self.assertEqual(start.command_name, succeed.command_name) - self.assertEqual(start.request_id, succeed.request_id) - self.assertEqual(start.connection_id, succeed.connection_id) - self.assertEqual(start.operation_id, operation_id) - self.assertEqual(succeed.operation_id, operation_id) - reply = succeed.reply - self.assertEqual(1, reply.get('ok')) - count += reply.get('n', 0) - self.assertEqual(documents, docs) - self.assertEqual(6, count) + # Force two batches on legacy servers. + big = 'x' * (1024 * 1024 * 12) + docs = [{'_id': i, 'big': big} for i in range(6)] + unack_coll.insert_many(docs) + results = self.listener.results + started = results['started'] + succeeded = results['succeeded'] + self.assertEqual(0, len(results['failed'])) + documents = [] + operation_id = started[0].operation_id + self.assertIsInstance(operation_id, int) + for start, succeed in zip(started, succeeded): + self.assertIsInstance(start, monitoring.CommandStartedEvent) + cmd = sanitize_cmd(start.command) + cmd.pop('writeConcern', None) + self.assertEqual(['insert', 'ordered', 'documents'], + list(cmd.keys())) + self.assertEqual(coll.name, cmd['insert']) + self.assertIs(True, cmd['ordered']) + documents.extend(cmd['documents']) + self.assertEqual('pymongo_test', start.database_name) + self.assertEqual('insert', start.command_name) + self.assertIsInstance(start.request_id, int) + self.assertEqual(self.client.address, start.connection_id) + self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) + self.assertIsInstance(succeed.duration_micros, int) + self.assertEqual(start.command_name, succeed.command_name) + self.assertEqual(start.request_id, succeed.request_id) + self.assertEqual(start.connection_id, succeed.connection_id) + self.assertEqual(start.operation_id, operation_id) + self.assertEqual(succeed.operation_id, operation_id) + self.assertEqual(1, succeed.reply.get('ok')) + self.assertEqual(documents, docs) + wait_until(lambda: coll.count_documents({}) == 6, + 'insert documents with w=0') def test_bulk_write(self): coll = self.client.pymongo_test.test diff --git a/test/test_retryable_writes.py b/test/test_retryable_writes.py index 91d3d8257..484ef740d 100644 --- a/test/test_retryable_writes.py +++ b/test/test_retryable_writes.py @@ -88,7 +88,7 @@ test_creator = TestCreator(create_test, TestAllScenarios, _TEST_PATH) test_creator.create_tests() -def _retryable_single_statement_ops(coll): +def retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}), (coll.bulk_write, [[InsertOne({}), @@ -110,29 +110,6 @@ def _retryable_single_statement_ops(coll): ] -def retryable_single_statement_ops(coll): - return _retryable_single_statement_ops(coll) + [ - # Deprecated methods. - # Insert with single or multiple documents. - (coll.insert, [{}], {}), - (coll.insert, [[{}]], {}), - (coll.insert, [[{}, {}]], {}), - # Save with and without an _id. - (coll.save, [{}], {}), - (coll.save, [{'_id': ObjectId()}], {}), - # Non-multi update. - (coll.update, [{}, {'$set': {'a': 1}}], {}), - # Non-multi remove. - (coll.remove, [{}], {'multi': False}), - # Replace. - (coll.find_and_modify, [{}, {'a': 3}], {}), - # Update. - (coll.find_and_modify, [{}, {'$set': {'a': 1}}], {}), - # Delete. - (coll.find_and_modify, [{}, {}], {'remove': True}), - ] - - def non_retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), @@ -140,25 +117,6 @@ def non_retryable_single_statement_ops(coll): (coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_many, [{}], {}), - # Deprecated methods. - # Multi remove. - (coll.remove, [{}], {}), - # Multi update. - (coll.update, [{}, {'$set': {'a': 1}}], {'multi': True}), - # Unacknowledged deprecated methods. - (coll.insert, [{}], {'w': 0}), - # Unacknowledged Non-multi update. - (coll.update, [{}, {'$set': {'a': 1}}], {'w': 0}), - # Unacknowledged Non-multi remove. - (coll.remove, [{}], {'multi': False, 'w': 0}), - # Unacknowledged Replace. - (coll.find_and_modify, [{}, {'a': 3}], {'writeConcern': {'w': 0}}), - # Unacknowledged Update. - (coll.find_and_modify, [{}, {'$set': {'a': 1}}], - {'writeConcern': {'w': 0}}), - # Unacknowledged Delete. - (coll.find_and_modify, [{}, {}], - {'remove': True, 'writeConcern': {'w': 0}}), ] @@ -533,7 +491,7 @@ class TestRetryableWritesTxnNumber(IgnoreDeprecationsTest): topology.select_server = select_server raise ConnectionFailure('Connection refused') - for method, args, kwargs in _retryable_single_statement_ops( + for method, args, kwargs in retryable_single_statement_ops( client.db.retryable_write_test): listener.results.clear() topology.select_server = raise_connection_err_select_server diff --git a/test/test_write_concern.py b/test/test_write_concern.py index c6b803618..5e1c0f73f 100644 --- a/test/test_write_concern.py +++ b/test/test_write_concern.py @@ -17,11 +17,18 @@ import collections import unittest +from pymongo.errors import ConfigurationError from pymongo.write_concern import WriteConcern class TestWriteConcern(unittest.TestCase): + def test_invalid(self): + # Can't use fsync and j options together + self.assertRaises(ConfigurationError, WriteConcern, j=True, fsync=True) + # Can't use w=0 and j options together + self.assertRaises(ConfigurationError, WriteConcern, w=0, j=True) + def test_equality(self): concern = WriteConcern(j=True, wtimeout=3000) self.assertEqual(concern, WriteConcern(j=True, wtimeout=3000)) diff --git a/tools/benchmark.py b/tools/benchmark.py deleted file mode 100644 index a0fc74a12..000000000 --- a/tools/benchmark.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2009-2015 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""MongoDB benchmarking suite.""" -from __future__ import print_function - -import time -import sys -sys.path[0:0] = [""] - -import datetime - -from pymongo import mongo_client -from pymongo import ASCENDING - -trials = 2 -per_trial = 5000 -batch_size = 100 -small = {} -medium = {"integer": 5, - "number": 5.05, - "boolean": False, - "array": ["test", "benchmark"] - } -# this is similar to the benchmark data posted to the user list -large = {"base_url": "http://www.example.com/test-me", - "total_word_count": 6743, - "access_time": datetime.datetime.utcnow(), - "meta_tags": {"description": "i am a long description string", - "author": "Holly Man", - "dynamically_created_meta_tag": "who know\n what" - }, - "page_structure": {"counted_tags": 3450, - "no_of_js_attached": 10, - "no_of_images": 6 - }, - "harvested_words": ["10gen", "web", "open", "source", "application", - "paas", "platform-as-a-service", "technology", - "helps", "developers", "focus", "building", - "mongodb", "mongo"] * 20 - } - - -def setup_insert(db, collection, object): - db.drop_collection(collection) - - -def insert(db, collection, object): - for i in range(per_trial): - to_insert = object.copy() - to_insert["x"] = i - db[collection].insert(to_insert) - - -def insert_batch(db, collection, object): - for i in range(per_trial / batch_size): - db[collection].insert([object] * batch_size) - - -def find_one(db, collection, x): - for _ in range(per_trial): - db[collection].find_one({"x": x}) - - -def find(db, collection, x): - for _ in range(per_trial): - for _ in db[collection].find({"x": x}): - pass - - -def timed(name, function, args=[], setup=None): - times = [] - for _ in range(trials): - if setup: - setup(*args) - start = time.time() - function(*args) - times.append(time.time() - start) - best_time = min(times) - print("{0:s}{1:d}".format(name + (60 - len(name)) * ".", per_trial / best_time)) - return best_time - - -def main(): - c = mongo_client.MongoClient(connectTimeoutMS=60*1000) # jack up timeout - c.drop_database("benchmark") - db = c.benchmark - - timed("insert (small, no index)", insert, - [db, 'small_none', small], setup_insert) - timed("insert (medium, no index)", insert, - [db, 'medium_none', medium], setup_insert) - timed("insert (large, no index)", insert, - [db, 'large_none', large], setup_insert) - - db.small_index.create_index("x", ASCENDING) - timed("insert (small, indexed)", insert, [db, 'small_index', small]) - db.medium_index.create_index("x", ASCENDING) - timed("insert (medium, indexed)", insert, [db, 'medium_index', medium]) - db.large_index.create_index("x", ASCENDING) - timed("insert (large, indexed)", insert, [db, 'large_index', large]) - - timed("batch insert (small, no index)", insert_batch, - [db, 'small_bulk', small], setup_insert) - timed("batch insert (medium, no index)", insert_batch, - [db, 'medium_bulk', medium], setup_insert) - timed("batch insert (large, no index)", insert_batch, - [db, 'large_bulk', large], setup_insert) - - timed("find_one (small, no index)", find_one, - [db, 'small_none', per_trial / 2]) - timed("find_one (medium, no index)", find_one, - [db, 'medium_none', per_trial / 2]) - timed("find_one (large, no index)", find_one, - [db, 'large_none', per_trial / 2]) - - timed("find_one (small, indexed)", find_one, - [db, 'small_index', per_trial / 2]) - timed("find_one (medium, indexed)", find_one, - [db, 'medium_index', per_trial / 2]) - timed("find_one (large, indexed)", find_one, - [db, 'large_index', per_trial / 2]) - - timed("find (small, no index)", find, [db, 'small_none', per_trial / 2]) - timed("find (medium, no index)", find, [db, 'medium_none', per_trial / 2]) - timed("find (large, no index)", find, [db, 'large_none', per_trial / 2]) - - timed("find (small, indexed)", find, [db, 'small_index', per_trial / 2]) - timed("find (medium, indexed)", find, [db, 'medium_index', per_trial / 2]) - timed("find (large, indexed)", find, [db, 'large_index', per_trial / 2]) - -# timed("find range (small, no index)", find, -# [db, 'small_none', -# {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) -# timed("find range (medium, no index)", find, -# [db, 'medium_none', -# {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) -# timed("find range (large, no index)", find, -# [db, 'large_none', -# {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) - - timed("find range (small, indexed)", find, - [db, 'small_index', - {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) - timed("find range (medium, indexed)", find, - [db, 'medium_index', - {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) - timed("find range (large, indexed)", find, - [db, 'large_index', - {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) - -if __name__ == "__main__": -# cProfile.run("main()") - main()