diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 45ac6c414..af9a49659 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2126,7 +2126,8 @@ tasks: script: | ${PREPARE_SHELL} export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 - export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/${bucket_name}/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz + export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz + SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg - name: testazurekms-task @@ -3144,7 +3145,7 @@ buildvariants: - name: testgcpkms-variant display_name: "GCP KMS" run_on: - - debian10-small + - debian11-small tasks: - name: testgcpkms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 7271e8d46..ac6944972 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -137,6 +137,9 @@ do srv|SRV|initial-dns-seedlist-discovery|srv_seedlist) cpjson initial-dns-seedlist-discovery/tests/ srv_seedlist ;; + read-write-concern|read_write_concern) + cpjson read-write-concern/tests/operation read_write_concern/operation + ;; retryable-reads|retryable_reads) cpjson retryable-reads/tests/ retryable_reads ;; diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh index 65c5cd0bb..d99c178fb 100644 --- a/.evergreen/run-azurekms-fail-test.sh +++ b/.evergreen/run-azurekms-fail-test.sh @@ -1,10 +1,12 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail - +HERE=$(dirname ${BASH_SOURCE:-$0}) . $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh +export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz +SKIP_SERVERS=1 bash $HERE/setup-encryption.sh PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ KEY_NAME="${AZUREKMS_KEYNAME}" \ KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ - LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \ SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ - ./.evergreen/hatch.sh test:test-eg + $HERE/hatch.sh test:test-eg +bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh index 961bcf507..bb515a938 100644 --- a/.evergreen/run-azurekms-test.sh +++ b/.evergreen/run-azurekms-test.sh @@ -1,11 +1,13 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail - +HERE=$(dirname ${BASH_SOURCE:-$0}) source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh echo "Copying files ... begin" export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey +export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz +SKIP_SERVERS=1 bash $HERE/setup-encryption.sh tar czf /tmp/mongo-python-driver.tgz . # shellcheck disable=SC2088 AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \ @@ -16,6 +18,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \ +AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Running test ... end" +bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh index 8a5fef04c..7ccc74b45 100644 --- a/.evergreen/run-gcpkms-test.sh +++ b/.evergreen/run-gcpkms-test.sh @@ -1,5 +1,6 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail +HERE=$(dirname ${BASH_SOURCE:-$0}) source ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/secrets-export.sh echo "Copying files ... begin" @@ -7,6 +8,8 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} +export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz +SKIP_SERVERS=1 bash $HERE/setup-encryption.sh tar czf /tmp/mongo-python-driver.tgz . GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" @@ -14,5 +17,6 @@ echo "Untarring file ... begin" GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Running test ... end" +bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 4ada73141..beee1ed28 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -121,14 +121,14 @@ if [ -n "$TEST_PYOPENSSL" ]; then fi if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then + # Check for libmongocrypt checkout. + if [ ! -d "libmongocrypt" ]; then + echo "Run encryption setup first!" + exit 1 + fi python -m pip install '.[encryption]' - # Setup encryption if necessary. - if [ ! -d "libmongocrypt" ]; then - bash ./.evergreen/setup-encryption.sh - fi - # Use the nocrypto build to avoid dependency issues with older windows/python versions. BASE=$(pwd)/libmongocrypt/nocrypto if [ -f "${BASE}/lib/libmongocrypt.so" ]; then diff --git a/.evergreen/setup-encryption.sh b/.evergreen/setup-encryption.sh index b439c15dd..71231e173 100644 --- a/.evergreen/setup-encryption.sh +++ b/.evergreen/setup-encryption.sh @@ -4,6 +4,7 @@ set -o xtrace if [ -z "${DRIVERS_TOOLS}" ]; then echo "Missing environment variable DRIVERS_TOOLS" + exit 1 fi TARGET="" @@ -50,5 +51,7 @@ tar xzf libmongocrypt.tar.gz -C ./libmongocrypt ls -la libmongocrypt ls -la libmongocrypt/nocrypto -bash ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh -bash ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh +if [ -z "${SKIP_SERVERS:-}" ]; then + bash ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh + bash ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh +fi diff --git a/doc/async-tutorial.rst b/doc/async-tutorial.rst new file mode 100644 index 000000000..caa277f9d --- /dev/null +++ b/doc/async-tutorial.rst @@ -0,0 +1,417 @@ +Async Tutorial +============== + +.. code-block:: pycon + + from pymongo import AsyncMongoClient + + client = AsyncMongoClient() + await client.drop_database("test-database") + +This tutorial is intended as an introduction to working with +**MongoDB** and **PyMongo** using the asynchronous API. + +Prerequisites +------------- +Before we start, make sure that you have the **PyMongo** distribution +:doc:`installed `. In the Python shell, the following +should run without raising an exception: + +.. code-block:: pycon + + >>> import pymongo + +This tutorial also assumes that a MongoDB instance is running on the +default host and port. Assuming you have `downloaded and installed +`_ MongoDB, you +can start it like so: + +.. code-block:: bash + + $ mongod + +Making a Connection with AsyncMongoClient +----------------------------------------- +The first step when working with **PyMongo** is to create a +:class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` to the running **mongod** +instance. Doing so is easy: + +.. code-block:: pycon + + >>> from pymongo import AsyncMongoClient + >>> client = AsyncMongoClient() + +The above code will connect on the default host and port. We can also +specify the host and port explicitly, as follows: + +.. code-block:: pycon + + >>> client = AsyncMongoClient("localhost", 27017) + +Or use the MongoDB URI format: + +.. code-block:: pycon + + >>> client = AsyncMongoClient("mongodb://localhost:27017/") + +By default, :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` only connects to the database on its first operation. +To explicitly connect before performing an operation, use :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.aconnect`: + +.. code-block:: pycon + + >>> client = await AsyncMongoClient().aconnect() + +Getting a Database +------------------ +A single instance of MongoDB can support multiple independent +`databases `_. When +working with PyMongo you access databases using attribute style access +on :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` instances: + +.. code-block:: pycon + + >>> db = client.test_database + +If your database name is such that using attribute style access won't +work (like ``test-database``), you can use dictionary style access +instead: + +.. code-block:: pycon + + >>> db = client["test-database"] + +Getting a Collection +-------------------- +A `collection `_ is a +group of documents stored in MongoDB, and can be thought of as roughly +the equivalent of a table in a relational database. Getting a +collection in PyMongo works the same as getting a database: + +.. code-block:: pycon + + >>> collection = db.test_collection + +or (using dictionary style access): + +.. code-block:: pycon + + >>> collection = db["test-collection"] + +An important note about collections (and databases) in MongoDB is that +they are created lazily - none of the above commands have actually +performed any operations on the MongoDB server. Collections and +databases are created when the first document is inserted into them. + +Documents +--------- +Data in MongoDB is represented (and stored) using JSON-style +documents. In PyMongo we use dictionaries to represent documents. As +an example, the following dictionary might be used to represent a blog +post: + +.. code-block:: pycon + + >>> import datetime + >>> post = { + ... "author": "Mike", + ... "text": "My first blog post!", + ... "tags": ["mongodb", "python", "pymongo"], + ... "date": datetime.datetime.now(tz=datetime.timezone.utc), + ... } + +Note that documents can contain native Python types (like +:class:`datetime.datetime` instances) which will be automatically +converted to and from the appropriate `BSON +`_ types. + +Inserting a Document +-------------------- +To insert a document into a collection we can use the +:meth:`~pymongo.asynchronous.collection.AsyncCollection.insert_one` method: + +.. code-block:: pycon + + >>> posts = db.posts + >>> post_id = (await posts.insert_one(post)).inserted_id + >>> post_id + ObjectId('...') + +When a document is inserted a special key, ``"_id"``, is automatically +added if the document doesn't already contain an ``"_id"`` key. The value +of ``"_id"`` must be unique across the +collection. :meth:`~pymongo.asynchronous.collection.AsyncCollection.insert_one` returns an +instance of :class:`~pymongo.results.InsertOneResult`. For more information +on ``"_id"``, see the `documentation on _id +`_. + +After inserting the first document, the *posts* collection has +actually been created on the server. We can verify this by listing all +of the collections in our database: + +.. code-block:: pycon + + >>> await db.list_collection_names() + ['posts'] + +Getting a Single Document With :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one` +------------------------------------------------------------------------------------------------ +The most basic type of query that can be performed in MongoDB is +:meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one`. This method returns a +single document matching a query (or ``None`` if there are no +matches). It is useful when you know there is only one matching +document, or are only interested in the first match. Here we use +:meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one` to get the first +document from the posts collection: + +.. code-block:: pycon + + >>> import pprint + >>> pprint.pprint(await posts.find_one()) + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['mongodb', 'python', 'pymongo'], + 'text': 'My first blog post!'} + +The result is a dictionary matching the one that we inserted previously. + +.. note:: The returned document contains an ``"_id"``, which was + automatically added on insert. + +:meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one` also supports querying +on specific elements that the resulting document must match. To limit +our results to a document with author "Mike" we do: + +.. code-block:: pycon + + >>> pprint.pprint(await posts.find_one({"author": "Mike"})) + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['mongodb', 'python', 'pymongo'], + 'text': 'My first blog post!'} + +If we try with a different author, like "Eliot", we'll get no result: + +.. code-block:: pycon + + >>> await posts.find_one({"author": "Eliot"}) + >>> + +.. _async-querying-by-objectid: + +Querying By ObjectId +-------------------- +We can also find a post by its ``_id``, which in our example is an ObjectId: + +.. code-block:: pycon + + >>> post_id + ObjectId(...) + >>> pprint.pprint(await posts.find_one({"_id": post_id})) + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['mongodb', 'python', 'pymongo'], + 'text': 'My first blog post!'} + +Note that an ObjectId is not the same as its string representation: + +.. code-block:: pycon + + >>> post_id_as_str = str(post_id) + >>> await posts.find_one({"_id": post_id_as_str}) # No result + >>> + +A common task in web applications is to get an ObjectId from the +request URL and find the matching document. It's necessary in this +case to **convert the ObjectId from a string** before passing it to +``find_one``:: + + from bson.objectid import ObjectId + + # The web framework gets post_id from the URL and passes it as a string + async def get(post_id): + # Convert from string to ObjectId: + document = await client.db.collection.find_one({'_id': ObjectId(post_id)}) + +.. seealso:: :ref:`web-application-querying-by-objectid` + +Bulk Inserts +------------ +In order to make querying a little more interesting, let's insert a +few more documents. In addition to inserting a single document, we can +also perform *bulk insert* operations, by passing a list as the +first argument to :meth:`~pymongo.asynchronous.collection.AsyncCollection.insert_many`. +This will insert each document in the list, sending only a single +command to the server: + +.. code-block:: pycon + + >>> new_posts = [ + ... { + ... "author": "Mike", + ... "text": "Another post!", + ... "tags": ["bulk", "insert"], + ... "date": datetime.datetime(2009, 11, 12, 11, 14), + ... }, + ... { + ... "author": "Eliot", + ... "title": "MongoDB is fun", + ... "text": "and pretty easy too!", + ... "date": datetime.datetime(2009, 11, 10, 10, 45), + ... }, + ... ] + >>> result = await posts.insert_many(new_posts) + >>> result.inserted_ids + [ObjectId('...'), ObjectId('...')] + +There are a couple of interesting things to note about this example: + + - The result from :meth:`~pymongo.asynchronous.collection.AsyncCollection.insert_many` now + returns two :class:`~bson.objectid.ObjectId` instances, one for + each inserted document. + - ``new_posts[1]`` has a different "shape" than the other posts - + there is no ``"tags"`` field and we've added a new field, + ``"title"``. This is what we mean when we say that MongoDB is + *schema-free*. + +Querying for More Than One Document +----------------------------------- +To get more than a single document as the result of a query we use the +:meth:`~pymongo.asynchronous.collection.AsyncCollection.find` +method. :meth:`~pymongo.asynchronous.collection.AsyncCollection.find` returns a +:class:`~pymongo.asynchronous.cursor.AsyncCursor` instance, which allows us to iterate +over all matching documents. For example, we can iterate over every +document in the ``posts`` collection: + +.. code-block:: pycon + + >>> async for post in posts.find(): + ... pprint.pprint(post) + ... + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['mongodb', 'python', 'pymongo'], + 'text': 'My first blog post!'} + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['bulk', 'insert'], + 'text': 'Another post!'} + {'_id': ObjectId('...'), + 'author': 'Eliot', + 'date': datetime.datetime(...), + 'text': 'and pretty easy too!', + 'title': 'MongoDB is fun'} + +Just like we did with :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one`, +we can pass a document to :meth:`~pymongo.asynchronous.collection.AsyncCollection.find` +to limit the returned results. Here, we get only those documents whose +author is "Mike": + +.. code-block:: pycon + + >>> async for post in posts.find({"author": "Mike"}): + ... pprint.pprint(post) + ... + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['mongodb', 'python', 'pymongo'], + 'text': 'My first blog post!'} + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['bulk', 'insert'], + 'text': 'Another post!'} + +Counting +-------- +If we just want to know how many documents match a query we can +perform a :meth:`~pymongo.asynchronous.collection.AsyncCollection.count_documents` operation +instead of a full query. We can get a count of all of the documents +in a collection: + +.. code-block:: pycon + + >>> await posts.count_documents({}) + 3 + +or just of those documents that match a specific query: + +.. code-block:: pycon + + >>> await posts.count_documents({"author": "Mike"}) + 2 + +Range Queries +------------- +MongoDB supports many different types of `advanced queries +`_. As an +example, lets perform a query where we limit results to posts older +than a certain date, but also sort the results by author: + +.. code-block:: pycon + + >>> d = datetime.datetime(2009, 11, 12, 12) + >>> async for post in posts.find({"date": {"$lt": d}}).sort("author"): + ... pprint.pprint(post) + ... + {'_id': ObjectId('...'), + 'author': 'Eliot', + 'date': datetime.datetime(...), + 'text': 'and pretty easy too!', + 'title': 'MongoDB is fun'} + {'_id': ObjectId('...'), + 'author': 'Mike', + 'date': datetime.datetime(...), + 'tags': ['bulk', 'insert'], + 'text': 'Another post!'} + +Here we use the special ``"$lt"`` operator to do a range query, and +also call :meth:`~pymongo.asynchronous.cursor.AsyncCursor.sort` to sort the results +by author. + +Indexing +-------- + +Adding indexes can help accelerate certain queries and can also add additional +functionality to querying and storing documents. In this example, we'll +demonstrate how to create a `unique index +`_ on a key that rejects +documents whose value for that key already exists in the index. + +First, we'll need to create the index: + +.. code-block:: pycon + + >>> result = await db.profiles.create_index([("user_id", pymongo.ASCENDING)], unique=True) + >>> sorted(list(await db.profiles.index_information())) + ['_id_', 'user_id_1'] + +Notice that we have two indexes now: one is the index on ``_id`` that MongoDB +creates automatically, and the other is the index on ``user_id`` we just +created. + +Now let's set up some user profiles: + +.. code-block:: pycon + + >>> user_profiles = [{"user_id": 211, "name": "Luke"}, {"user_id": 212, "name": "Ziltoid"}] + >>> result = await db.profiles.insert_many(user_profiles) + +The index prevents us from inserting a document whose ``user_id`` is already in +the collection: + +.. code-block:: pycon + + >>> new_profile = {"user_id": 213, "name": "Drew"} + >>> duplicate_profile = {"user_id": 212, "name": "Tommy"} + >>> result = await db.profiles.insert_one(new_profile) # This is fine. + >>> result = await db.profiles.insert_one(duplicate_profile) + Traceback (most recent call last): + DuplicateKeyError: E11000 duplicate key error index: test_database.profiles.$user_id_1 dup key: { : 212 } + +.. seealso:: The MongoDB documentation on `indexes `_ diff --git a/doc/contributors.rst b/doc/contributors.rst index 49fb2d844..272b81d6a 100644 --- a/doc/contributors.rst +++ b/doc/contributors.rst @@ -101,3 +101,4 @@ The following is a list of people who have contributed to - Casey Clements (caseyclements) - Ivan Lukyanchikov (ilukyanchikov) - Terry Patterson +- Romain Morotti diff --git a/doc/index.rst b/doc/index.rst index 096a6a8ae..71e142381 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,6 +16,9 @@ everything you need to know to use **PyMongo**. :doc:`tutorial` Start here for a quick overview. +:doc:`async-tutorial` + Start here for a quick overview of the asynchronous API. + :doc:`examples/index` Examples of how to perform specific tasks. @@ -121,6 +124,7 @@ Indices and tables atlas installation tutorial + async-tutorial examples/index faq compatibility-policy diff --git a/pymongo/_client_bulk_shared.py b/pymongo/_client_bulk_shared.py new file mode 100644 index 000000000..4dd1af210 --- /dev/null +++ b/pymongo/_client_bulk_shared.py @@ -0,0 +1,77 @@ +# Copyright 2024-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + + +"""Constants, types, and classes shared across Client Bulk Write API implementations.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, NoReturn + +from pymongo.errors import ClientBulkWriteException, OperationFailure +from pymongo.helpers_shared import _get_wce_doc + +if TYPE_CHECKING: + from pymongo.typings import _DocumentOut + + +def _merge_command( + ops: list[tuple[str, Mapping[str, Any]]], + offset: int, + full_result: MutableMapping[str, Any], + result: Mapping[str, Any], +) -> None: + """Merge result of a single bulk write batch into the full result.""" + if result.get("error"): + full_result["error"] = result["error"] + + full_result["nInserted"] += result.get("nInserted", 0) + full_result["nDeleted"] += result.get("nDeleted", 0) + full_result["nMatched"] += result.get("nMatched", 0) + full_result["nModified"] += result.get("nModified", 0) + full_result["nUpserted"] += result.get("nUpserted", 0) + + write_errors = result.get("writeErrors") + if write_errors: + for doc in write_errors: + # Leave the server response intact for APM. + replacement = doc.copy() + original_index = doc["idx"] + offset + replacement["idx"] = original_index + # Add the failed operation to the error document. + replacement["op"] = ops[original_index][1] + full_result["writeErrors"].append(replacement) + + wce = _get_wce_doc(result) + if wce: + full_result["writeConcernErrors"].append(wce) + + +def _throw_client_bulk_write_exception( + full_result: _DocumentOut, verbose_results: bool +) -> NoReturn: + """Raise a ClientBulkWriteException from the full result.""" + # retryWrites on MMAPv1 should raise an actionable error. + if full_result["writeErrors"]: + full_result["writeErrors"].sort(key=lambda error: error["idx"]) + err = full_result["writeErrors"][0] + code = err["code"] + msg = err["errmsg"] + if code == 20 and msg.startswith("Transaction numbers"): + errmsg = ( + "This MongoDB deployment does not support " + "retryable writes. Please add retryWrites=false " + "to your connection string." + ) + raise OperationFailure(errmsg, code, full_result) + raise ClientBulkWriteException(full_result, verbose_results) diff --git a/pymongo/_csot.py b/pymongo/_csot.py index 2ac02aa9e..94328f981 100644 --- a/pymongo/_csot.py +++ b/pymongo/_csot.py @@ -149,10 +149,7 @@ class MovingMinimum: def add_sample(self, sample: float) -> None: if sample < 0: - # Likely system time change while waiting for hello response - # and not using time.monotonic. Ignore it, the next one will - # probably be valid. - return + raise ValueError(f"duration cannot be negative {sample}") self.samples.append(sample) def get(self) -> float: diff --git a/pymongo/asynchronous/aggregation.py b/pymongo/asynchronous/aggregation.py index fa6cefd53..768415189 100644 --- a/pymongo/asynchronous/aggregation.py +++ b/pymongo/asynchronous/aggregation.py @@ -40,8 +40,8 @@ class _AggregationCommand: """The internal abstract base class for aggregation cursors. Should not be called directly by application developers. Use - :meth:`pymongo.collection.AsyncCollection.aggregate`, or - :meth:`pymongo.database.AsyncDatabase.aggregate` instead. + :meth:`pymongo.asynchronous.collection.AsyncCollection.aggregate`, or + :meth:`pymongo.asynchronous.database.AsyncDatabase.aggregate` instead. """ def __init__( diff --git a/pymongo/asynchronous/change_stream.py b/pymongo/asynchronous/change_stream.py index 92cc95cf5..719020c40 100644 --- a/pymongo/asynchronous/change_stream.py +++ b/pymongo/asynchronous/change_stream.py @@ -91,9 +91,9 @@ class AsyncChangeStream(Generic[_DocumentType]): """The internal abstract base class for change stream cursors. Should not be called directly by application developers. Use - :meth:`pymongo.collection.AsyncCollection.watch`, - :meth:`pymongo.database.AsyncDatabase.watch`, or - :meth:`pymongo.mongo_client.AsyncMongoClient.watch` instead. + :meth:`pymongo.asynchronous.collection.AsyncCollection.watch`, + :meth:`pymongo.asynchronous.database.AsyncDatabase.watch`, or + :meth:`pymongo.asynchronous.mongo_client.AsyncMongoClient.watch` instead. .. versionadded:: 3.6 .. seealso:: The MongoDB documentation on `changeStreams `_. @@ -166,7 +166,7 @@ class AsyncChangeStream(Generic[_DocumentType]): @property def _client(self) -> AsyncMongoClient: """The client against which the aggregation commands for - this ChangeStream will be run. + this AsyncChangeStream will be run. """ raise NotImplementedError @@ -204,7 +204,7 @@ class AsyncChangeStream(Generic[_DocumentType]): return options def _aggregation_pipeline(self) -> list[dict[str, Any]]: - """Return the full aggregation pipeline for this ChangeStream.""" + """Return the full aggregation pipeline for this AsyncChangeStream.""" options = self._change_stream_options() full_pipeline: list = [{"$changeStream": options}] full_pipeline.extend(self._pipeline) @@ -238,7 +238,7 @@ class AsyncChangeStream(Generic[_DocumentType]): async def _run_aggregation_cmd( self, session: Optional[AsyncClientSession], explicit_session: bool ) -> AsyncCommandCursor: - """Run the full aggregation pipeline for this ChangeStream and return + """Run the full aggregation pipeline for this AsyncChangeStream and return the corresponding AsyncCommandCursor. """ cmd = self._aggregation_command_class( @@ -272,7 +272,7 @@ class AsyncChangeStream(Generic[_DocumentType]): self._cursor = await self._create_cursor() async def close(self) -> None: - """Close this ChangeStream.""" + """Close this AsyncChangeStream.""" self._closed = True await self._cursor.close() @@ -299,27 +299,27 @@ class AsyncChangeStream(Generic[_DocumentType]): try: resume_token = None pipeline = [{'$match': {'operationType': 'insert'}}] - async with db.collection.watch(pipeline) as stream: + async with await db.collection.watch(pipeline) as stream: async for insert_change in stream: print(insert_change) resume_token = stream.resume_token except pymongo.errors.PyMongoError: - # The ChangeStream encountered an unrecoverable error or the + # The AsyncChangeStream encountered an unrecoverable error or the # resume attempt failed to recreate the cursor. if resume_token is None: # There is no usable resume token because there was a - # failure during ChangeStream initialization. + # failure during AsyncChangeStream initialization. logging.error('...') else: - # Use the interrupted ChangeStream's resume token to create - # a new ChangeStream. The new stream will continue from the + # Use the interrupted AsyncChangeStream's resume token to create + # a new AsyncChangeStream. The new stream will continue from the # last seen insert change without missing any events. - async with db.collection.watch( + async with await db.collection.watch( pipeline, resume_after=resume_token) as stream: async for insert_change in stream: print(insert_change) - Raises :exc:`StopIteration` if this ChangeStream is closed. + Raises :exc:`StopIteration` if this AsyncChangeStream is closed. """ while self.alive: doc = await self.try_next() @@ -348,10 +348,10 @@ class AsyncChangeStream(Generic[_DocumentType]): This method returns the next change document without waiting indefinitely for the next change. For example:: - async with db.collection.watch() as stream: + async with await db.collection.watch() as stream: while stream.alive: change = await stream.try_next() - # Note that the ChangeStream's resume token may be updated + # Note that the AsyncChangeStream's resume token may be updated # even when no changes are returned. print("Current resume token: %r" % (stream.resume_token,)) if change is not None: @@ -447,7 +447,7 @@ class AsyncCollectionChangeStream(AsyncChangeStream[_DocumentType]): """A change stream that watches changes on a single collection. Should not be called directly by application developers. Use - helper method :meth:`pymongo.collection.AsyncCollection.watch` instead. + helper method :meth:`pymongo.asynchronous.collection.AsyncCollection.watch` instead. .. versionadded:: 3.7 """ @@ -467,7 +467,7 @@ class AsyncDatabaseChangeStream(AsyncChangeStream[_DocumentType]): """A change stream that watches changes on all collections in a database. Should not be called directly by application developers. Use - helper method :meth:`pymongo.database.AsyncDatabase.watch` instead. + helper method :meth:`pymongo.asynchronous.database.AsyncDatabase.watch` instead. .. versionadded:: 3.7 """ @@ -487,7 +487,7 @@ class AsyncClusterChangeStream(AsyncDatabaseChangeStream[_DocumentType]): """A change stream that watches changes on all collections in the cluster. Should not be called directly by application developers. Use - helper method :meth:`pymongo.mongo_client.AsyncMongoClient.watch` instead. + helper method :meth:`pymongo.asynchronous.mongo_client.AsyncMongoClient.watch` instead. .. versionadded:: 3.7 """ diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py new file mode 100644 index 000000000..671d989c2 --- /dev/null +++ b/pymongo/asynchronous/client_bulk.py @@ -0,0 +1,788 @@ +# Copyright 2024-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The client-level bulk write operations interface. + +.. versionadded:: 4.9 +""" +from __future__ import annotations + +import copy +import datetime +import logging +from collections.abc import MutableMapping +from itertools import islice +from typing import ( + TYPE_CHECKING, + Any, + Mapping, + Optional, + Type, + Union, +) + +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo import _csot, common +from pymongo.asynchronous.client_session import AsyncClientSession, _validate_session_write_concern +from pymongo.asynchronous.collection import AsyncCollection +from pymongo.asynchronous.command_cursor import AsyncCommandCursor +from pymongo.asynchronous.database import AsyncDatabase +from pymongo.asynchronous.helpers import _handle_reauth + +if TYPE_CHECKING: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.asynchronous.pool import AsyncConnection +from pymongo._client_bulk_shared import ( + _merge_command, + _throw_client_bulk_write_exception, +) +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) +from pymongo.errors import ( + ConfigurationError, + ConnectionFailure, + InvalidOperation, + NotPrimaryError, + OperationFailure, + WaitQueueTimeoutError, +) +from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES +from pymongo.logger import _COMMAND_LOGGER, _CommandStatusMessage, _debug_log +from pymongo.message import ( + _ClientBulkWriteContext, + _convert_client_bulk_exception, + _convert_exception, + _convert_write_result, + _randint, +) +from pymongo.read_preferences import ReadPreference +from pymongo.results import ( + ClientBulkWriteResult, + DeleteResult, + InsertOneResult, + UpdateResult, +) +from pymongo.typings import _DocumentOut, _Pipeline +from pymongo.write_concern import WriteConcern + +_IS_SYNC = False + + +class _AsyncClientBulk: + """The private guts of the client-level bulk write API.""" + + def __init__( + self, + client: AsyncMongoClient, + write_concern: WriteConcern, + ordered: bool = True, + bypass_document_validation: Optional[bool] = None, + comment: Optional[str] = None, + let: Optional[Any] = None, + verbose_results: bool = False, + ) -> None: + """Initialize a _AsyncClientBulk instance.""" + self.client = client + self.write_concern = write_concern + self.let = let + if self.let is not None: + common.validate_is_document_type("let", self.let) + self.ordered = ordered + self.bypass_doc_val = bypass_document_validation + self.comment = comment + self.verbose_results = verbose_results + + self.ops: list[tuple[str, Mapping[str, Any]]] = [] + self.idx_offset: int = 0 + self.total_ops: int = 0 + + self.executed = False + self.uses_upsert = False + self.uses_collation = False + self.uses_array_filters = False + self.uses_hint_update = False + self.uses_hint_delete = False + + self.is_retryable = self.client.options.retry_writes + self.retrying = False + self.started_retryable_write = False + + @property + def bulk_ctx_class(self) -> Type[_ClientBulkWriteContext]: + return _ClientBulkWriteContext + + def add_insert(self, namespace: str, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + cmd = {"insert": namespace, "document": document} + self.ops.append(("insert", cmd)) + self.total_ops += 1 + + def add_update( + self, + namespace: str, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool = False, + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd = { + "update": namespace, + "filter": selector, + "updateMods": update, + "multi": multi, + } + if upsert is not None: + self.uses_upsert = True + cmd["upsert"] = upsert + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + self.uses_hint_update = True + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("update", cmd)) + self.total_ops += 1 + + def add_replace( + self, + namespace: str, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd = { + "update": namespace, + "filter": selector, + "updateMods": replacement, + "multi": False, + } + if upsert is not None: + self.uses_upsert = True + cmd["upsert"] = upsert + if hint is not None: + self.uses_hint_update = True + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + self.ops.append(("replace", cmd)) + self.total_ops += 1 + + def add_delete( + self, + namespace: str, + selector: Mapping[str, Any], + multi: bool, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd = {"delete": namespace, "filter": selector, "multi": multi} + if hint is not None: + self.uses_hint_delete = True + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("delete", cmd)) + self.total_ops += 1 + + @_handle_reauth + async def write_command( + self, + bwc: _ClientBulkWriteContext, + cmd: MutableMapping[str, Any], + request_id: int, + msg: Union[bytes, dict[str, Any]], + op_docs: list[Mapping[str, Any]], + ns_docs: list[Mapping[str, Any]], + client: AsyncMongoClient, + ) -> dict[str, Any]: + """A proxy for AsyncConnection.write_command that handles event publishing.""" + cmd["ops"] = op_docs + cmd["nsInfo"] = ns_docs + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.STARTED, + command=cmd, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + bwc._start(cmd, request_id, op_docs, ns_docs) + try: + reply = await bwc.conn.write_command(request_id, msg, bwc.codec) # type: ignore[misc, arg-type] + duration = datetime.datetime.now() - bwc.start_time + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.SUCCEEDED, + durationMS=duration, + reply=reply, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + bwc._succeed(request_id, reply, duration) # type: ignore[arg-type] + except Exception as exc: + duration = datetime.datetime.now() - bwc.start_time + if isinstance(exc, (NotPrimaryError, OperationFailure)): + failure: _DocumentOut = exc.details # type: ignore[assignment] + else: + failure = _convert_exception(exc) + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.FAILED, + durationMS=duration, + failure=failure, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + isServerSideError=isinstance(exc, OperationFailure), + ) + + if bwc.publish: + bwc._fail(request_id, failure, duration) + # Top-level error will be embedded in ClientBulkWriteException. + reply = {"error": exc} + finally: + bwc.start_time = datetime.datetime.now() + return reply # type: ignore[return-value] + + async def unack_write( + self, + bwc: _ClientBulkWriteContext, + cmd: MutableMapping[str, Any], + request_id: int, + msg: bytes, + op_docs: list[Mapping[str, Any]], + ns_docs: list[Mapping[str, Any]], + client: AsyncMongoClient, + ) -> Optional[Mapping[str, Any]]: + """A proxy for AsyncConnection.unack_write that handles event publishing.""" + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.STARTED, + command=cmd, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + cmd = bwc._start(cmd, request_id, op_docs, ns_docs) + try: + result = await bwc.conn.unack_write(msg, bwc.max_bson_size) # type: ignore[func-returns-value, misc, override] + duration = datetime.datetime.now() - bwc.start_time + if result is not None: + reply = _convert_write_result(bwc.name, cmd, result) # type: ignore[arg-type] + else: + # Comply with APM spec. + reply = {"ok": 1} + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.SUCCEEDED, + durationMS=duration, + reply=reply, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + bwc._succeed(request_id, reply, duration) + except Exception as exc: + duration = datetime.datetime.now() - bwc.start_time + if isinstance(exc, OperationFailure): + failure: _DocumentOut = _convert_write_result(bwc.name, cmd, exc.details) # type: ignore[arg-type] + elif isinstance(exc, NotPrimaryError): + failure = exc.details # type: ignore[assignment] + else: + failure = _convert_exception(exc) + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.FAILED, + durationMS=duration, + failure=failure, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + isServerSideError=isinstance(exc, OperationFailure), + ) + if bwc.publish: + assert bwc.start_time is not None + bwc._fail(request_id, failure, duration) + # Top-level error will be embedded in ClientBulkWriteException. + reply = {"error": exc} + finally: + bwc.start_time = datetime.datetime.now() + return result # type: ignore[return-value] + + async def _execute_batch_unack( + self, + bwc: _ClientBulkWriteContext, + cmd: dict[str, Any], + ops: list[tuple[str, Mapping[str, Any]]], + ) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Executes a batch of bulkWrite server commands (unack).""" + request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops) + await self.unack_write(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type] + return to_send_ops, to_send_ns + + async def _execute_batch( + self, + bwc: _ClientBulkWriteContext, + cmd: dict[str, Any], + ops: list[tuple[str, Mapping[str, Any]]], + ) -> tuple[dict[str, Any], list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Executes a batch of bulkWrite server commands (ack).""" + request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops) + result = await self.write_command( + bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client + ) # type: ignore[arg-type] + await self.client._process_response(result, bwc.session) # type: ignore[arg-type] + return result, to_send_ops, to_send_ns # type: ignore[return-value] + + async def _process_results_cursor( + self, + full_result: MutableMapping[str, Any], + result: MutableMapping[str, Any], + conn: AsyncConnection, + session: Optional[AsyncClientSession], + ) -> None: + """Internal helper for processing the server reply command cursor.""" + if result.get("cursor"): + coll = AsyncCollection( + database=AsyncDatabase(self.client, "admin"), + name="$cmd.bulkWrite", + ) + cmd_cursor = AsyncCommandCursor( + coll, + result["cursor"], + conn.address, + session=session, + explicit_session=session is not None, + comment=self.comment, + ) + await cmd_cursor._maybe_pin_connection(conn) + + # Iterate the cursor to get individual write results. + try: + async for doc in cmd_cursor: + original_index = doc["idx"] + self.idx_offset + op_type, op = self.ops[original_index] + + if not doc["ok"]: + result["writeErrors"].append(doc) + if self.ordered: + return + + # Record individual write result. + if doc["ok"] and self.verbose_results: + if op_type == "insert": + inserted_id = op["document"]["_id"] + res = InsertOneResult(inserted_id, acknowledged=True) # type: ignore[assignment] + if op_type in ["update", "replace"]: + op_type = "update" + res = UpdateResult(doc, acknowledged=True, in_client_bulk=True) # type: ignore[assignment] + if op_type == "delete": + res = DeleteResult(doc, acknowledged=True) # type: ignore[assignment] + full_result[f"{op_type}Results"][original_index] = res + + except Exception as exc: + # Attempt to close the cursor, then raise top-level error. + if cmd_cursor.alive: + await cmd_cursor.close() + result["error"] = _convert_client_bulk_exception(exc) + + async def _execute_command( + self, + write_concern: WriteConcern, + session: Optional[AsyncClientSession], + conn: AsyncConnection, + op_id: int, + retryable: bool, + full_result: MutableMapping[str, Any], + final_write_concern: Optional[WriteConcern] = None, + ) -> None: + """Internal helper for executing batches of bulkWrite commands.""" + db_name = "admin" + cmd_name = "bulkWrite" + listeners = self.client._event_listeners + + # AsyncConnection.command validates the session, but we use + # AsyncConnection.write_command + conn.validate_session(self.client, session) + + bwc = self.bulk_ctx_class( + db_name, + cmd_name, + conn, + op_id, + listeners, # type: ignore[arg-type] + session, + self.client.codec_options, + ) + + while self.idx_offset < self.total_ops: + # If this is the last possible batch, use the + # final write concern. + if self.total_ops - self.idx_offset <= bwc.max_write_batch_size: + write_concern = final_write_concern or write_concern + + # Construct the server command, specifying the relevant options. + cmd = {"bulkWrite": 1} + cmd["errorsOnly"] = not self.verbose_results + cmd["ordered"] = self.ordered # type: ignore[assignment] + not_in_transaction = session and not session.in_transaction + if not_in_transaction or not session: + _csot.apply_write_concern(cmd, write_concern) + if self.bypass_doc_val is not None: + cmd["bypassDocumentValidation"] = self.bypass_doc_val + if self.comment: + cmd["comment"] = self.comment # type: ignore[assignment] + if self.let: + cmd["let"] = self.let + + if session: + # Start a new retryable write unless one was already + # started for this command. + if retryable and not self.started_retryable_write: + session._start_retryable_write() + self.started_retryable_write = True + session._apply_to(cmd, retryable, ReadPreference.PRIMARY, conn) + conn.send_cluster_time(cmd, session, self.client) + conn.add_server_api(cmd) + # CSOT: apply timeout before encoding the command. + conn.apply_timeout(self.client, cmd) + ops = islice(self.ops, self.idx_offset, None) + + # Run as many ops as possible in one server command. + if write_concern.acknowledged: + raw_result, to_send_ops, _ = await self._execute_batch(bwc, cmd, ops) # type: ignore[arg-type] + result = copy.deepcopy(raw_result) + + # Top-level server/network error. + if result.get("error"): + error = result["error"] + retryable_top_level_error = ( + isinstance(error.details, dict) + and error.details.get("code", 0) in _RETRYABLE_ERROR_CODES + ) + retryable_network_error = isinstance( + error, ConnectionFailure + ) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError)) + + # Synthesize the full bulk result without modifying the + # current one because this write operation may be retried. + if retryable and (retryable_top_level_error or retryable_network_error): + full = copy.deepcopy(full_result) + _merge_command(self.ops, self.idx_offset, full, result) + _throw_client_bulk_write_exception(full, self.verbose_results) + else: + _merge_command(self.ops, self.idx_offset, full_result, result) + _throw_client_bulk_write_exception(full_result, self.verbose_results) + + result["error"] = None + result["writeErrors"] = [] + if result.get("nErrors", 0) < len(to_send_ops): + full_result["anySuccessful"] = True + + # Top-level command error. + if not result["ok"]: + result["error"] = raw_result + _merge_command(self.ops, self.idx_offset, full_result, result) + break + + if retryable: + # Retryable writeConcernErrors halt the execution of this batch. + wce = result.get("writeConcernError", {}) + if wce.get("code", 0) in _RETRYABLE_ERROR_CODES: + # Synthesize the full bulk result without modifying the + # current one because this write operation may be retried. + full = copy.deepcopy(full_result) + _merge_command(self.ops, self.idx_offset, full, result) + _throw_client_bulk_write_exception(full, self.verbose_results) + + # Process the server reply as a command cursor. + await self._process_results_cursor(full_result, result, conn, session) + + # Merge this batch's results with the full results. + _merge_command(self.ops, self.idx_offset, full_result, result) + + # We're no longer in a retry once a command succeeds. + self.retrying = False + self.started_retryable_write = False + + else: + to_send_ops, _ = await self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type] + + self.idx_offset += len(to_send_ops) + + # We halt execution if we hit a top-level error, + # or an individual error in an ordered bulk write. + if full_result["error"] or (self.ordered and full_result["writeErrors"]): + break + + async def execute_command( + self, + session: Optional[AsyncClientSession], + operation: str, + ) -> MutableMapping[str, Any]: + """Execute commands with w=1 WriteConcern.""" + full_result: MutableMapping[str, Any] = { + "anySuccessful": False, + "error": None, + "writeErrors": [], + "writeConcernErrors": [], + "nInserted": 0, + "nUpserted": 0, + "nMatched": 0, + "nModified": 0, + "nDeleted": 0, + "insertResults": {}, + "updateResults": {}, + "deleteResults": {}, + } + op_id = _randint() + + async def retryable_bulk( + session: Optional[AsyncClientSession], + conn: AsyncConnection, + retryable: bool, + ) -> None: + if conn.max_wire_version < 25: + raise InvalidOperation( + "MongoClient.bulk_write requires MongoDB server version 8.0+." + ) + await self._execute_command( + self.write_concern, + session, + conn, + op_id, + retryable, + full_result, + ) + + await self.client._retryable_write( + self.is_retryable, + retryable_bulk, + session, + operation, + bulk=self, + operation_id=op_id, + ) + + if full_result["error"] or full_result["writeErrors"] or full_result["writeConcernErrors"]: + _throw_client_bulk_write_exception(full_result, self.verbose_results) + return full_result + + async def execute_command_unack_unordered( + self, + conn: AsyncConnection, + ) -> None: + """Execute commands with OP_MSG and w=0 writeConcern, unordered.""" + db_name = "admin" + cmd_name = "bulkWrite" + listeners = self.client._event_listeners + op_id = _randint() + + bwc = self.bulk_ctx_class( + db_name, + cmd_name, + conn, + op_id, + listeners, # type: ignore[arg-type] + None, + self.client.codec_options, + ) + + while self.idx_offset < self.total_ops: + # Construct the server command, specifying the relevant options. + cmd = {"bulkWrite": 1} + cmd["errorsOnly"] = not self.verbose_results + cmd["ordered"] = self.ordered # type: ignore[assignment] + if self.bypass_doc_val is not None: + cmd["bypassDocumentValidation"] = self.bypass_doc_val + cmd["writeConcern"] = {"w": 0} # type: ignore[assignment] + if self.comment: + cmd["comment"] = self.comment # type: ignore[assignment] + if self.let: + cmd["let"] = self.let + + conn.add_server_api(cmd) + ops = islice(self.ops, self.idx_offset, None) + + # Run as many ops as possible in one server command. + to_send_ops, _ = await self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type] + + self.idx_offset += len(to_send_ops) + + async def execute_command_unack_ordered( + self, + conn: AsyncConnection, + ) -> None: + """Execute commands with OP_MSG and w=0 WriteConcern, ordered.""" + full_result: MutableMapping[str, Any] = { + "anySuccessful": False, + "error": None, + "writeErrors": [], + "writeConcernErrors": [], + "nInserted": 0, + "nUpserted": 0, + "nMatched": 0, + "nModified": 0, + "nDeleted": 0, + "insertResults": {}, + "updateResults": {}, + "deleteResults": {}, + } + # Ordered bulk writes have to be acknowledged so that we stop + # processing at the first error, even when the application + # specified unacknowledged writeConcern. + initial_write_concern = WriteConcern() + op_id = _randint() + try: + await self._execute_command( + initial_write_concern, + None, + conn, + op_id, + False, + full_result, + self.write_concern, + ) + except OperationFailure: + pass + + async def execute_no_results( + self, + conn: AsyncConnection, + ) -> None: + """Execute all operations, returning no results (w=0).""" + if self.uses_collation: + raise ConfigurationError("Collation is unsupported for unacknowledged writes.") + if self.uses_array_filters: + raise ConfigurationError("arrayFilters is unsupported for unacknowledged writes.") + # Cannot have both unacknowledged writes and bypass document validation. + if self.bypass_doc_val is not None: + raise OperationFailure( + "Cannot set bypass_document_validation with unacknowledged write concern" + ) + + if self.ordered: + return await self.execute_command_unack_ordered(conn) + return await self.execute_command_unack_unordered(conn) + + async def execute( + self, + session: Optional[AsyncClientSession], + operation: str, + ) -> Any: + """Execute operations.""" + if not self.ops: + raise InvalidOperation("No operations to execute") + if self.executed: + raise InvalidOperation("Bulk operations can only be executed once.") + self.executed = True + session = _validate_session_write_concern(session, self.write_concern) + + if not self.write_concern.acknowledged: + async with await self.client._conn_for_writes(session, operation) as connection: + if connection.max_wire_version < 25: + raise InvalidOperation( + "MongoClient.bulk_write requires MongoDB server version 8.0+." + ) + await self.execute_no_results(connection) + return ClientBulkWriteResult(None, False, False) # type: ignore[arg-type] + + result = await self.execute_command(session, operation) + return ClientBulkWriteResult( + result, + self.write_concern.acknowledged, + self.verbose_results, + ) diff --git a/pymongo/asynchronous/client_session.py b/pymongo/asynchronous/client_session.py index 2aff95ee5..d80495d80 100644 --- a/pymongo/asynchronous/client_session.py +++ b/pymongo/asynchronous/client_session.py @@ -102,7 +102,7 @@ Snapshot Reads MongoDB 5.0 adds support for snapshot reads. Snapshot reads are requested by passing the ``snapshot`` option to -:meth:`~pymongo.mongo_client.AsyncMongoClient.start_session`. +:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.start_session`. If ``snapshot`` is True, all read operations that use this session read data from the same snapshot timestamp. The server chooses the latest majority-committed snapshot timestamp when executing the first read operation @@ -123,11 +123,11 @@ Snapshot Reads Limitations Snapshot reads sessions are incompatible with ``causal_consistency=True``. Only the following read operations are supported in a snapshot reads session: -- :meth:`~pymongo.collection.AsyncCollection.find` -- :meth:`~pymongo.collection.AsyncCollection.find_one` -- :meth:`~pymongo.collection.AsyncCollection.aggregate` -- :meth:`~pymongo.collection.AsyncCollection.count_documents` -- :meth:`~pymongo.collection.AsyncCollection.distinct` (on unsharded collections) +- :meth:`~pymongo.asynchronous.collection.AsyncCollection.find` +- :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one` +- :meth:`~pymongo.asynchronous.collection.AsyncCollection.aggregate` +- :meth:`~pymongo.asynchronous.collection.AsyncCollection.count_documents` +- :meth:`~pymongo.asynchronous.collection.AsyncCollection.distinct` (on unsharded collections) Classes ======= @@ -492,7 +492,7 @@ class AsyncClientSession: Should not be initialized directly by application developers - to create a :class:`AsyncClientSession`, call - :meth:`~pymongo.mongo_client.AsyncMongoClient.start_session`. + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.start_session`. """ def __init__( @@ -550,7 +550,7 @@ class AsyncClientSession: @property def client(self) -> AsyncMongoClient: - """The :class:`~pymongo.mongo_client.AsyncMongoClient` this session was + """The :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` this session was created from. """ return self._client @@ -898,7 +898,7 @@ class AsyncClientSession: """Update the cluster time for this session. :param cluster_time: The - :data:`~pymongo.client_session.AsyncClientSession.cluster_time` from + :data:`~pymongo.asynchronous.client_session.AsyncClientSession.cluster_time` from another `AsyncClientSession` instance. """ if not isinstance(cluster_time, _Mapping): @@ -919,7 +919,7 @@ class AsyncClientSession: """Update the operation time for this session. :param operation_time: The - :data:`~pymongo.client_session.AsyncClientSession.operation_time` from + :data:`~pymongo.asynchronous.client_session.AsyncClientSession.operation_time` from another `AsyncClientSession` instance. """ if not isinstance(operation_time, Timestamp): @@ -1133,7 +1133,7 @@ class _ServerSessionPool(collections.deque): def get_server_session(self, session_timeout_minutes: Optional[int]) -> _ServerSession: # Although the Driver Sessions Spec says we only clear stale sessions # in return_server_session, PyMongo can't take a lock when returning - # sessions from a __del__ method (like in Cursor.__die), so it can't + # sessions from a __del__ method (like in AsyncCursor.__die), so it can't # clear stale sessions there. In case many sessions were returned via # __del__, check for stale sessions here too. self._clear_stale(session_timeout_minutes) diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index b05a92202..e634b449f 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -111,8 +111,8 @@ _WriteOp = Union[ class ReturnDocument: """An enum used with - :meth:`~pymongo.collection.AsyncCollection.find_one_and_replace` and - :meth:`~pymongo.collection.AsyncCollection.find_one_and_update`. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one_and_replace` and + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one_and_update`. """ BEFORE = False @@ -155,7 +155,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :class:`str`. Raises :class:`~pymongo.errors.InvalidName` if `name` is not a valid collection name. Any additional keyword arguments will be used as options passed to the create command. See - :meth:`~pymongo.database.AsyncDatabase.create_collection` for valid + :meth:`~pymongo.asynchronous.database.AsyncDatabase.create_collection` for valid options. If `create` is ``True``, `collation` is specified, or any additional @@ -207,8 +207,8 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. Removed the uuid_subtype attribute. - :class:`~pymongo.collection.Collection` no longer returns an - instance of :class:`~pymongo.collection.Collection` for attribute + :class:`~pymongo.asynchronous.collection.AsyncCollection` no longer returns an + instance of :class:`~pymongo.asynchronous.collection.AsyncCollection` for attribute names with leading underscores. You must use dict-style lookups instead:: @@ -249,7 +249,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): if create or kwargs: if _IS_SYNC: warnings.warn( - "The `create` and `kwargs` arguments to Collection are deprecated and will be removed in PyMongo 5.0", + "The `create` and `kwargs` arguments to AsyncCollection are deprecated and will be removed in PyMongo 5.0", DeprecationWarning, stacklevel=2, ) @@ -321,7 +321,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): @property def database(self) -> AsyncDatabase[_DocumentType]: - """The :class:`~pymongo.database.AsyncDatabase` that this + """The :class:`~pymongo.asynchronous.database.AsyncDatabase` that this :class:`AsyncCollection` is a part of. """ return self._database @@ -346,19 +346,19 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the - default) the :attr:`codec_options` of this :class:`Collection` + default) the :attr:`codec_options` of this :class:`AsyncCollection` is used. :param read_preference: The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this - :class:`Collection` is used. See :mod:`~pymongo.read_preferences` + :class:`AsyncCollection` is used. See :mod:`~pymongo.read_preferences` for options. :param write_concern: An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the - default) the :attr:`write_concern` of this :class:`Collection` + default) the :attr:`write_concern` of this :class:`AsyncCollection` is used. :param read_concern: An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the - default) the :attr:`read_concern` of this :class:`Collection` + default) the :attr:`read_concern` of this :class:`AsyncCollection` is used. """ return AsyncCollection( @@ -384,7 +384,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): __iter__ = None def __next__(self) -> NoReturn: - raise TypeError(f"'{type(self).__name__}' object is not iterable") + raise TypeError("'AsyncCollection' object is not iterable") next = __next__ @@ -393,7 +393,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): if "." not in self._name: raise TypeError( f"'{type(self).__name__}' object is not callable. If you " - "meant to call the '%s' method on a 'Database' " + "meant to call the '%s' method on an 'AsyncDatabase' " "object it is failing because no such method " "exists." % self._name ) @@ -427,7 +427,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): .. code-block:: python - async with db.collection.watch() as stream: + async with await db.collection.watch() as stream: async for change in stream: print(change) @@ -443,11 +443,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): .. code-block:: python try: - async with db.collection.watch([{"$match": {"operationType": "insert"}}]) as stream: + async with await db.coll.watch([{"$match": {"operationType": "insert"}}]) as stream: async for insert_change in stream: print(insert_change) except pymongo.errors.PyMongoError: - # The ChangeStream encountered an unrecoverable error or the + # The AsyncChangeStream encountered an unrecoverable error or the # resume attempt failed to recreate the cursor. logging.error("...") @@ -455,7 +455,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): `change streams specification`_. .. note:: Using this helper method is preferred to directly calling - :meth:`~pymongo.collection.AsyncCollection.aggregate` with a + :meth:`~pymongo.asynchronous.collection.AsyncCollection.aggregate` with a ``$changeStream`` stage, for the purpose of supporting resumability. @@ -493,7 +493,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): the specified :class:`~bson.timestamp.Timestamp`. Requires MongoDB >= 4.0. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param start_after: The same as `resume_after` except that `start_after` can resume notifications after an invalidate event. This option and `resume_after` are mutually exclusive. @@ -580,7 +580,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param collation` (optional) - An instance of :class:`~pymongo.collation.Collation`. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param retryable_write: True if this command is a retryable write. :param user_fields: Response fields that should be decoded @@ -689,7 +689,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :class:`~pymongo.operations.DeleteOne`, or :class:`~pymongo.operations.DeleteMany`). - >>> for doc in db.test.find({}): + >>> async for doc in db.test.find({}): ... print(doc) ... {'x': 1, '_id': ObjectId('54f62e60fba5226811f634ef')} @@ -699,7 +699,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): >>> from pymongo import InsertOne, DeleteOne, ReplaceOne >>> requests = [InsertOne({'y': 1}), DeleteOne({'x': 1}), ... ReplaceOne({'w': 1}, {'z': 1}, upsert=True)] - >>> result = db.test.bulk_write(requests) + >>> result = await db.test.bulk_write(requests) >>> result.inserted_count 1 >>> result.deleted_count @@ -708,7 +708,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): 0 >>> result.upserted_ids {2: ObjectId('54f62ee28891e756a6e1abd5')} - >>> for doc in db.test.find({}): + >>> async for doc in db.test.find({}): ... print(doc) ... {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} @@ -725,7 +725,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): write to opt-out of document level validation. Default is ``False``. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param let: Map of parameter names and values. Values must be @@ -834,7 +834,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): write to opt-out of document level validation. Default is ``False``. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. @@ -903,7 +903,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): write to opt-out of document level validation. Default is ``False``. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. @@ -1138,11 +1138,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -1247,11 +1247,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -1347,11 +1347,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -1407,10 +1407,10 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): comment: Optional[Any] = None, encrypted_fields: Optional[Mapping[str, Any]] = None, ) -> None: - """Alias for :meth:`~pymongo.database.AsyncDatabase.drop_collection`. + """Alias for :meth:`~pymongo.asynchronous.database.AsyncDatabase.drop_collection`. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param encrypted_fields: **(BETA)** Document that describes the encrypted fields for @@ -1565,11 +1565,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -1630,11 +1630,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -1722,7 +1722,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): Raises :class:`TypeError` if any of the arguments are of improper type. Returns an instance of - :class:`~pymongo.cursor.AsyncCursor` corresponding to this query. + :class:`~pymongo.asynchronous.cursor.AsyncCursor` corresponding to this query. The :meth:`find` method obeys the :attr:`read_preference` of this :class:`AsyncCollection`. @@ -1736,7 +1736,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): always be returned. Use a dict to exclude fields from the result (e.g. projection={'_id': False}). :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param skip: the number of documents to omit (from the start of the result set) when returning the results :param limit: the maximum number of results to @@ -1772,7 +1772,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param sort: a list of (key, direction) pairs specifying the sort order for this query. See - :meth:`~pymongo.cursor.Cursor.sort` for details. + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.sort` for details. :param allow_partial_results: if True, mongos will return partial results if some shards are down instead of returning an error. @@ -1790,32 +1790,32 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): cursor from returning a document more than once because of an intervening write operation. :param hint: An index, in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). Pass this as an alternative to calling - :meth:`~pymongo.cursor.Cursor.hint` on the cursor to tell Mongo the + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.hint` on the cursor to tell Mongo the proper index to use for the query. :param max_time_ms: Specifies a time limit for a query operation. If the specified time is exceeded, the operation will be aborted and :exc:`~pymongo.errors.ExecutionTimeout` is raised. Pass this as an alternative to calling - :meth:`~pymongo.cursor.AsyncCursor.max_time_ms` on the cursor. + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.max_time_ms` on the cursor. :param max_scan: **DEPRECATED** - The maximum number of documents to scan. Pass this as an alternative to calling - :meth:`~pymongo.cursor.AsyncCursor.max_scan` on the cursor. + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.max_scan` on the cursor. :param min: A list of field, limit pairs specifying the inclusive lower bound for all keys of a specific index in order. Pass this as an alternative to calling - :meth:`~pymongo.cursor.AsyncCursor.min` on the cursor. ``hint`` must + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.min` on the cursor. ``hint`` must also be passed to ensure the query utilizes the correct index. :param max: A list of field, limit pairs specifying the exclusive upper bound for all keys of a specific index in order. Pass this as an alternative to calling - :meth:`~pymongo.cursor.Cursor.max` on the cursor. ``hint`` must + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.max` on the cursor. ``hint`` must also be passed to ensure the query utilizes the correct index. :param comment: A string to attach to the query to help interpret and trace the operation in the server logs and in profile data. Pass this as an alternative to calling - :meth:`~pymongo.cursor.AsyncCursor.comment` on the cursor. + :meth:`~pymongo.asynchronous.cursor.AsyncCursor.comment` on the cursor. :param allow_disk_use: if True, MongoDB may use temporary disk files to store data exceeding the system memory limit while processing a blocking sort operation. The option has no effect if @@ -1834,7 +1834,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): - A :class:`~pymongo.cursor.AsyncCursor` instance created with the :attr:`~pymongo.cursor.CursorType.EXHAUST` cursor_type requires an exclusive :class:`~socket.socket` connection to MongoDB. If the - :class:`~pymongo.cursor.AsyncCursor` is discarded without being + :class:`~pymongo.asynchronous.cursor.AsyncCursor` is discarded without being completely iterated the underlying :class:`~socket.socket` connection will be closed and discarded without being returned to the connection pool. @@ -1899,7 +1899,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): """Query the database and retrieve batches of raw BSON. Similar to the :meth:`find` method but returns a - :class:`~pymongo.cursor.AsyncRawBatchCursor`. + :class:`~pymongo.asynchronous.cursor.AsyncRawBatchCursor`. This example demonstrates how to work with raw batches, but in practice raw batches should be passed to an external library that can decode @@ -2069,7 +2069,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): to count in the collection. Can be an empty document to count all documents. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: See list of options above. @@ -2144,7 +2144,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param indexes: A list of :class:`~pymongo.operations.IndexModel` instances. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the createIndexes @@ -2153,7 +2153,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): - .. note:: The :attr:`~pymongo.collection.AsyncCollection.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.collection.AsyncCollection.write_concern` of this collection is automatically applied to this operation. .. versionchanged:: 3.6 @@ -2181,7 +2181,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param indexes: A list of :class:`~pymongo.operations.IndexModel` instances. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param kwargs: optional arguments to the createIndexes command (like maxTimeMS) can be passed as keyword arguments. """ @@ -2290,13 +2290,13 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): option is silently ignored by the server and unique index builds using the option will fail if a duplicate value is detected. - .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.collection.AsyncCollection.write_concern` of this collection is automatically applied to this operation. :param keys: a single key or a list of (key, direction) pairs specifying the index to create :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: any additional index creation @@ -2347,13 +2347,13 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): Raises OperationFailure on an error. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the createIndexes command (like maxTimeMS) can be passed as keyword arguments. - .. note:: The :attr:`~pymongo.collection.AsyncCollection.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.collection.AsyncCollection.write_concern` of this collection is automatically applied to this operation. .. versionchanged:: 3.6 @@ -2394,7 +2394,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param index_or_name: index (or name of index) to drop :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the createIndexes @@ -2402,7 +2402,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): - .. note:: The :attr:`~pymongo.collection.AsyncCollection.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.collection.AsyncCollection.write_concern` of this collection is automatically applied to this operation. @@ -2453,17 +2453,17 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): ) -> AsyncCommandCursor[MutableMapping[str, Any]]: """Get a cursor over the index documents for this collection. - >>> async for index in db.test.list_indexes(): + >>> async for index in await db.test.list_indexes(): ... print(index) ... SON([('v', 2), ('key', SON([('_id', 1)])), ('name', '_id_')]) :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. - :return: An instance of :class:`~pymongo.command_cursor.AsyncCommandCursor`. + :return: An instance of :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor`. .. versionchanged:: 4.1 Added ``comment`` parameter. @@ -2541,14 +2541,14 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): ``"name"`` keys, which are cleaned. Example output might look like this: - >>> db.test.create_index("x", unique=True) + >>> await db.test.create_index("x", unique=True) 'x_1' - >>> db.test.index_information() + >>> await db.test.index_information() {'_id_': {'key': [('_id', 1)]}, 'x_1': {'unique': True, 'key': [('x', 1)]}} :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. @@ -2579,11 +2579,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): for. Only indexes with matching index names will be returned. If not given, all search indexes for the current collection will be returned. - :param session: a :class:`~pymongo.client_session.AsyncClientSession`. + :param session: a :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. - :return: A :class:`~pymongo.command_cursor.AsyncCommandCursor` over the result + :return: A :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor` over the result set. .. note:: requires a MongoDB server version 7.0+ Atlas cluster. @@ -2633,7 +2633,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): instance or a dictionary with a model "definition" and optional "name". :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the createSearchIndexes @@ -2659,7 +2659,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): """Create multiple search indexes for the current collection. :param models: A list of :class:`~pymongo.operations.SearchIndexModel` instances. - :param session: a :class:`~pymongo.client_session.AsyncClientSession`. + :param session: a :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the createSearchIndexes @@ -2716,7 +2716,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param name: The name of the search index to be deleted. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the dropSearchIndexes @@ -2752,7 +2752,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param name: The name of the search index to be updated. :param definition: The new search index definition. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: optional arguments to the updateSearchIndexes @@ -2783,12 +2783,12 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): """Get the options set on this collection. Returns a dictionary of options and their values - see - :meth:`~pymongo.database.AsyncDatabase.create_collection` for more + :meth:`~pymongo.asynchronous.database.AsyncDatabase.create_collection` for more information on the possible options. Returns an empty dictionary if the collection has not been created yet. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. @@ -2874,12 +2874,12 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): instead. An example is included in the :ref:`aggregate-examples` documentation. - .. note:: The :attr:`~pymongo.collection.AsyncCollection.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.collection.AsyncCollection.write_concern` of this collection is automatically applied to this operation. :param pipeline: a list of aggregation pipeline stages :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: A dict of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -2905,7 +2905,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :class:`~pymongo.collation.Collation`. - :return: A :class:`~pymongo.command_cursor.AsyncCommandCursor` over the result + :return: A :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor` over the result set. .. versionchanged:: 4.1 @@ -2928,7 +2928,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. Support the `collation` option. .. versionchanged:: 3.0 - The :meth:`aggregate` method always returns a CommandCursor. The + The :meth:`aggregate` method always returns an AsyncCommandCursor. The pipeline argument must be a list. .. seealso:: :doc:`/examples/aggregation` @@ -2958,7 +2958,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): """Perform an aggregation and retrieve batches of raw BSON. Similar to the :meth:`aggregate` method but returns a - :class:`~pymongo.cursor.AsyncRawBatchCursor`. + :class:`~pymongo.asynchronous.cursor.AsyncRawBatchCursor`. This example demonstrates how to work with raw batches, but in practice raw batches should be passed to an external library that can decode @@ -3014,14 +3014,14 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param new_name: new name for this collection :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: additional arguments to the rename command may be passed as keyword arguments to this helper method (i.e. ``dropTarget=True``) - .. note:: The :attr:`~pymongo.collection.AsyncCollection.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.collection.AsyncCollection.write_concern` of this collection is automatically applied to this operation. .. versionchanged:: 3.6 @@ -3083,14 +3083,14 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :class:`~pymongo.collation.Collation`. The :meth:`distinct` method obeys the :attr:`read_preference` of - this :class:`Collection`. + this :class:`AsyncCollection`. :param key: name of the field for which we want to get the distinct values :param filter: A query document that specifies the documents from which to retrieve the distinct values. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: See list of options above. @@ -3261,11 +3261,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): match the query, they are sorted and the first is deleted. :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as - passed to :meth:`~pymongo.collection.AsyncCollection.create_index` + passed to :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -3287,7 +3287,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this - :class:`~pymongo.collection.AsyncCollection` when connected to MongoDB >= + :class:`~pymongo.asynchronous.collection.AsyncCollection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. @@ -3359,11 +3359,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -3387,7 +3387,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this - :class:`~pymongo.collection.AsyncCollection` when connected to MongoDB >= + :class:`~pymongo.asynchronous.collection.AsyncCollection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. @@ -3461,7 +3461,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): The *upsert* option can be used to create the document if it doesn't already exist. - >>> await db.example.delete_many({}).deleted_count + >>> (await db.example.delete_many({})).deleted_count 1 >>> await db.example.find_one_and_update( ... {'_id': 'userid'}, @@ -3506,11 +3506,11 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): :param hint: An index to use to support the query predicate specified either by its string name, or in the same format as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` (e.g. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param let: Map of parameter names and values. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an @@ -3534,7 +3534,7 @@ class AsyncCollection(common.BaseObject, Generic[_DocumentType]): .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this - :class:`~pymongo.collection.AsyncCollection` when connected to MongoDB >= + :class:`~pymongo.asynchronous.collection.AsyncCollection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. diff --git a/pymongo/asynchronous/command_cursor.py b/pymongo/asynchronous/command_cursor.py index 9816ccaaf..b28f983b1 100644 --- a/pymongo/asynchronous/command_cursor.py +++ b/pymongo/asynchronous/command_cursor.py @@ -189,7 +189,7 @@ class AsyncCommandCursor(Generic[_DocumentType]): @property def session(self) -> Optional[AsyncClientSession]: - """The cursor's :class:`~pymongo.client_session.AsyncClientSession`, or None. + """The cursor's :class:`~pymongo.asynchronous.client_session.AsyncClientSession`, or None. .. versionadded:: 3.6 """ @@ -416,7 +416,7 @@ class AsyncRawBatchCommandCursor(AsyncCommandCursor[_DocumentType]): """Create a new cursor / iterator over raw batches of BSON data. Should not be called directly by application developers - - see :meth:`~pymongo.collection.AsyncCollection.aggregate_raw_batches` + see :meth:`~pymongo.asynchronous.collection.AsyncCollection.aggregate_raw_batches` instead. .. seealso:: The MongoDB documentation on `cursors `_. diff --git a/pymongo/asynchronous/cursor.py b/pymongo/asynchronous/cursor.py index fa4431622..8421667be 100644 --- a/pymongo/asynchronous/cursor.py +++ b/pymongo/asynchronous/cursor.py @@ -123,7 +123,7 @@ class AsyncCursor(Generic[_DocumentType]): """Create a new cursor. Should not be called directly by application developers - see - :meth:`~pymongo.collection.AsyncCollection.find` instead. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find` instead. .. seealso:: The MongoDB documentation on `cursors `_. """ @@ -256,7 +256,7 @@ class AsyncCursor(Generic[_DocumentType]): @property def collection(self) -> AsyncCollection[_DocumentType]: - """The :class:`~pymongo.collection.AsyncCollection` that this + """The :class:`~pymongo.asynchronous.collection.AsyncCollection` that this :class:`AsyncCursor` is iterating. """ return self._collection @@ -322,7 +322,7 @@ class AsyncCursor(Generic[_DocumentType]): return base def _clone_base(self, session: Optional[AsyncClientSession]) -> AsyncCursor: - """Creates an empty Cursor object for information to be copied into.""" + """Creates an empty AsyncCursor object for information to be copied into.""" return self.__class__(self._collection, session=session) def _query_spec(self) -> Mapping[str, Any]: @@ -527,7 +527,7 @@ class AsyncCursor(Generic[_DocumentType]): def max_await_time_ms(self, max_await_time_ms: Optional[int]) -> AsyncCursor[_DocumentType]: """Specifies a time limit for a getMore operation on a - :attr:`~pymongo.cursor_shared.CursorType.TAILABLE_AWAIT` cursor. For all other + :attr:`~pymongo.cursor.CursorType.TAILABLE_AWAIT` cursor. For all other types of cursor max_await_time_ms is ignored. Raises :exc:`TypeError` if `max_await_time_ms` is not an integer or @@ -606,12 +606,12 @@ class AsyncCursor(Generic[_DocumentType]): self._empty = False if isinstance(index, slice): if index.step is not None: - raise IndexError("Cursor instances do not support slice steps") + raise IndexError("AsyncCursor instances do not support slice steps") skip = 0 if index.start is not None: if index.start < 0: - raise IndexError("Cursor instances do not support negative indices") + raise IndexError("AsyncCursor instances do not support negative indices") skip = index.start if index.stop is not None: @@ -631,15 +631,15 @@ class AsyncCursor(Generic[_DocumentType]): if isinstance(index, int): if index < 0: - raise IndexError("Cursor instances do not support negative indices") + raise IndexError("AsyncCursor instances do not support negative indices") clone = self.clone() clone.skip(index + self._skip) clone.limit(-1) # use a hard limit clone._query_flags &= ~CursorType.TAILABLE_AWAIT # PYTHON-1371 for doc in clone: # type: ignore[attr-defined] return doc - raise IndexError("no such item for Cursor instance") - raise TypeError("index %r cannot be applied to Cursor instances" % index) + raise IndexError("no such item for AsyncCursor instance") + raise TypeError("index %r cannot be applied to AsyncCursor instances" % index) else: raise IndexError("AsyncCursor does not support indexing") @@ -727,7 +727,7 @@ class AsyncCursor(Generic[_DocumentType]): Text search results can be sorted by relevance:: - cursor = await db.test.find( + cursor = db.test.find( {'$text': {'$search': 'some words'}}, {'score': {'$meta': 'textScore'}}) @@ -761,7 +761,7 @@ class AsyncCursor(Generic[_DocumentType]): `explain command `_, ``allPlansExecution``. To use a different verbosity use - :meth:`~pymongo.database.AsyncDatabase.command` to run the explain + :meth:`~pymongo.asynchronous.database.AsyncDatabase.command` to run the explain command directly. .. seealso:: The MongoDB documentation on `explain `_. @@ -796,7 +796,7 @@ class AsyncCursor(Generic[_DocumentType]): already been used. `index` should be an index as passed to - :meth:`~pymongo.collection.AsyncCollection.create_index` + :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` (e.g. ``[('field', ASCENDING)]``) or the name of the index. If `index` is ``None`` any existing hint for this query is cleared. The last hint applied to this cursor takes precedence @@ -838,7 +838,7 @@ class AsyncCursor(Generic[_DocumentType]): Raises :class:`TypeError` if `code` is not an instance of :class:`str`. Raises :class:`~pymongo.errors.InvalidOperation` if this - :class:`Cursor` has already been used. Only the last call to + :class:`AsyncCursor` has already been used. Only the last call to :meth:`where` applied to a :class:`AsyncCursor` has any effect. .. note:: MongoDB 4.4 drops support for :class:`~bson.code.Code` @@ -936,7 +936,7 @@ class AsyncCursor(Generic[_DocumentType]): @property def session(self) -> Optional[AsyncClientSession]: - """The cursor's :class:`~pymongo.client_session.AsyncClientSession`, or None. + """The cursor's :class:`~pymongo.asynchronous.client_session.AsyncClientSession`, or None. .. versionadded:: 3.6 """ @@ -1063,13 +1063,13 @@ class AsyncCursor(Generic[_DocumentType]): :class:`str`. The :meth:`distinct` method obeys the - :attr:`~pymongo.collection.AsyncCollection.read_preference` of the - :class:`~pymongo.collection.AsyncCollection` instance on which - :meth:`~pymongo.collection.AsyncCollection.find` was called. + :attr:`~pymongo.asynchronous.collection.AsyncCollection.read_preference` of the + :class:`~pymongo.asynchronous.collection.AsyncCollection` instance on which + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find` was called. :param key: name of key for which we want to get the distinct values - .. seealso:: :meth:`pymongo.collection.AsyncCollection.distinct` + .. seealso:: :meth:`pymongo.asynchronous.collection.AsyncCollection.distinct` """ options: dict[str, Any] = {} if self._spec: @@ -1111,7 +1111,7 @@ class AsyncCursor(Generic[_DocumentType]): await self.close() # If this is a tailable cursor the error is likely # due to capped collection roll over. Setting - # self._killed to True ensures Cursor.alive will be + # self._killed to True ensures AsyncCursor.alive will be # False. No need to re-raise. if ( exc.code in _CURSOR_CLOSED_ERRORS @@ -1316,7 +1316,7 @@ class AsyncRawBatchCursor(AsyncCursor, Generic[_DocumentType]): """Create a new cursor / iterator over raw batches of BSON data. Should not be called directly by application developers - - see :meth:`~pymongo.collection.AsyncCollection.find_raw_batches` + see :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_raw_batches` instead. .. seealso:: The MongoDB documentation on `cursors `_. diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 4bdf0ff51..b61d58183 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -74,7 +74,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): :class:`str`. Raises :class:`~pymongo.errors.InvalidName` if `name` is not a valid database name. - :param client: A :class:`~pymongo.mongo_client.AsyncMongoClient` instance. + :param client: A :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` instance. :param name: The database name. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the @@ -102,8 +102,8 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. - :class:`~pymongo.database.AsyncDatabase` no longer returns an instance - of :class:`~pymongo.collection.AsyncCollection` for attribute names + :class:`~pymongo.asynchronous.database.AsyncDatabase` no longer returns an instance + of :class:`~pymongo.asynchronous.collection.AsyncCollection` for attribute names with leading underscores. You must use dict-style lookups instead:: db['__my_collection__'] @@ -230,10 +230,10 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): write_concern: Optional[WriteConcern] = None, read_concern: Optional[ReadConcern] = None, ) -> AsyncCollection[_DocumentType]: - """Get a :class:`~pymongo.collection.AsyncCollection` with the given name + """Get a :class:`~pymongo.asynchronous.collection.AsyncCollection` with the given name and options. - Useful for creating a :class:`~pymongo.collection.AsyncCollection` with + Useful for creating a :class:`~pymongo.asynchronous.collection.AsyncCollection` with different codec options, read preference, and/or write concern from this :class:`AsyncDatabase`. @@ -307,7 +307,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): __iter__ = None def __next__(self) -> NoReturn: - raise TypeError("'Database' object is not iterable") + raise TypeError("'AsyncDatabase' object is not iterable") next = __next__ @@ -364,7 +364,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): async for insert_change in stream: print(insert_change) except pymongo.errors.PyMongoError: - # The ChangeStream encountered an unrecoverable error or the + # The AsyncChangeStream encountered an unrecoverable error or the # resume attempt failed to recreate the cursor. logging.error("...") @@ -401,7 +401,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): the specified :class:`~bson.timestamp.Timestamp`. Requires MongoDB >= 4.0. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param start_after: The same as `resume_after` except that `start_after` can resume notifications after an invalidate event. This option and `resume_after` are mutually exclusive. @@ -461,7 +461,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): check_exists: Optional[bool] = True, **kwargs: Any, ) -> AsyncCollection[_DocumentType]: - """Create a new :class:`~pymongo.collection.AsyncCollection` in this + """Create a new :class:`~pymongo.asynchronous.collection.AsyncCollection` in this database. Normally collection creation is automatic. This method should @@ -488,7 +488,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): :param collation: An instance of :class:`~pymongo.collation.Collation`. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param check_exists: if True (the default), send a listCollections command to check if the collection already exists before creation. :param kwargs: additional keyword arguments will @@ -621,19 +621,19 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): print(operation) The :meth:`aggregate` method obeys the :attr:`read_preference` of this - :class:`Database`, except when ``$out`` or ``$merge`` are used, in + :class:`AsyncDatabase`, except when ``$out`` or ``$merge`` are used, in which case :attr:`~pymongo.read_preferences.ReadPreference.PRIMARY` is used. .. note:: This method does not support the 'explain' option. Please - use :meth:`~pymongo.database.Database.command` instead. + use :meth:`~pymongo.asynchronous.database.AsyncDatabase.command` instead. - .. note:: The :attr:`~pymongo.database.AsyncDatabase.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.database.AsyncDatabase.write_concern` of this collection is automatically applied to this operation. :param pipeline: a list of aggregation pipeline stages :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param kwargs: extra `aggregate command`_ parameters. All optional `aggregate command`_ parameters should be passed as @@ -656,7 +656,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): aggregate expression context (e.g. ``"$$var"``). This option is only supported on MongoDB >= 5.0. - :return: A :class:`~pymongo.command_cursor.AsyncCommandCursor` over the result + :return: A :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor` over the result set. .. versionadded:: 3.9 @@ -851,7 +851,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): :param codec_options: A :class:`~bson.codec_options.CodecOptions` instance. :param session: A - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: additional keyword arguments will @@ -952,7 +952,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): :param codec_options: A :class:`~bson.codec_options.CodecOptions` instance. :param session: A - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to future getMores for this command. :param max_await_time_ms: The number of ms to wait for more data on future getMores for this command. @@ -1082,7 +1082,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): """Get a cursor over the collections of this database. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param filter: A query document to filter the list of collections returned from the listCollections command. :param comment: A user-provided comment to attach to this @@ -1094,7 +1094,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): options differ by server version. - :return: An instance of :class:`~pymongo.command_cursor.AsyncCommandCursor`. + :return: An instance of :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor`. .. versionadded:: 3.6 """ @@ -1128,7 +1128,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): """Get a cursor over the collections of this database. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param filter: A query document to filter the list of collections returned from the listCollections command. :param comment: A user-provided comment to attach to this @@ -1140,7 +1140,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): options differ by server version. - :return: An instance of :class:`~pymongo.command_cursor.AsyncCommandCursor`. + :return: An instance of :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor`. .. versionadded:: 3.6 """ @@ -1186,7 +1186,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): db.list_collection_names(filter=filter) :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param filter: A query document to filter the list of collections returned from the listCollections command. :param comment: A user-provided comment to attach to this @@ -1235,7 +1235,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): :param name_or_collection: the name of a collection to drop or the collection object itself :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param encrypted_fields: **(BETA)** Document that describes the encrypted fields for @@ -1261,7 +1261,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): } - .. note:: The :attr:`~pymongo.database.Database.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.database.AsyncDatabase.write_concern` of this database is automatically applied to this operation. .. versionchanged:: 4.2 @@ -1325,7 +1325,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): of the structure of the collection and the individual documents. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param background: A boolean flag that determines whether the command runs in the background. Requires MongoDB 4.4+. :param comment: A user-provided comment to attach to this @@ -1347,7 +1347,7 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): name = name.name if not isinstance(name, str): - raise TypeError("name_or_collection must be an instance of str or Collection") + raise TypeError("name_or_collection must be an instance of str or AsyncCollection") cmd = {"validate": name, "scandata": scandata, "full": full} if comment is not None: cmd["comment"] = comment @@ -1400,12 +1400,12 @@ class AsyncDatabase(common.BaseObject, Generic[_DocumentType]): :param dbref: the reference :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: any additional keyword arguments are the same as the arguments to - :meth:`~pymongo.collection.AsyncCollection.find`. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find`. .. versionchanged:: 4.1 diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index b1a36eefa..f8c03b21c 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -673,7 +673,7 @@ class ClientEncryption(Generic[_DocumentType]): All optional `create collection command`_ parameters should be passed as keyword arguments to this method. - See the documentation for :meth:`~pymongo.database.AsyncDatabase.create_collection` for all valid options. + See the documentation for :meth:`~pymongo.asynchronous.database.AsyncDatabase.create_collection` for all valid options. :raises: - :class:`~pymongo.errors.EncryptedCollectionError`: When either data-key creation or creating the collection fails. @@ -978,7 +978,7 @@ class ClientEncryption(Generic[_DocumentType]): def get_keys(self) -> AsyncCursor[RawBSONDocument]: """Get all of the data keys. - :return: An instance of :class:`~pymongo.cursor.Cursor` over the data key + :return: An instance of :class:`~pymongo.asynchronous.cursor.AsyncCursor` over the data key documents. .. versionadded:: 4.2 diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index a320249b4..90e40978a 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -17,18 +17,18 @@ .. seealso:: :doc:`/examples/high_availability` for examples of connecting to replica sets or sets of mongos servers. -To get a :class:`~pymongo.database.Database` instance from a -:class:`MongoClient` use either dictionary-style or attribute-style +To get a :class:`~pymongo.asynchronous.database.AsyncDatabase` instance from a +:class:`AsyncMongoClient` use either dictionary-style or attribute-style access: .. doctest:: - >>> from pymongo import MongoClient - >>> c = MongoClient() + >>> from pymongo import AsyncMongoClient + >>> c = AsyncMongoClient() >>> c.test_database - Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'test_database') + AsyncDatabase(AsyncMongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'test_database') >>> c["test-database"] - Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'test-database') + AsyncDatabase(AsyncMongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'test-database') """ from __future__ import annotations @@ -61,6 +61,7 @@ from bson.timestamp import Timestamp from pymongo import _csot, common, helpers_shared, uri_parser from pymongo.asynchronous import client_session, database, periodic_executor from pymongo.asynchronous.change_stream import AsyncChangeStream, AsyncClusterChangeStream +from pymongo.asynchronous.client_bulk import _AsyncClientBulk from pymongo.asynchronous.client_session import _EmptyServerSession from pymongo.asynchronous.command_cursor import AsyncCommandCursor from pymongo.asynchronous.settings import TopologySettings @@ -69,6 +70,7 @@ from pymongo.client_options import ClientOptions from pymongo.errors import ( AutoReconnect, BulkWriteError, + ClientBulkWriteException, ConfigurationError, ConnectionFailure, InvalidOperation, @@ -83,8 +85,17 @@ from pymongo.lock import _HAS_REGISTER_AT_FORK, _ALock, _create_lock, _release_l from pymongo.logger import _CLIENT_LOGGER, _log_or_warn from pymongo.message import _CursorAddress, _GetMore, _Query from pymongo.monitoring import ConnectionClosedReason -from pymongo.operations import _Op +from pymongo.operations import ( + DeleteMany, + DeleteOne, + InsertOne, + ReplaceOne, + UpdateMany, + UpdateOne, + _Op, +) from pymongo.read_preferences import ReadPreference, _ServerMode +from pymongo.results import ClientBulkWriteResult from pymongo.server_selectors import writable_server_selector from pymongo.server_type import SERVER_TYPE from pymongo.topology_description import TOPOLOGY_TYPE, TopologyDescription @@ -130,6 +141,15 @@ _ReadCall = Callable[ _IS_SYNC = False +_WriteOp = Union[ + InsertOne, + DeleteOne, + DeleteMany, + ReplaceOne, + UpdateOne, + UpdateMany, +] + class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): HOST = "localhost" @@ -175,18 +195,18 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): uri = "mongodb://%s:%s@%s" % ( quote_plus(user), quote_plus(password), host) - client = MongoClient(uri) + client = AsyncMongoClient(uri) Unix domain sockets are also supported. The socket path must be percent encoded in the URI:: uri = "mongodb://%s:%s@%s" % ( quote_plus(user), quote_plus(password), quote_plus(socket_path)) - client = MongoClient(uri) + client = AsyncMongoClient(uri) But not when passed as a simple hostname:: - client = MongoClient('/tmp/mongodb-27017.sock') + client = AsyncMongoClient('/tmp/mongodb-27017.sock') Starting with version 3.6, PyMongo supports mongodb+srv:// URIs. The URI must include one, and only one, hostname. The hostname will be @@ -202,10 +222,10 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): for more details. Note that the use of SRV URIs implicitly enables TLS support. Pass tls=false in the URI to override. - .. note:: MongoClient creation will block waiting for answers from + .. note:: AsyncMongoClient creation will block waiting for answers from DNS when mongodb+srv:// URIs are used. - .. note:: Starting with version 3.0 the :class:`MongoClient` + .. note:: Starting with version 3.0 the :class:`AsyncMongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are @@ -216,7 +236,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): like this:: from pymongo.errors import ConnectionFailure - client = MongoClient() + client = AsyncMongoClient() try: # The ping command is cheap and does not require auth. client.admin.command('ping') @@ -242,7 +262,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): documents returned from queries on this client :param tz_aware: if ``True``, :class:`~datetime.datetime` instances returned as values - in a document by this :class:`MongoClient` will be timezone + in a document by this :class:`AsyncMongoClient` will be timezone aware (otherwise they will be naive) :param connect: **Not supported by AsyncMongoClient**. :param type_registry: instance of @@ -314,7 +334,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): - `serverMonitoringMode`: (optional) The server monitoring mode to use. Valid values are the strings: "auto", "stream", "poll". Defaults to "auto". - `appname`: (string or None) The name of the application that - created this MongoClient instance. The server will log this value + created this AsyncMongoClient instance. The server will log this value upon establishing each connection. It is also recorded in the slow query log and profile collections. - `driver`: (pair or None) A driver implemented on top of PyMongo can @@ -324,47 +344,47 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): - `event_listeners`: a list or tuple of event listeners. See :mod:`~pymongo.monitoring` for details. - `retryWrites`: (boolean) Whether supported write operations - executed within this MongoClient will be retried once after a + executed within this AsyncMongoClient will be retried once after a network error. Defaults to ``True``. The supported write operations are: - - :meth:`~pymongo.collection.Collection.bulk_write`, as long as - :class:`~pymongo.operations.UpdateMany` or - :class:`~pymongo.operations.DeleteMany` are not included. - - :meth:`~pymongo.collection.Collection.delete_one` - - :meth:`~pymongo.collection.Collection.insert_one` - - :meth:`~pymongo.collection.Collection.insert_many` - - :meth:`~pymongo.collection.Collection.replace_one` - - :meth:`~pymongo.collection.Collection.update_one` - - :meth:`~pymongo.collection.Collection.find_one_and_delete` - - :meth:`~pymongo.collection.Collection.find_one_and_replace` - - :meth:`~pymongo.collection.Collection.find_one_and_update` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, as long as + :class:`~pymongo.asynchronous.operations.UpdateMany` or + :class:`~pymongo.asynchronous.operations.DeleteMany` are not included. + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.delete_one` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.insert_one` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.insert_many` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.replace_one` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.update_one` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one_and_delete` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one_and_replace` + - :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one_and_update` Unsupported write operations include, but are not limited to, - :meth:`~pymongo.collection.Collection.aggregate` using the ``$out`` + :meth:`~pymongo.asynchronous.collection.AsyncCollection.aggregate` using the ``$out`` pipeline operator and any operation with an unacknowledged write concern (e.g. {w: 0})). See https://github.com/mongodb/specifications/blob/master/source/retryable-writes/retryable-writes.rst - `retryReads`: (boolean) Whether supported read operations - executed within this MongoClient will be retried once after a + executed within this AsyncMongoClient will be retried once after a network error. Defaults to ``True``. The supported read operations are: - :meth:`~pymongo.collection.Collection.find`, - :meth:`~pymongo.collection.Collection.find_one`, - :meth:`~pymongo.collection.Collection.aggregate` without ``$out``, - :meth:`~pymongo.collection.Collection.distinct`, - :meth:`~pymongo.collection.Collection.count`, - :meth:`~pymongo.collection.Collection.estimated_document_count`, - :meth:`~pymongo.collection.Collection.count_documents`, - :meth:`pymongo.collection.Collection.watch`, - :meth:`~pymongo.collection.Collection.list_indexes`, - :meth:`pymongo.database.Database.watch`, - :meth:`~pymongo.database.Database.list_collections`, - :meth:`pymongo.mongo_client.MongoClient.watch`, - and :meth:`~pymongo.mongo_client.MongoClient.list_databases`. + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find`, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.find_one`, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.aggregate` without ``$out``, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.distinct`, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.count`, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.estimated_document_count`, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.count_documents`, + :meth:`pymongo.asynchronous.collection.AsyncCollection.watch`, + :meth:`~pymongo.asynchronous.collection.AsyncCollection.list_indexes`, + :meth:`pymongo.asynchronous.database.AsyncDatabase.watch`, + :meth:`~pymongo.asynchronous.database.AsyncDatabase.list_collections`, + :meth:`pymongo.asynchronous.mongo_client.AsyncMongoClient.watch`, + and :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.list_databases`. Unsupported read operations include, but are not limited to - :meth:`~pymongo.database.Database.command` and any getMore + :meth:`~pymongo.asynchronous.database.AsyncDatabase.command` and any getMore operation on a cursor. Enabling retryable reads makes applications more resilient to @@ -403,7 +423,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): - `srvServiceName`: (string) The SRV service name to use for "mongodb+srv://" URIs. Defaults to "mongodb". Use it like so:: - MongoClient("mongodb+srv://example.com/?srvServiceName=customname") + AsyncMongoClient("mongodb+srv://example.com/?srvServiceName=customname") - `srvMaxHosts`: (int) limits the number of mongos-like hosts a client will connect to. More specifically, when a "mongodb+srv://" connection string resolves to more than srvMaxHosts number of hosts, the client will randomly @@ -470,7 +490,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): this example, both the space and slash special characters are passed as-is:: - MongoClient(username="user name", password="pass/word") + AsyncMongoClient(username="user name", password="pass/word") - `authSource`: The database to authenticate on. Defaults to the database specified in the URI, if provided, or to "admin". @@ -549,9 +569,9 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): configures this client to automatically encrypt collection commands and automatically decrypt results. See :ref:`automatic-client-side-encryption` for an example. - If a :class:`MongoClient` is configured with + If a :class:`AsyncMongoClient` is configured with ``auto_encryption_opts`` and a non-None ``maxPoolSize``, a - separate internal ``MongoClient`` is created if any of the + separate internal ``AsyncMongoClient`` is created if any of the following are true: - A ``key_vault_client`` is not passed to @@ -640,14 +660,14 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): ``socketKeepAlive`` now defaults to ``True``. .. versionchanged:: 3.0 - :class:`~pymongo.mongo_client.MongoClient` is now the one and only + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into - :class:`~pymongo.mongo_client.MongoReplicaSetClient`: it can connect + :class:`~pymongo.asynchronous.mongo_client.MongoReplicaSetClient`: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. - The :class:`~pymongo.mongo_client.MongoClient` constructor no + The :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :class:`~pymongo.errors.ConfigurationError` @@ -659,10 +679,10 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): provides meaningful information; even if the client is disconnected, it may discover a server in time to fulfill the next operation. - In PyMongo 2.x, :class:`~pymongo.MongoClient` accepted a list of + In PyMongo 2.x, :class:`~pymongo.asynchronous.AsyncMongoClient` accepted a list of standalone MongoDB servers and used the first it could connect to:: - MongoClient(['host1.com:27017', 'host2.com:27017']) + AsyncMongoClient(['host1.com:27017', 'host2.com:27017']) A list of multiple standalones is no longer supported; if multiple servers are listed they must be members of the same replica set, or @@ -685,11 +705,11 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): The ``copy_database`` method is removed, see the :doc:`copy_database examples ` for alternatives. - The :meth:`MongoClient.disconnect` method is removed; it was a - synonym for :meth:`~pymongo.MongoClient.close`. + The :meth:`AsyncMongoClient.disconnect` method is removed; it was a + synonym for :meth:`~pymongo.asynchronous.AsyncMongoClient.close`. - :class:`~pymongo.mongo_client.MongoClient` no longer returns an - instance of :class:`~pymongo.database.Database` for attribute names + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` no longer returns an + instance of :class:`~pymongo.asynchronous.database.AsyncDatabase` for attribute names with leading underscores. You must use dict-style lookups instead:: client['__my_database__'] @@ -924,7 +944,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): Performs an aggregation with an implicit initial ``$changeStream`` stage and returns a - :class:`~pymongo.change_stream.ClusterChangeStream` cursor which + :class:`~pymongo.asynchronous.change_stream.AsyncClusterChangeStream` cursor which iterates over changes on all databases on this cluster. Introduced in MongoDB 4.0. @@ -935,10 +955,10 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): for change in stream: print(change) - The :class:`~pymongo.change_stream.ClusterChangeStream` iterable + The :class:`~pymongo.asynchronous.change_stream.AsyncClusterChangeStream` iterable blocks until the next change document is returned or an error is raised. If the - :meth:`~pymongo.change_stream.ClusterChangeStream.next` method + :meth:`~pymongo.asynchronous.change_stream.AsyncClusterChangeStream.next` method encounters a network error when retrieving a batch from the server, it will automatically attempt to recreate the cursor such that no change events are missed. Any error encountered during the resume @@ -951,7 +971,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): for insert_change in stream: print(insert_change) except pymongo.errors.PyMongoError: - # The ChangeStream encountered an unrecoverable error or the + # The AsyncChangeStream encountered an unrecoverable error or the # resume attempt failed to recreate the cursor. logging.error("...") @@ -988,7 +1008,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): the specified :class:`~bson.timestamp.Timestamp`. Requires MongoDB >= 4.0. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param start_after: The same as `resume_after` except that `start_after` can resume notifications after an invalidate event. This option and `resume_after` are mutually exclusive. @@ -996,7 +1016,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): command. :param show_expanded_events: Include expanded events such as DDL events like `dropIndexes`. - :return: A :class:`~pymongo.change_stream.ClusterChangeStream` cursor. + :return: A :class:`~pymongo.asynchronous.change_stream.AsyncClusterChangeStream` cursor. .. versionchanged:: 4.3 Added `show_expanded_events` parameter. @@ -1062,9 +1082,9 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Set of all currently connected servers. .. warning:: When connected to a replica set the value of :attr:`nodes` - can change over time as :class:`MongoClient`'s view of the replica + can change over time as :class:`AsyncMongoClient`'s view of the replica set changes. :attr:`nodes` can also be an empty set when - :class:`MongoClient` is first instantiated and hasn't yet connected + :class:`AsyncMongoClient` is first instantiated and hasn't yet connected to any servers, or a network partition causes it to lose connection to all servers. """ @@ -1176,16 +1196,16 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Start a logical session. This method takes the same parameters as - :class:`~pymongo.client_session.SessionOptions`. See the - :mod:`~pymongo.client_session` module for details and examples. + :class:`~pymongo.asynchronous.client_session.SessionOptions`. See the + :mod:`~pymongo.asynchronous.client_session` module for details and examples. - A :class:`~pymongo.client_session.AsyncClientSession` may only be used with - the MongoClient that started it. :class:`AsyncClientSession` instances are + A :class:`~pymongo.asynchronous.client_session.AsyncClientSession` may only be used with + the AsyncMongoClient that started it. :class:`AsyncClientSession` instances are **not thread-safe or fork-safe**. They can only be used by one thread or process at a time. A single :class:`AsyncClientSession` cannot be used to run multiple operations concurrently. - :return: An instance of :class:`~pymongo.client_session.AsyncClientSession`. + :return: An instance of :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. .. versionadded:: 3.6 """ @@ -1237,7 +1257,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Get the database named in the MongoDB connection URI. >>> uri = 'mongodb://host/my_database' - >>> client = MongoClient(uri) + >>> client = AsyncMongoClient(uri) >>> db = client.get_default_database() >>> assert db.name == 'my_database' >>> db = client.get_database() @@ -1250,19 +1270,19 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): was provided in the URI. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the - default) the :attr:`codec_options` of this :class:`MongoClient` is + default) the :attr:`codec_options` of this :class:`AsyncMongoClient` is used. :param read_preference: The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this - :class:`MongoClient` is used. See :mod:`~pymongo.read_preferences` + :class:`AsyncMongoClient` is used. See :mod:`~pymongo.read_preferences` for options. :param write_concern: An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the - default) the :attr:`write_concern` of this :class:`MongoClient` is + default) the :attr:`write_concern` of this :class:`AsyncMongoClient` is used. :param read_concern: An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the - default) the :attr:`read_concern` of this :class:`MongoClient` is + default) the :attr:`read_concern` of this :class:`AsyncMongoClient` is used. :param comment: A user-provided comment to attach to this command. @@ -1294,12 +1314,12 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): write_concern: Optional[WriteConcern] = None, read_concern: Optional[ReadConcern] = None, ) -> database.AsyncDatabase[_DocumentType]: - """Get a :class:`~pymongo.database.Database` with the given name and + """Get a :class:`~pymongo.asynchronous.database.AsyncDatabase` with the given name and options. - Useful for creating a :class:`~pymongo.database.Database` with + Useful for creating a :class:`~pymongo.asynchronous.database.AsyncDatabase` with different codec options, read preference, and/or write concern from - this :class:`MongoClient`. + this :class:`AsyncMongoClient`. >>> client.read_preference Primary() @@ -1317,19 +1337,19 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): returned. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the - default) the :attr:`codec_options` of this :class:`MongoClient` is + default) the :attr:`codec_options` of this :class:`AsyncMongoClient` is used. :param read_preference: The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this - :class:`MongoClient` is used. See :mod:`~pymongo.read_preferences` + :class:`AsyncMongoClient` is used. See :mod:`~pymongo.read_preferences` for options. :param write_concern: An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the - default) the :attr:`write_concern` of this :class:`MongoClient` is + default) the :attr:`write_concern` of this :class:`AsyncMongoClient` is used. :param read_concern: An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the - default) the :attr:`read_concern` of this :class:`MongoClient` is + default) the :attr:`read_concern` of this :class:`AsyncMongoClient` is used. .. versionchanged:: 3.5 @@ -1346,7 +1366,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): ) def _database_default_options(self, name: str) -> database.AsyncDatabase: - """Get a Database instance with the default settings.""" + """Get a AsyncDatabase instance with the default settings.""" return self.get_database( name, codec_options=DEFAULT_CODEC_OPTIONS, @@ -1426,7 +1446,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): `replicaSet` option. .. versionadded:: 3.0 - MongoClient gained this property in version 3.0. + AsyncMongoClient gained this property in version 3.0. """ return await self._topology.get_primary() # type: ignore[return-value] @@ -1439,7 +1459,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): client was created without the `replicaSet` option. .. versionadded:: 3.0 - MongoClient gained this property in version 3.0. + AsyncMongoClient gained this property in version 3.0. """ return await self._topology.get_secondaries() @@ -1522,7 +1542,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): await self._encrypter.close() async def _get_topology(self) -> Topology: - """Get the internal :class:`~pymongo.topology.Topology` object. + """Get the internal :class:`~pymongo.asynchronous.topology.Topology` object. If this client was created with "connect=False", calling _get_topology launches the connection process in the background. @@ -1720,7 +1740,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): retryable: bool, func: _WriteCall[T], session: Optional[AsyncClientSession], - bulk: Optional[_AsyncBulk], + bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]], operation: str, operation_id: Optional[int] = None, ) -> T: @@ -1750,7 +1770,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): self, func: _WriteCall[T] | _ReadCall[T], session: Optional[AsyncClientSession], - bulk: Optional[_AsyncBulk], + bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]], operation: str, is_read: bool = False, address: Optional[_Address] = None, @@ -1833,7 +1853,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): func: _WriteCall[T], session: Optional[AsyncClientSession], operation: str, - bulk: Optional[_AsyncBulk] = None, + bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]] = None, operation_id: Optional[int] = None, ) -> T: """Execute an operation with consecutive retries if possible @@ -2074,7 +2094,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Get information about the MongoDB server we're connected to. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. @@ -2117,7 +2137,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Get a cursor over the databases of the connected server. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. :param kwargs: Optional parameters of the @@ -2127,7 +2147,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): options differ by server version. - :return: An instance of :class:`~pymongo.command_cursor.CommandCursor`. + :return: An instance of :class:`~pymongo.asynchronous.command_cursor.AsyncCommandCursor`. .. versionadded:: 3.6 """ @@ -2141,7 +2161,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Get a list of the names of all databases on the connected server. :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. @@ -2163,13 +2183,13 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): """Drop a database. Raises :class:`TypeError` if `name_or_database` is not an instance of - :class:`str` or :class:`~pymongo.database.Database`. + :class:`str` or :class:`~pymongo.asynchronous.database.AsyncDatabase`. :param name_or_database: the name of a database to drop, or a - :class:`~pymongo.database.Database` instance representing the + :class:`~pymongo.asynchronous.database.AsyncDatabase` instance representing the database to drop :param session: a - :class:`~pymongo.client_session.AsyncClientSession`. + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. :param comment: A user-provided comment to attach to this command. @@ -2179,7 +2199,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): .. versionchanged:: 3.6 Added ``session`` parameter. - .. note:: The :attr:`~pymongo.mongo_client.MongoClient.write_concern` of + .. note:: The :attr:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.write_concern` of this client is automatically applied to this operation. .. versionchanged:: 3.4 @@ -2192,7 +2212,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): name = name.name if not isinstance(name, str): - raise TypeError("name_or_database must be an instance of str or a Database") + raise TypeError("name_or_database must be an instance of str or a AsyncDatabase") async with await self._conn_for_writes(session, operation=_Op.DROP_DATABASE) as conn: await self[name]._command( @@ -2204,10 +2224,134 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): session=session, ) + @_csot.apply + async def bulk_write( + self, + models: Sequence[_WriteOp[_DocumentType]], + session: Optional[AsyncClientSession] = None, + ordered: bool = True, + verbose_results: bool = False, + bypass_document_validation: Optional[bool] = None, + comment: Optional[Any] = None, + let: Optional[Mapping] = None, + write_concern: Optional[WriteConcern] = None, + ) -> ClientBulkWriteResult: + """Send a batch of write operations, potentially across multiple namespaces, to the server. + + Requests are passed as a list of write operation instances ( + :class:`~pymongo.operations.InsertOne`, + :class:`~pymongo.operations.UpdateOne`, + :class:`~pymongo.operations.UpdateMany`, + :class:`~pymongo.operations.ReplaceOne`, + :class:`~pymongo.operations.DeleteOne`, or + :class:`~pymongo.operations.DeleteMany`). + + >>> async for doc in db.test.find({}): + ... print(doc) + ... + {'x': 1, '_id': ObjectId('54f62e60fba5226811f634ef')} + {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} + ... + >>> async for doc in db.coll.find({}): + ... print(doc) + ... + {'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')} + ... + >>> # DeleteMany, UpdateOne, and UpdateMany are also available. + >>> from pymongo import InsertOne, DeleteOne, ReplaceOne + >>> models = [InsertOne(namespace="db.test", document={'y': 1}), + ... DeleteOne(namespace="db.test", filter={'x': 1}), + ... InsertOne(namespace="db.coll", document={'y': 2}), + ... ReplaceOne(namespace="db.test", filter={'w': 1}, replacement={'z': 1}, upsert=True)] + >>> result = await client.bulk_write(models=models) + >>> result.inserted_count + 2 + >>> result.deleted_count + 1 + >>> result.modified_count + 0 + >>> result.upserted_ids + {3: ObjectId('54f62ee28891e756a6e1abd5')} + >>> async for doc in db.test.find({}): + ... print(doc) + ... + {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} + {'y': 1, '_id': ObjectId('54f62ee2fba5226811f634f1')} + {'z': 1, '_id': ObjectId('54f62ee28891e756a6e1abd5')} + ... + >>> async for doc in db.coll.find({}): + ... print(doc) + ... + {'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')} + {'y': 2, '_id': ObjectId('507f1f77bcf86cd799439012')} + + :param models: A list of write operation instances. + :param session: (optional) An instance of + :class:`~pymongo.asynchronous.client_session.AsyncClientSession`. + :param ordered: If ``True`` (the default), requests will be + performed on the server serially, in the order provided. If an error + occurs all remaining operations are aborted. If ``False``, requests + will be still performed on the server serially, in the order provided, + but all operations will be attempted even if any errors occur. + :param verbose_results: If ``True``, detailed results for each + successful operation will be included in the returned + :class:`~pymongo.results.ClientBulkWriteResult`. Default is ``False``. + :param bypass_document_validation: (optional) If ``True``, allows the + write to opt-out of document level validation. Default is ``False``. + :param comment: (optional) A user-provided comment to attach to this + command. + :param let: (optional) Map of parameter names and values. Values must be + constant or closed expressions that do not reference document + fields. Parameters can then be accessed as variables in an + aggregate expression context (e.g. "$$var"). + :param write_concern: (optional) The write concern to use for this bulk write. + + :return: An instance of :class:`~pymongo.results.ClientBulkWriteResult`. + + .. seealso:: :ref:`writes-and-ids` + + .. note:: requires MongoDB server version 8.0+. + + .. versionadded:: 4.9 + """ + if self._options.auto_encryption_opts: + raise InvalidOperation( + "MongoClient.bulk_write does not currently support automatic encryption" + ) + + if session and session.in_transaction: + # Inherit the transaction write concern. + if write_concern: + raise InvalidOperation("Cannot set write concern after starting a transaction") + write_concern = session._transaction.opts.write_concern # type: ignore[union-attr] + else: + # Inherit the client's write concern if none is provided. + if not write_concern: + write_concern = self.write_concern + + common.validate_list("models", models) + + blk = _AsyncClientBulk( + self, + write_concern=write_concern, # type: ignore[arg-type] + ordered=ordered, + bypass_document_validation=bypass_document_validation, + comment=comment, + let=let, + verbose_results=verbose_results, + ) + for model in models: + try: + model._add_to_client_bulk(blk) + except AttributeError: + raise TypeError(f"{model!r} is not a valid request") from None + + return await blk.execute(session, _Op.BULK_WRITE) + def _retryable_error_doc(exc: PyMongoError) -> Optional[Mapping[str, Any]]: """Return the server response from PyMongo exception or None.""" - if isinstance(exc, BulkWriteError): + if isinstance(exc, (BulkWriteError, ClientBulkWriteException)): # Check the last writeConcernError to determine if this # BulkWriteError is retryable. wces = exc.details["writeConcernErrors"] @@ -2242,10 +2386,14 @@ def _add_retryable_write_error(exc: PyMongoError, max_wire_version: int, is_mong # AsyncConnection errors are always retryable except NotPrimaryError and WaitQueueTimeoutError which is # handled above. - if isinstance(exc, ConnectionFailure) and not isinstance( - exc, (NotPrimaryError, WaitQueueTimeoutError) + if isinstance(exc, ClientBulkWriteException): + exc_to_check = exc.error + else: + exc_to_check = exc + if isinstance(exc_to_check, ConnectionFailure) and not isinstance( + exc_to_check, (NotPrimaryError, WaitQueueTimeoutError) ): - exc._add_error_label("RetryableWriteError") + exc_to_check._add_error_label("RetryableWriteError") class _MongoClientErrorHandler: @@ -2292,6 +2440,8 @@ class _MongoClientErrorHandler: return self.handled = True if self.session: + if isinstance(exc_val, ClientBulkWriteException): + exc_val = exc_val.error if isinstance(exc_val, ConnectionFailure): if self.session.in_transaction: exc_val._add_error_label("TransientTransactionError") @@ -2303,7 +2453,7 @@ class _MongoClientErrorHandler: ): await self.session._unpin() err_ctx = _ErrorContext( - exc_val, + exc_val, # type: ignore[arg-type] self.max_wire_version, self.sock_generation, self.completed_handshake, @@ -2330,7 +2480,7 @@ class _ClientConnectionRetryable(Generic[T]): self, mongo_client: AsyncMongoClient, func: _WriteCall[T] | _ReadCall[T], - bulk: Optional[_AsyncBulk], + bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]], operation: str, is_read: bool = False, session: Optional[AsyncClientSession] = None, @@ -2407,7 +2557,10 @@ class _ClientConnectionRetryable(Generic[T]): if not self._is_read: if not self._retryable: raise - retryable_write_error_exc = exc.has_error_label("RetryableWriteError") + if isinstance(exc, ClientBulkWriteException) and exc.error: + retryable_write_error_exc = exc.error.has_error_label("RetryableWriteError") + else: + retryable_write_error_exc = exc.has_error_label("RetryableWriteError") if retryable_write_error_exc: assert self._session await self._session._unpin() diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index a5f743512..d2ac8868e 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -48,6 +48,15 @@ def _sanitize(error: Exception) -> None: error.__cause__ = None +def _monotonic_duration(start: float) -> float: + """Return the duration since the given start time. + + Accounts for buggy platforms where time.monotonic() is not monotonic. + See PYTHON-4600. + """ + return max(0.0, time.monotonic() - start) + + class MonitorBase: def __init__(self, topology: Topology, name: str, interval: int, min_interval: float): """Base class to do periodic work on a background thread. @@ -247,7 +256,7 @@ class Monitor(MonitorBase): _sanitize(error) sd = self._server_description address = sd.address - duration = time.monotonic() - start + duration = _monotonic_duration(start) if self._publish: awaited = bool(self._stream and sd.is_server_type_known and sd.topology_version) assert self._listeners is not None @@ -317,7 +326,8 @@ class Monitor(MonitorBase): else: # New connection handshake or polling hello (MongoDB <4.4). response = await conn._hello(cluster_time, None, None) - return response, time.monotonic() - start + duration = _monotonic_duration(start) + return response, duration class SrvMonitor(MonitorBase): @@ -441,7 +451,7 @@ class _RttMonitor(MonitorBase): raise Exception("_RttMonitor closed") start = time.monotonic() await conn.hello() - return time.monotonic() - start + return _monotonic_duration(start) # Close monitors to cancel any in progress streaming checks before joining diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index 16cdd0eba..4e55db498 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -645,6 +645,7 @@ class Topology: :exc:`~.errors.InvalidOperation`. """ async with self._lock: + old_td = self._description for server in self._servers.values(): await server.close() @@ -664,9 +665,30 @@ class Topology: # Publish only after releasing the lock. if self._publish_tp: assert self._events is not None + self._description = TopologyDescription( + TOPOLOGY_TYPE.Unknown, + {}, + self._description.replica_set_name, + self._description.max_set_version, + self._description.max_election_id, + self._description._topology_settings, + ) + self._events.put( + ( + self._listeners.publish_topology_description_changed, + ( + old_td, + self._description, + self._topology_id, + ), + ) + ) self._events.put((self._listeners.publish_topology_closed, (self._topology_id,))) if self._publish_server or self._publish_tp: + # Make sure the events executor thread is fully closed before publishing the remaining events self.__events_executor.close() + self.__events_executor.join(1) + process_events_queue(weakref.ref(self._events)) # type: ignore[arg-type] @property def description(self) -> TopologyDescription: diff --git a/pymongo/errors.py b/pymongo/errors.py index a0f1ba2e9..1c51708c7 100644 --- a/pymongo/errors.py +++ b/pymongo/errors.py @@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Any, Iterable, Mapping, Optional, Sequence, Un from bson.errors import InvalidDocument if TYPE_CHECKING: + from pymongo.results import ClientBulkWriteResult from pymongo.typings import _DocumentOut @@ -308,6 +309,62 @@ class BulkWriteError(OperationFailure): return False +class ClientBulkWriteException(OperationFailure): + """Exception class for client-level bulk write errors.""" + + details: _DocumentOut + verbose: bool + + def __init__(self, results: _DocumentOut, verbose: bool) -> None: + super().__init__("batch op errors occurred", 65, results) + self.verbose = verbose + + def __reduce__(self) -> tuple[Any, Any]: + return self.__class__, (self.details,) + + @property + def error(self) -> Optional[Any]: + """A top-level error that occurred when attempting to + communicate with the server or execute the bulk write. + + This value may not be populated if the exception was + thrown due to errors occurring on individual writes. + """ + return self.details.get("error", None) + + @property + def write_concern_errors(self) -> Optional[list[WriteConcernError]]: + """Write concern errors that occurred during the bulk write. + + This list may have multiple items if more than one + server command was required to execute the bulk write. + """ + return self.details.get("writeConcernErrors", []) + + @property + def write_errors(self) -> Optional[Mapping[int, WriteError]]: + """Errors that occurred during the execution of individual write operations. + + This map will contain at most one entry if the bulk write was ordered. + """ + return self.details.get("writeErrors", {}) + + @property + def partial_result(self) -> Optional[ClientBulkWriteResult]: + """The results of any successful operations that were + performed before the error was encountered. + """ + from pymongo.results import ClientBulkWriteResult + + if self.details.get("anySuccessful"): + return ClientBulkWriteResult( + self.details, # type: ignore[arg-type] + acknowledged=True, + has_verbose_results=self.verbose, + ) + return None + + class InvalidOperation(PyMongoError): """Raised when a client attempts to perform an invalid operation.""" diff --git a/pymongo/message.py b/pymongo/message.py index bcb4ce10e..90fac8545 100644 --- a/pymongo/message.py +++ b/pymongo/message.py @@ -21,6 +21,7 @@ MongoDB. """ from __future__ import annotations +import copy import datetime import random import struct @@ -101,7 +102,12 @@ _OP_MAP = { _UPDATE: b"\x04updates\x00\x00\x00\x00\x00", _DELETE: b"\x04deletes\x00\x00\x00\x00\x00", } -_FIELD_MAP = {"insert": "documents", "update": "updates", "delete": "deletes"} +_FIELD_MAP = { + "insert": "documents", + "update": "updates", + "delete": "deletes", + "bulkWrite": "bulkWrite", +} _UNICODE_REPLACE_CODEC_OPTIONS: CodecOptions[Mapping[str, Any]] = CodecOptions( unicode_decode_error_handler="replace" @@ -136,6 +142,17 @@ def _convert_exception(exception: Exception) -> dict[str, Any]: return {"errmsg": str(exception), "errtype": exception.__class__.__name__} +def _convert_client_bulk_exception(exception: Exception) -> dict[str, Any]: + """Convert an Exception into a failure document for publishing, + for use in client-level bulk write API. + """ + return { + "errmsg": str(exception), + "code": exception.code, # type: ignore[attr-defined] + "errtype": exception.__class__.__name__, + } + + def _convert_write_result( operation: str, command: Mapping[str, Any], result: Mapping[str, Any] ) -> dict[str, Any]: @@ -551,8 +568,8 @@ _OP_MSG_MAP = { } -class _BulkWriteContext: - """A wrapper around AsyncConnection for use with write splitting functions.""" +class _BulkWriteContextBase: + """Private base class for wrapping around AsyncConnection to use with write splitting functions.""" __slots__ = ( "db_name", @@ -576,7 +593,7 @@ class _BulkWriteContext: conn: _AgnosticConnection, operation_id: int, listeners: _EventListeners, - session: _AgnosticClientSession, + session: Optional[_AgnosticClientSession], op_type: int, codec: CodecOptions, ): @@ -593,17 +610,6 @@ class _BulkWriteContext: self.op_type = op_type self.codec = codec - def batch_command( - self, cmd: MutableMapping[str, Any], docs: list[Mapping[str, Any]] - ) -> tuple[int, Union[bytes, dict[str, Any]], list[Mapping[str, Any]]]: - namespace = self.db_name + ".$cmd" - request_id, msg, to_send = _do_batched_op_msg( - namespace, self.op_type, cmd, docs, self.codec, self - ) - if not to_send: - raise InvalidOperation("cannot do an empty bulk write") - return request_id, msg, to_send - @property def max_bson_size(self) -> int: """A proxy for SockInfo.max_bson_size.""" @@ -627,22 +633,6 @@ class _BulkWriteContext: """The maximum size of a BSON command before batch splitting.""" return self.max_bson_size - def _start( - self, cmd: MutableMapping[str, Any], request_id: int, docs: list[Mapping[str, Any]] - ) -> MutableMapping[str, Any]: - """Publish a CommandStartedEvent.""" - cmd[self.field] = docs - self.listeners.publish_command_start( - cmd, - self.db_name, - request_id, - self.conn.address, - self.conn.server_connection_id, - self.op_id, - self.conn.service_id, - ) - return cmd - def _succeed(self, request_id: int, reply: _DocumentOut, duration: datetime.timedelta) -> None: """Publish a CommandSucceededEvent.""" self.listeners.publish_command_success( @@ -672,6 +662,61 @@ class _BulkWriteContext: ) +class _BulkWriteContext(_BulkWriteContextBase): + """A wrapper around AsyncConnection/Connection for use with the collection-level bulk write API.""" + + __slots__ = () + + def __init__( + self, + database_name: str, + cmd_name: str, + conn: _AgnosticConnection, + operation_id: int, + listeners: _EventListeners, + session: Optional[_AgnosticClientSession], + op_type: int, + codec: CodecOptions, + ): + super().__init__( + database_name, + cmd_name, + conn, + operation_id, + listeners, + session, + op_type, + codec, + ) + + def batch_command( + self, cmd: MutableMapping[str, Any], docs: list[Mapping[str, Any]] + ) -> tuple[int, Union[bytes, dict[str, Any]], list[Mapping[str, Any]]]: + namespace = self.db_name + ".$cmd" + request_id, msg, to_send = _do_batched_op_msg( + namespace, self.op_type, cmd, docs, self.codec, self + ) + if not to_send: + raise InvalidOperation("cannot do an empty bulk write") + return request_id, msg, to_send + + def _start( + self, cmd: MutableMapping[str, Any], request_id: int, docs: list[Mapping[str, Any]] + ) -> MutableMapping[str, Any]: + """Publish a CommandStartedEvent.""" + cmd[self.field] = docs + self.listeners.publish_command_start( + cmd, + self.db_name, + request_id, + self.conn.address, + self.conn.server_connection_id, + self.op_id, + self.conn.service_id, + ) + return cmd + + class _EncryptedBulkWriteContext(_BulkWriteContext): __slots__ = () @@ -878,6 +923,304 @@ def _do_batched_op_msg( return _batched_op_msg(operation, command, docs, ack, opts, ctx) +class _ClientBulkWriteContext(_BulkWriteContextBase): + """A wrapper around AsyncConnection/Connection for use with the client-level bulk write API.""" + + __slots__ = () + + def __init__( + self, + database_name: str, + cmd_name: str, + conn: _AgnosticConnection, + operation_id: int, + listeners: _EventListeners, + session: Optional[_AgnosticClientSession], + codec: CodecOptions, + ): + super().__init__( + database_name, + cmd_name, + conn, + operation_id, + listeners, + session, + 0, + codec, + ) + + def batch_command( + self, cmd: MutableMapping[str, Any], operations: list[tuple[str, Mapping[str, Any]]] + ) -> tuple[int, Union[bytes, dict[str, Any]], list[Mapping[str, Any]], list[Mapping[str, Any]]]: + request_id, msg, to_send_ops, to_send_ns = _client_do_batched_op_msg( + cmd, operations, self.codec, self + ) + if not to_send_ops: + raise InvalidOperation("cannot do an empty bulk write") + return request_id, msg, to_send_ops, to_send_ns + + def _start( + self, + cmd: MutableMapping[str, Any], + request_id: int, + op_docs: list[Mapping[str, Any]], + ns_docs: list[Mapping[str, Any]], + ) -> MutableMapping[str, Any]: + """Publish a CommandStartedEvent.""" + cmd["ops"] = op_docs + cmd["nsInfo"] = ns_docs + self.listeners.publish_command_start( + cmd, + self.db_name, + request_id, + self.conn.address, + self.conn.server_connection_id, + self.op_id, + self.conn.service_id, + ) + return cmd + + +_OP_MSG_OVERHEAD = 1000 + + +def _client_construct_op_msg( + command: Mapping[str, Any], + to_send_ops: list[Mapping[str, Any]], + to_send_ns: list[Mapping[str, Any]], + ack: bool, + opts: CodecOptions, + buf: _BytesIO, +) -> int: + # Write flags + flags = b"\x00\x00\x00\x00" if ack else b"\x02\x00\x00\x00" + buf.write(flags) + + # Type 0 Section + buf.write(b"\x00") + buf.write(_dict_to_bson(command, False, opts)) + + # Type 1 Section for ops + buf.write(b"\x01") + size_location = buf.tell() + # Save space for size + buf.write(b"\x00\x00\x00\x00") + buf.write(b"ops\x00") + # Write all the ops documents + for op in to_send_ops: + buf.write(_dict_to_bson(op, False, opts)) + resume_location = buf.tell() + # Write type 1 section size + length = buf.tell() + buf.seek(size_location) + buf.write(_pack_int(length - size_location)) + buf.seek(resume_location) + + # Type 1 Section for nsInfo + buf.write(b"\x01") + size_location = buf.tell() + # Save space for size + buf.write(b"\x00\x00\x00\x00") + buf.write(b"nsInfo\x00") + # Write all the nsInfo documents + for ns in to_send_ns: + buf.write(_dict_to_bson(ns, False, opts)) + # Write type 1 section size + length = buf.tell() + buf.seek(size_location) + buf.write(_pack_int(length - size_location)) + + return length + + +def _client_batched_op_msg_impl( + command: Mapping[str, Any], + operations: list[tuple[str, Mapping[str, Any]]], + ack: bool, + opts: CodecOptions, + ctx: _ClientBulkWriteContext, + buf: _BytesIO, +) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]], int]: + """Create a batched OP_MSG write for client-level bulk write.""" + + def _check_doc_size_limits( + op_type: str, + document: Mapping[str, Any], + limit: int, + ) -> int: + doc_size = len(_dict_to_bson(document, False, opts)) + if doc_size > limit: + _raise_document_too_large(op_type, doc_size, limit) + return doc_size + + max_bson_size = ctx.max_bson_size + max_write_batch_size = ctx.max_write_batch_size + max_message_size = ctx.max_message_size + + # Don't include bulkWrite-command-agnostic fields in document size calculations. + abridged_keys = ["bulkWrite", "errorsOnly", "ordered"] + if command.get("bypassDocumentValidation"): + abridged_keys.append("bypassDocumentValidation") + if command.get("comment"): + abridged_keys.append("comment") + if command.get("let"): + abridged_keys.append("let") + command_abridged = {key: command[key] for key in abridged_keys} + command_len_abridged = len(_dict_to_bson(command_abridged, False, opts)) + + # When OP_MSG is used unacknowledged we have to check command + # document size client-side or applications won't be notified. + if not ack: + _check_doc_size_limits("bulkWrite", command_abridged, max_bson_size + _COMMAND_OVERHEAD) + + # Maximum combined size of the ops and nsInfo document sequences. + max_doc_sequences_bytes = max_message_size - (_OP_MSG_OVERHEAD + command_len_abridged) + + ns_info = {} + to_send_ops: list[Mapping[str, Any]] = [] + to_send_ns: list[Mapping[str, int]] = [] + total_ops_length = 0 + total_ns_length = 0 + idx = 0 + + for real_op_type, op_doc in operations: + op_type = real_op_type + # Check insert/replace document size if unacknowledged. + if real_op_type == "insert": + if not ack: + _check_doc_size_limits(real_op_type, op_doc["document"], max_bson_size) + if real_op_type == "replace": + op_type = "update" + if not ack: + _check_doc_size_limits(real_op_type, op_doc["updateMods"], max_bson_size) + + ns_doc_to_send = None + ns_length = 0 + namespace = op_doc[op_type] + if namespace not in ns_info: + ns_doc_to_send = {"ns": namespace} + new_ns_index = len(to_send_ns) + ns_info[namespace] = new_ns_index + + # First entry in the operation doc has the operation type as its + # key and the index of its namespace within ns_info as its value. + op_doc_to_send = copy.deepcopy(op_doc) + op_doc_to_send[op_type] = ns_info[namespace] # type: ignore[index] + + # Encode current operation doc and, if newly added, namespace doc. + op_length = len(_dict_to_bson(op_doc_to_send, False, opts)) + if ns_doc_to_send: + ns_length = len(_dict_to_bson(ns_doc_to_send, False, opts)) + + # Check operation document size if unacknowledged. + if not ack: + _check_doc_size_limits(op_type, op_doc_to_send, max_bson_size + _COMMAND_OVERHEAD) + + new_message_size = total_ops_length + total_ns_length + op_length + ns_length + # We have enough data, return this batch. + if new_message_size > max_doc_sequences_bytes: + break + to_send_ops.append(op_doc_to_send) + total_ops_length += op_length + if ns_doc_to_send: + to_send_ns.append(ns_doc_to_send) + total_ns_length += ns_length + idx += 1 + # We have enough documents, return this batch. + if idx == max_write_batch_size: + break + + # Construct the entire OP_MSG. + length = _client_construct_op_msg(command, to_send_ops, to_send_ns, ack, opts, buf) + + return to_send_ops, to_send_ns, length + + +def _client_encode_batched_op_msg( + command: Mapping[str, Any], + operations: list[tuple[str, Mapping[str, Any]]], + ack: bool, + opts: CodecOptions, + ctx: _ClientBulkWriteContext, +) -> tuple[bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Encode the next batched client-level bulkWrite + operation as OP_MSG. + """ + buf = _BytesIO() + + to_send_ops, to_send_ns, _ = _client_batched_op_msg_impl( + command, operations, ack, opts, ctx, buf + ) + return buf.getvalue(), to_send_ops, to_send_ns + + +def _client_batched_op_msg_compressed( + command: Mapping[str, Any], + operations: list[tuple[str, Mapping[str, Any]]], + ack: bool, + opts: CodecOptions, + ctx: _ClientBulkWriteContext, +) -> tuple[int, bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Create the next batched client-level bulkWrite operation + with OP_MSG, compressed. + """ + data, to_send_ops, to_send_ns = _client_encode_batched_op_msg( + command, operations, ack, opts, ctx + ) + + assert ctx.conn.compression_context is not None + request_id, msg = _compress(2013, data, ctx.conn.compression_context) + return request_id, msg, to_send_ops, to_send_ns + + +def _client_batched_op_msg( + command: Mapping[str, Any], + operations: list[tuple[str, Mapping[str, Any]]], + ack: bool, + opts: CodecOptions, + ctx: _ClientBulkWriteContext, +) -> tuple[int, bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """OP_MSG implementation entry point for client-level bulkWrite.""" + buf = _BytesIO() + + # Save space for message length and request id + buf.write(_ZERO_64) + # responseTo, opCode + buf.write(b"\x00\x00\x00\x00\xdd\x07\x00\x00") + + to_send_ops, to_send_ns, length = _client_batched_op_msg_impl( + command, operations, ack, opts, ctx, buf + ) + + # Header - request id and message length + buf.seek(4) + request_id = _randint() + buf.write(_pack_int(request_id)) + buf.seek(0) + buf.write(_pack_int(length)) + + return request_id, buf.getvalue(), to_send_ops, to_send_ns + + +def _client_do_batched_op_msg( + command: MutableMapping[str, Any], + operations: list[tuple[str, Mapping[str, Any]]], + opts: CodecOptions, + ctx: _ClientBulkWriteContext, +) -> tuple[int, bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Create the next batched client-level bulkWrite + operation using OP_MSG. + """ + command["$db"] = "admin" + if "writeConcern" in command: + ack = bool(command["writeConcern"].get("w", 1)) + else: + ack = True + if ctx.conn.compression_context: + return _client_batched_op_msg_compressed(command, operations, ack, opts, ctx) + return _client_batched_op_msg(command, operations, ack, opts, ctx) + + # End OP_MSG ----------------------------------------------------- diff --git a/pymongo/ocsp_cache.py b/pymongo/ocsp_cache.py index 742579312..3facefe35 100644 --- a/pymongo/ocsp_cache.py +++ b/pymongo/ocsp_cache.py @@ -19,7 +19,7 @@ from __future__ import annotations from collections import namedtuple from datetime import datetime as _datetime from datetime import timezone -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional from pymongo.lock import _create_lock @@ -27,6 +27,22 @@ if TYPE_CHECKING: from cryptography.x509.ocsp import OCSPRequest, OCSPResponse +def _next_update(value: OCSPResponse) -> Optional[_datetime]: + """Compat helper to return the response's next_update_utc.""" + # Added in cryptography 43.0.0. + if hasattr(value, "next_update_utc"): + return value.next_update_utc + return value.next_update + + +def _this_update(value: OCSPResponse) -> Optional[_datetime]: + """Compat helper to return the response's this_update_utc.""" + # Added in cryptography 43.0.0. + if hasattr(value, "this_update_utc"): + return value.this_update_utc + return value.this_update + + class _OCSPCache: """A cache for OCSP responses.""" @@ -62,25 +78,30 @@ class _OCSPCache: # As per the OCSP protocol, if the response's nextUpdate field is # not set, the responder is indicating that newer revocation # information is available all the time. - if value.next_update is None: + next_update = _next_update(value) + if next_update is None: self._data.pop(cache_key, None) return + this_update = _this_update(value) + if this_update is None: + return + now = _datetime.now(tz=timezone.utc) + if this_update.tzinfo is None: + # Make naive to match cryptography. + now = now.replace(tzinfo=None) # Do nothing if the response is invalid. - if not ( - value.this_update - <= _datetime.now(tz=timezone.utc).replace(tzinfo=None) - < value.next_update - ): + if not (this_update <= now < next_update): return # Cache new response OR update cached response if new response # has longer validity. cached_value = self._data.get(cache_key, None) - if cached_value is None or ( - cached_value.next_update is not None - and cached_value.next_update < value.next_update - ): + if cached_value is None: + self._data[cache_key] = value + return + cached_next_update = _next_update(cached_value) + if cached_next_update is not None and cached_next_update < next_update: self._data[cache_key] = value def __getitem__(self, item: OCSPRequest) -> OCSPResponse: @@ -95,13 +116,15 @@ class _OCSPCache: value = self._data[cache_key] # Return cached response if it is still valid. - assert value.this_update is not None - assert value.next_update is not None - if ( - value.this_update - <= _datetime.now(tz=timezone.utc).replace(tzinfo=None) - < value.next_update - ): + this_update = _this_update(value) + next_update = _next_update(value) + assert this_update is not None + assert next_update is not None + now = _datetime.now(tz=timezone.utc) + if this_update.tzinfo is None: + # Make naive to match cryptography. + now = now.replace(tzinfo=None) + if this_update <= now < next_update: return value self._data.pop(cache_key, None) diff --git a/pymongo/ocsp_support.py b/pymongo/ocsp_support.py index 1bda3b4d7..ee359b71c 100644 --- a/pymongo/ocsp_support.py +++ b/pymongo/ocsp_support.py @@ -58,6 +58,7 @@ from requests import post as _post from requests.exceptions import RequestException as _RequestException from pymongo import _csot +from pymongo.ocsp_cache import _next_update, _this_update if TYPE_CHECKING: from cryptography.hazmat.primitives.asymmetric import ( @@ -275,13 +276,18 @@ def _verify_response(issuer: Certificate, response: OCSPResponse) -> int: # Note that we are not using a "tolerance period" as discussed in # https://tools.ietf.org/rfc/rfc5019.txt? - now = _datetime.now(tz=timezone.utc).replace(tzinfo=None) + this_update = _this_update(response) + now = _datetime.now(tz=timezone.utc) + if this_update and this_update.tzinfo is None: + # Make naive to match cryptography. + now = now.replace(tzinfo=None) # RFC6960, Section 3.2, Number 5 - if response.this_update > now: + if this_update and this_update > now: _LOGGER.debug("thisUpdate is in the future") return 0 # RFC6960, Section 3.2, Number 6 - if response.next_update and response.next_update < now: + next_update = _next_update(response) + if next_update and next_update < now: _LOGGER.debug("nextUpdate is in the past") return 0 return 1 diff --git a/pymongo/operations.py b/pymongo/operations.py index 7bb861ae4..d2e1feba6 100644 --- a/pymongo/operations.py +++ b/pymongo/operations.py @@ -34,12 +34,13 @@ from bson.raw_bson import RawBSONDocument from pymongo import helpers_shared from pymongo.collation import validate_collation_or_none from pymongo.common import validate_is_mapping, validate_list +from pymongo.errors import InvalidOperation from pymongo.helpers_shared import _gen_index_name, _index_document, _index_list from pymongo.typings import _CollationIn, _DocumentType, _Pipeline from pymongo.write_concern import validate_boolean if TYPE_CHECKING: - from pymongo.typings import _AgnosticBulk + from pymongo.typings import _AgnosticBulk, _AgnosticClientBulk # Hint supports index name, "myIndex", a list of either strings or index pairs: [('x', 1), ('y', -1), 'z''], or a dictionary @@ -52,6 +53,7 @@ _IndexKeyHint = Union[str, _IndexList] class _Op(str, enum.Enum): ABORT = "abortTransaction" AGGREGATE = "aggregate" + BULK_WRITE = "bulkWrite" COMMIT = "commitTransaction" COUNT = "count" CREATE = "create" @@ -83,48 +85,130 @@ class _Op(str, enum.Enum): class InsertOne(Generic[_DocumentType]): """Represents an insert_one operation.""" - __slots__ = ("_doc",) + __slots__ = ( + "_doc", + "_namespace", + ) - def __init__(self, document: _DocumentType) -> None: + def __init__(self, document: _DocumentType, namespace: Optional[str] = None) -> None: """Create an InsertOne instance. - For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`. + For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`, + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. :param document: The document to insert. If the document is missing an _id field one will be added. + :param namespace: (optional) The namespace in which to insert a document. + + .. versionchanged:: 4.9 + Added the `namespace` option to support `MongoClient.bulk_write`. """ self._doc = document + self._namespace = namespace def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" bulkobj.add_insert(self._doc) # type: ignore[arg-type] + def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: + """Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`.""" + if not self._namespace: + raise InvalidOperation( + "MongoClient.bulk_write requires a namespace to be provided for each write operation" + ) + bulkobj.add_insert( + self._namespace, + self._doc, # type: ignore[arg-type] + ) + def __repr__(self) -> str: - return f"InsertOne({self._doc!r})" + if self._namespace: + return f"{self.__class__.__name__}({self._doc!r}, {self._namespace!r})" + return f"{self.__class__.__name__}({self._doc!r})" def __eq__(self, other: Any) -> bool: if type(other) == type(self): - return other._doc == self._doc + return other._doc == self._doc and other._namespace == self._namespace return NotImplemented def __ne__(self, other: Any) -> bool: return not self == other -class DeleteOne: - """Represents a delete_one operation.""" +class _DeleteOp: + """Private base class for delete operations.""" - __slots__ = ("_filter", "_collation", "_hint") + __slots__ = ( + "_filter", + "_collation", + "_hint", + "_namespace", + ) def __init__( self, filter: Mapping[str, Any], collation: Optional[_CollationIn] = None, hint: Optional[_IndexKeyHint] = None, + namespace: Optional[str] = None, + ) -> None: + if filter is not None: + validate_is_mapping("filter", filter) + if hint is not None and not isinstance(hint, str): + self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint) + else: + self._hint = hint + + self._filter = filter + self._collation = collation + self._namespace = namespace + + def __eq__(self, other: Any) -> bool: + if type(other) == type(self): + return ( + other._filter, + other._collation, + other._hint, + other._namespace, + ) == ( + self._filter, + self._collation, + self._hint, + self._namespace, + ) + return NotImplemented + + def __ne__(self, other: Any) -> bool: + return not self == other + + def __repr__(self) -> str: + if self._namespace: + return "{}({!r}, {!r}, {!r}, {!r})".format( + self.__class__.__name__, + self._filter, + self._collation, + self._hint, + self._namespace, + ) + return f"{self.__class__.__name__}({self._filter!r}, {self._collation!r}, {self._hint!r})" + + +class DeleteOne(_DeleteOp): + """Represents a delete_one operation.""" + + __slots__ = () + + def __init__( + self, + filter: Mapping[str, Any], + collation: Optional[_CollationIn] = None, + hint: Optional[_IndexKeyHint] = None, + namespace: Optional[str] = None, ) -> None: """Create a DeleteOne instance. - For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`. + For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`, + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. :param filter: A query that matches the document to delete. :param collation: An instance of @@ -135,20 +219,16 @@ class DeleteOne: :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. + :param namespace: (optional) The namespace in which to delete a document. + .. versionchanged:: 4.9 + Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 Added the ``hint`` option. .. versionchanged:: 3.5 Added the `collation` option. """ - if filter is not None: - validate_is_mapping("filter", filter) - if hint is not None and not isinstance(hint, str): - self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint) - else: - self._hint = hint - self._filter = filter - self._collation = collation + super().__init__(filter, collation, hint, namespace) def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" @@ -159,36 +239,37 @@ class DeleteOne: hint=self._hint, ) - def __repr__(self) -> str: - return f"DeleteOne({self._filter!r}, {self._collation!r}, {self._hint!r})" - - def __eq__(self, other: Any) -> bool: - if type(other) == type(self): - return (other._filter, other._collation, other._hint) == ( - self._filter, - self._collation, - self._hint, + def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: + """Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`.""" + if not self._namespace: + raise InvalidOperation( + "MongoClient.bulk_write requires a namespace to be provided for each write operation" ) - return NotImplemented - - def __ne__(self, other: Any) -> bool: - return not self == other + bulkobj.add_delete( + self._namespace, + self._filter, + multi=False, + collation=validate_collation_or_none(self._collation), + hint=self._hint, + ) -class DeleteMany: +class DeleteMany(_DeleteOp): """Represents a delete_many operation.""" - __slots__ = ("_filter", "_collation", "_hint") + __slots__ = () def __init__( self, filter: Mapping[str, Any], collation: Optional[_CollationIn] = None, hint: Optional[_IndexKeyHint] = None, + namespace: Optional[str] = None, ) -> None: """Create a DeleteMany instance. - For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`. + For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`, + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. :param filter: A query that matches the documents to delete. :param collation: An instance of @@ -199,20 +280,16 @@ class DeleteMany: :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.4 and above. + :param namespace: (optional) The namespace in which to delete documents. + .. versionchanged:: 4.9 + Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 Added the ``hint`` option. .. versionchanged:: 3.5 Added the `collation` option. """ - if filter is not None: - validate_is_mapping("filter", filter) - if hint is not None and not isinstance(hint, str): - self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint) - else: - self._hint = hint - self._filter = filter - self._collation = collation + super().__init__(filter, collation, hint, namespace) def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" @@ -223,26 +300,32 @@ class DeleteMany: hint=self._hint, ) - def __repr__(self) -> str: - return f"DeleteMany({self._filter!r}, {self._collation!r}, {self._hint!r})" - - def __eq__(self, other: Any) -> bool: - if type(other) == type(self): - return (other._filter, other._collation, other._hint) == ( - self._filter, - self._collation, - self._hint, + def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: + """Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`.""" + if not self._namespace: + raise InvalidOperation( + "MongoClient.bulk_write requires a namespace to be provided for each write operation" ) - return NotImplemented - - def __ne__(self, other: Any) -> bool: - return not self == other + bulkobj.add_delete( + self._namespace, + self._filter, + multi=True, + collation=validate_collation_or_none(self._collation), + hint=self._hint, + ) class ReplaceOne(Generic[_DocumentType]): """Represents a replace_one operation.""" - __slots__ = ("_filter", "_doc", "_upsert", "_collation", "_hint") + __slots__ = ( + "_filter", + "_doc", + "_upsert", + "_collation", + "_hint", + "_namespace", + ) def __init__( self, @@ -251,10 +334,12 @@ class ReplaceOne(Generic[_DocumentType]): upsert: bool = False, collation: Optional[_CollationIn] = None, hint: Optional[_IndexKeyHint] = None, + namespace: Optional[str] = None, ) -> None: """Create a ReplaceOne instance. - For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`. + For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`, + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. :param filter: A query that matches the document to replace. :param replacement: The new document. @@ -268,7 +353,10 @@ class ReplaceOne(Generic[_DocumentType]): :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. + :param namespace: (optional) The namespace in which to replace a document. + .. versionchanged:: 4.9 + Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 Added the ``hint`` option. .. versionchanged:: 3.5 @@ -282,10 +370,12 @@ class ReplaceOne(Generic[_DocumentType]): self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint) else: self._hint = hint + self._filter = filter self._doc = replacement self._upsert = upsert self._collation = collation + self._namespace = namespace def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" @@ -297,6 +387,21 @@ class ReplaceOne(Generic[_DocumentType]): hint=self._hint, ) + def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: + """Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`.""" + if not self._namespace: + raise InvalidOperation( + "MongoClient.bulk_write requires a namespace to be provided for each write operation" + ) + bulkobj.add_replace( + self._namespace, + self._filter, + self._doc, + self._upsert, + collation=validate_collation_or_none(self._collation), + hint=self._hint, + ) + def __eq__(self, other: Any) -> bool: if type(other) == type(self): return ( @@ -305,12 +410,14 @@ class ReplaceOne(Generic[_DocumentType]): other._upsert, other._collation, other._hint, + other._namespace, ) == ( self._filter, self._doc, self._upsert, self._collation, other._hint, + self._namespace, ) return NotImplemented @@ -318,6 +425,16 @@ class ReplaceOne(Generic[_DocumentType]): return not self == other def __repr__(self) -> str: + if self._namespace: + return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( + self.__class__.__name__, + self._filter, + self._doc, + self._upsert, + self._collation, + self._hint, + self._namespace, + ) return "{}({!r}, {!r}, {!r}, {!r}, {!r})".format( self.__class__.__name__, self._filter, @@ -331,16 +448,25 @@ class ReplaceOne(Generic[_DocumentType]): class _UpdateOp: """Private base class for update operations.""" - __slots__ = ("_filter", "_doc", "_upsert", "_collation", "_array_filters", "_hint") + __slots__ = ( + "_filter", + "_doc", + "_upsert", + "_collation", + "_array_filters", + "_hint", + "_namespace", + ) def __init__( self, filter: Mapping[str, Any], doc: Union[Mapping[str, Any], _Pipeline], - upsert: bool, + upsert: Optional[bool], collation: Optional[_CollationIn], array_filters: Optional[list[Mapping[str, Any]]], hint: Optional[_IndexKeyHint], + namespace: Optional[str], ): if filter is not None: validate_is_mapping("filter", filter) @@ -358,6 +484,7 @@ class _UpdateOp: self._upsert = upsert self._collation = collation self._array_filters = array_filters + self._namespace = namespace def __eq__(self, other: object) -> bool: if isinstance(other, type(self)): @@ -368,6 +495,7 @@ class _UpdateOp: other._collation, other._array_filters, other._hint, + other._namespace, ) == ( self._filter, self._doc, @@ -375,10 +503,25 @@ class _UpdateOp: self._collation, self._array_filters, self._hint, + self._namespace, ) return NotImplemented + def __ne__(self, other: Any) -> bool: + return not self == other + def __repr__(self) -> str: + if self._namespace: + return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( + self.__class__.__name__, + self._filter, + self._doc, + self._upsert, + self._collation, + self._array_filters, + self._hint, + self._namespace, + ) return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( self.__class__.__name__, self._filter, @@ -399,14 +542,16 @@ class UpdateOne(_UpdateOp): self, filter: Mapping[str, Any], update: Union[Mapping[str, Any], _Pipeline], - upsert: bool = False, + upsert: Optional[bool] = None, collation: Optional[_CollationIn] = None, array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Optional[_IndexKeyHint] = None, + namespace: Optional[str] = None, ) -> None: """Represents an update_one operation. - For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`. + For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`, + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. :param filter: A query that matches the document to update. :param update: The modifications to apply. @@ -422,7 +567,10 @@ class UpdateOne(_UpdateOp): :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. + :param namespace: (optional) The namespace in which to update a document. + .. versionchanged:: 4.9 + Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 Added the `hint` option. .. versionchanged:: 3.9 @@ -432,11 +580,28 @@ class UpdateOne(_UpdateOp): .. versionchanged:: 3.5 Added the `collation` option. """ - super().__init__(filter, update, upsert, collation, array_filters, hint) + super().__init__(filter, update, upsert, collation, array_filters, hint, namespace) def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" bulkobj.add_update( + self._filter, + self._doc, + False, + bool(self._upsert), + collation=validate_collation_or_none(self._collation), + array_filters=self._array_filters, + hint=self._hint, + ) + + def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: + """Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`.""" + if not self._namespace: + raise InvalidOperation( + "MongoClient.bulk_write requires a namespace to be provided for each write operation" + ) + bulkobj.add_update( + self._namespace, self._filter, self._doc, False, @@ -456,14 +621,16 @@ class UpdateMany(_UpdateOp): self, filter: Mapping[str, Any], update: Union[Mapping[str, Any], _Pipeline], - upsert: bool = False, + upsert: Optional[bool] = None, collation: Optional[_CollationIn] = None, array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Optional[_IndexKeyHint] = None, + namespace: Optional[str] = None, ) -> None: """Create an UpdateMany instance. - For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`. + For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`, + :meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. :param filter: A query that matches the documents to update. :param update: The modifications to apply. @@ -479,7 +646,10 @@ class UpdateMany(_UpdateOp): :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. + :param namespace: (optional) The namespace in which to update documents. + .. versionchanged:: 4.9 + Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 Added the `hint` option. .. versionchanged:: 3.9 @@ -489,11 +659,28 @@ class UpdateMany(_UpdateOp): .. versionchanged:: 3.5 Added the `collation` option. """ - super().__init__(filter, update, upsert, collation, array_filters, hint) + super().__init__(filter, update, upsert, collation, array_filters, hint, namespace) def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" bulkobj.add_update( + self._filter, + self._doc, + True, + bool(self._upsert), + collation=validate_collation_or_none(self._collation), + array_filters=self._array_filters, + hint=self._hint, + ) + + def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: + """Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`.""" + if not self._namespace: + raise InvalidOperation( + "MongoClient.bulk_write requires a namespace to be provided for each write operation" + ) + bulkobj.add_update( + self._namespace, self._filter, self._doc, True, diff --git a/pymongo/read_preferences.py b/pymongo/read_preferences.py index 19b908a8c..8c6e6de45 100644 --- a/pymongo/read_preferences.py +++ b/pymongo/read_preferences.py @@ -607,10 +607,7 @@ class MovingAverage: def add_sample(self, sample: float) -> None: if sample < 0: - # Likely system time change while waiting for hello response - # and not using time.monotonic. Ignore it, the next one will - # probably be valid. - return + raise ValueError(f"duration cannot be negative {sample}") if self.average is None: self.average = sample else: diff --git a/pymongo/results.py b/pymongo/results.py index 1744f2c9e..b34f6c492 100644 --- a/pymongo/results.py +++ b/pymongo/results.py @@ -18,7 +18,7 @@ """ from __future__ import annotations -from typing import Any, Mapping, Optional, cast +from typing import Any, Mapping, MutableMapping, Optional, cast from pymongo.errors import InvalidOperation @@ -65,7 +65,9 @@ class _WriteResult: class InsertOneResult(_WriteResult): - """The return type for :meth:`~pymongo.collection.Collection.insert_one`.""" + """The return type for :meth:`~pymongo.collection.Collection.insert_one` + and as part of :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. + """ __slots__ = ("__inserted_id",) @@ -113,13 +115,23 @@ class InsertManyResult(_WriteResult): class UpdateResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.update_one`, :meth:`~pymongo.collection.Collection.update_many`, and - :meth:`~pymongo.collection.Collection.replace_one`. + :meth:`~pymongo.collection.Collection.replace_one`, and as part of + :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. """ - __slots__ = ("__raw_result",) + __slots__ = ( + "__raw_result", + "__in_client_bulk", + ) - def __init__(self, raw_result: Optional[Mapping[str, Any]], acknowledged: bool): + def __init__( + self, + raw_result: Optional[Mapping[str, Any]], + acknowledged: bool, + in_client_bulk: bool = False, + ): self.__raw_result = raw_result + self.__in_client_bulk = in_client_bulk super().__init__(acknowledged) def __repr__(self) -> str: @@ -134,9 +146,9 @@ class UpdateResult(_WriteResult): def matched_count(self) -> int: """The number of documents matched for this update.""" self._raise_if_unacknowledged("matched_count") - if self.upserted_id is not None: - return 0 assert self.__raw_result is not None + if not self.__in_client_bulk and self.upserted_id is not None: + return 0 return self.__raw_result.get("n", 0) @property @@ -153,12 +165,21 @@ class UpdateResult(_WriteResult): """ self._raise_if_unacknowledged("upserted_id") assert self.__raw_result is not None - return self.__raw_result.get("upserted") + if self.__in_client_bulk and self.__raw_result.get("upserted"): + return self.__raw_result["upserted"]["_id"] + return self.__raw_result.get("upserted", None) + + @property + def did_upsert(self) -> bool: + """Whether or not an upsert took place.""" + assert self.__raw_result is not None + return len(self.__raw_result.get("upserted", {})) > 0 class DeleteResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.delete_one` and :meth:`~pymongo.collection.Collection.delete_many` + and as part of :meth:`~pymongo.mongo_client.MongoClient.bulk_write`. """ __slots__ = ("__raw_result",) @@ -182,19 +203,12 @@ class DeleteResult(_WriteResult): return self.__raw_result.get("n", 0) -class BulkWriteResult(_WriteResult): - """An object wrapper for bulk API write results.""" +class _BulkWriteResultBase(_WriteResult): + """Private base class for bulk write API results.""" __slots__ = ("__bulk_api_result",) def __init__(self, bulk_api_result: dict[str, Any], acknowledged: bool) -> None: - """Create a BulkWriteResult instance. - - :param bulk_api_result: A result dict from the bulk API - :param acknowledged: Was this write result acknowledged? If ``False`` - then all properties of this object will raise - :exc:`~pymongo.errors.InvalidOperation`. - """ self.__bulk_api_result = bulk_api_result super().__init__(acknowledged) @@ -203,7 +217,7 @@ class BulkWriteResult(_WriteResult): @property def bulk_api_result(self) -> dict[str, Any]: - """The raw bulk API result.""" + """The raw bulk write API result.""" return self.__bulk_api_result @property @@ -228,7 +242,10 @@ class BulkWriteResult(_WriteResult): def deleted_count(self) -> int: """The number of documents deleted.""" self._raise_if_unacknowledged("deleted_count") - return cast(int, self.__bulk_api_result.get("nRemoved")) + if "nRemoved" in self.__bulk_api_result: + return cast(int, self.__bulk_api_result.get("nRemoved")) + else: + return cast(int, self.__bulk_api_result.get("nDeleted")) @property def upserted_count(self) -> int: @@ -236,10 +253,112 @@ class BulkWriteResult(_WriteResult): self._raise_if_unacknowledged("upserted_count") return cast(int, self.__bulk_api_result.get("nUpserted")) + +class BulkWriteResult(_BulkWriteResultBase): + """An object wrapper for collection-level bulk write API results.""" + + __slots__ = () + + def __init__(self, bulk_api_result: dict[str, Any], acknowledged: bool) -> None: + """Create a BulkWriteResult instance. + + :param bulk_api_result: A result dict from the collection-level bulk write API + :param acknowledged: Was this write result acknowledged? If ``False`` + then all properties of this object will raise + :exc:`~pymongo.errors.InvalidOperation`. + """ + super().__init__(bulk_api_result, acknowledged) + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}({self.bulk_api_result!r}, acknowledged={self.acknowledged})" + ) + @property def upserted_ids(self) -> Optional[dict[int, Any]]: """A map of operation index to the _id of the upserted document.""" self._raise_if_unacknowledged("upserted_ids") - if self.__bulk_api_result: + if self.bulk_api_result: return {upsert["index"]: upsert["_id"] for upsert in self.bulk_api_result["upserted"]} return None + + +class ClientBulkWriteResult(_BulkWriteResultBase): + """An object wrapper for client-level bulk write API results.""" + + __slots__ = ("__has_verbose_results",) + + def __init__( + self, + bulk_api_result: MutableMapping[str, Any], + acknowledged: bool, + has_verbose_results: bool, + ) -> None: + """Create a ClientBulkWriteResult instance. + + :param bulk_api_result: A result dict from the client-level bulk write API + :param acknowledged: Was this write result acknowledged? If ``False`` + then all properties of this object will raise + :exc:`~pymongo.errors.InvalidOperation`. + :param has_verbose_results: Should the returned result be verbose? + If ``False``, then the ``insert_results``, ``update_results``, and + ``delete_results`` properties of this object will raise + :exc:`~pymongo.errors.InvalidOperation`. + """ + self.__has_verbose_results = has_verbose_results + super().__init__( + bulk_api_result, # type: ignore[arg-type] + acknowledged, + ) + + def __repr__(self) -> str: + return "{}({!r}, acknowledged={}, verbose={})".format( + self.__class__.__name__, + self.bulk_api_result, + self.acknowledged, + self.has_verbose_results, + ) + + def _raise_if_not_verbose(self, property_name: str) -> None: + """Raise an exception on property access if verbose results are off.""" + if not self.__has_verbose_results: + raise InvalidOperation( + f"A value for {property_name} is not available when " + "the results are not set to be verbose. Check the " + "verbose_results attribute to avoid this error." + ) + + @property + def has_verbose_results(self) -> bool: + """Whether the returned results should be verbose.""" + return self.__has_verbose_results + + @property + def insert_results(self) -> Mapping[int, InsertOneResult]: + """A map of successful insertion operations to their results.""" + self._raise_if_unacknowledged("insert_results") + self._raise_if_not_verbose("insert_results") + return cast( + Mapping[int, InsertOneResult], + self.bulk_api_result.get("insertResults"), + ) + + @property + def update_results(self) -> Mapping[int, UpdateResult]: + """A map of successful update operations to their results.""" + self._raise_if_unacknowledged("update_results") + self._raise_if_not_verbose("update_results") + return cast( + Mapping[int, UpdateResult], + self.bulk_api_result.get("updateResults"), + ) + + @property + def delete_results(self) -> Mapping[int, DeleteResult]: + """A map of successful delete operations to their results.""" + self._raise_if_unacknowledged("delete_results") + self._raise_if_not_verbose("delete_results") + return cast( + Mapping[int, DeleteResult], + self.bulk_api_result.get("deleteResults"), + ) diff --git a/pymongo/synchronous/change_stream.py b/pymongo/synchronous/change_stream.py index f7489249d..a971ad08c 100644 --- a/pymongo/synchronous/change_stream.py +++ b/pymongo/synchronous/change_stream.py @@ -297,8 +297,8 @@ class ChangeStream(Generic[_DocumentType]): try: resume_token = None pipeline = [{'$match': {'operationType': 'insert'}}] - async with db.collection.watch(pipeline) as stream: - async for insert_change in stream: + with db.collection.watch(pipeline) as stream: + for insert_change in stream: print(insert_change) resume_token = stream.resume_token except pymongo.errors.PyMongoError: @@ -312,9 +312,9 @@ class ChangeStream(Generic[_DocumentType]): # Use the interrupted ChangeStream's resume token to create # a new ChangeStream. The new stream will continue from the # last seen insert change without missing any events. - async with db.collection.watch( + with db.collection.watch( pipeline, resume_after=resume_token) as stream: - async for insert_change in stream: + for insert_change in stream: print(insert_change) Raises :exc:`StopIteration` if this ChangeStream is closed. @@ -346,9 +346,9 @@ class ChangeStream(Generic[_DocumentType]): This method returns the next change document without waiting indefinitely for the next change. For example:: - async with db.collection.watch() as stream: + with db.collection.watch() as stream: while stream.alive: - change = await stream.try_next() + change = stream.try_next() # Note that the ChangeStream's resume token may be updated # even when no changes are returned. print("Current resume token: %r" % (stream.resume_token,)) diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py new file mode 100644 index 000000000..229abd433 --- /dev/null +++ b/pymongo/synchronous/client_bulk.py @@ -0,0 +1,786 @@ +# Copyright 2024-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The client-level bulk write operations interface. + +.. versionadded:: 4.9 +""" +from __future__ import annotations + +import copy +import datetime +import logging +from collections.abc import MutableMapping +from itertools import islice +from typing import ( + TYPE_CHECKING, + Any, + Mapping, + Optional, + Type, + Union, +) + +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo import _csot, common +from pymongo.synchronous.client_session import ClientSession, _validate_session_write_concern +from pymongo.synchronous.collection import Collection +from pymongo.synchronous.command_cursor import CommandCursor +from pymongo.synchronous.database import Database +from pymongo.synchronous.helpers import _handle_reauth + +if TYPE_CHECKING: + from pymongo.synchronous.mongo_client import MongoClient + from pymongo.synchronous.pool import Connection +from pymongo._client_bulk_shared import ( + _merge_command, + _throw_client_bulk_write_exception, +) +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) +from pymongo.errors import ( + ConfigurationError, + ConnectionFailure, + InvalidOperation, + NotPrimaryError, + OperationFailure, + WaitQueueTimeoutError, +) +from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES +from pymongo.logger import _COMMAND_LOGGER, _CommandStatusMessage, _debug_log +from pymongo.message import ( + _ClientBulkWriteContext, + _convert_client_bulk_exception, + _convert_exception, + _convert_write_result, + _randint, +) +from pymongo.read_preferences import ReadPreference +from pymongo.results import ( + ClientBulkWriteResult, + DeleteResult, + InsertOneResult, + UpdateResult, +) +from pymongo.typings import _DocumentOut, _Pipeline +from pymongo.write_concern import WriteConcern + +_IS_SYNC = True + + +class _ClientBulk: + """The private guts of the client-level bulk write API.""" + + def __init__( + self, + client: MongoClient, + write_concern: WriteConcern, + ordered: bool = True, + bypass_document_validation: Optional[bool] = None, + comment: Optional[str] = None, + let: Optional[Any] = None, + verbose_results: bool = False, + ) -> None: + """Initialize a _ClientBulk instance.""" + self.client = client + self.write_concern = write_concern + self.let = let + if self.let is not None: + common.validate_is_document_type("let", self.let) + self.ordered = ordered + self.bypass_doc_val = bypass_document_validation + self.comment = comment + self.verbose_results = verbose_results + + self.ops: list[tuple[str, Mapping[str, Any]]] = [] + self.idx_offset: int = 0 + self.total_ops: int = 0 + + self.executed = False + self.uses_upsert = False + self.uses_collation = False + self.uses_array_filters = False + self.uses_hint_update = False + self.uses_hint_delete = False + + self.is_retryable = self.client.options.retry_writes + self.retrying = False + self.started_retryable_write = False + + @property + def bulk_ctx_class(self) -> Type[_ClientBulkWriteContext]: + return _ClientBulkWriteContext + + def add_insert(self, namespace: str, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + cmd = {"insert": namespace, "document": document} + self.ops.append(("insert", cmd)) + self.total_ops += 1 + + def add_update( + self, + namespace: str, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool = False, + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd = { + "update": namespace, + "filter": selector, + "updateMods": update, + "multi": multi, + } + if upsert is not None: + self.uses_upsert = True + cmd["upsert"] = upsert + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + self.uses_hint_update = True + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("update", cmd)) + self.total_ops += 1 + + def add_replace( + self, + namespace: str, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd = { + "update": namespace, + "filter": selector, + "updateMods": replacement, + "multi": False, + } + if upsert is not None: + self.uses_upsert = True + cmd["upsert"] = upsert + if hint is not None: + self.uses_hint_update = True + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + self.ops.append(("replace", cmd)) + self.total_ops += 1 + + def add_delete( + self, + namespace: str, + selector: Mapping[str, Any], + multi: bool, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd = {"delete": namespace, "filter": selector, "multi": multi} + if hint is not None: + self.uses_hint_delete = True + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("delete", cmd)) + self.total_ops += 1 + + @_handle_reauth + def write_command( + self, + bwc: _ClientBulkWriteContext, + cmd: MutableMapping[str, Any], + request_id: int, + msg: Union[bytes, dict[str, Any]], + op_docs: list[Mapping[str, Any]], + ns_docs: list[Mapping[str, Any]], + client: MongoClient, + ) -> dict[str, Any]: + """A proxy for Connection.write_command that handles event publishing.""" + cmd["ops"] = op_docs + cmd["nsInfo"] = ns_docs + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.STARTED, + command=cmd, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + bwc._start(cmd, request_id, op_docs, ns_docs) + try: + reply = bwc.conn.write_command(request_id, msg, bwc.codec) # type: ignore[misc, arg-type] + duration = datetime.datetime.now() - bwc.start_time + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.SUCCEEDED, + durationMS=duration, + reply=reply, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + bwc._succeed(request_id, reply, duration) # type: ignore[arg-type] + except Exception as exc: + duration = datetime.datetime.now() - bwc.start_time + if isinstance(exc, (NotPrimaryError, OperationFailure)): + failure: _DocumentOut = exc.details # type: ignore[assignment] + else: + failure = _convert_exception(exc) + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.FAILED, + durationMS=duration, + failure=failure, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + isServerSideError=isinstance(exc, OperationFailure), + ) + + if bwc.publish: + bwc._fail(request_id, failure, duration) + # Top-level error will be embedded in ClientBulkWriteException. + reply = {"error": exc} + finally: + bwc.start_time = datetime.datetime.now() + return reply # type: ignore[return-value] + + def unack_write( + self, + bwc: _ClientBulkWriteContext, + cmd: MutableMapping[str, Any], + request_id: int, + msg: bytes, + op_docs: list[Mapping[str, Any]], + ns_docs: list[Mapping[str, Any]], + client: MongoClient, + ) -> Optional[Mapping[str, Any]]: + """A proxy for Connection.unack_write that handles event publishing.""" + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.STARTED, + command=cmd, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + cmd = bwc._start(cmd, request_id, op_docs, ns_docs) + try: + result = bwc.conn.unack_write(msg, bwc.max_bson_size) # type: ignore[func-returns-value, misc, override] + duration = datetime.datetime.now() - bwc.start_time + if result is not None: + reply = _convert_write_result(bwc.name, cmd, result) # type: ignore[arg-type] + else: + # Comply with APM spec. + reply = {"ok": 1} + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.SUCCEEDED, + durationMS=duration, + reply=reply, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + ) + if bwc.publish: + bwc._succeed(request_id, reply, duration) + except Exception as exc: + duration = datetime.datetime.now() - bwc.start_time + if isinstance(exc, OperationFailure): + failure: _DocumentOut = _convert_write_result(bwc.name, cmd, exc.details) # type: ignore[arg-type] + elif isinstance(exc, NotPrimaryError): + failure = exc.details # type: ignore[assignment] + else: + failure = _convert_exception(exc) + if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG): + _debug_log( + _COMMAND_LOGGER, + clientId=client._topology_settings._topology_id, + message=_CommandStatusMessage.FAILED, + durationMS=duration, + failure=failure, + commandName=next(iter(cmd)), + databaseName=bwc.db_name, + requestId=request_id, + operationId=request_id, + driverConnectionId=bwc.conn.id, + serverConnectionId=bwc.conn.server_connection_id, + serverHost=bwc.conn.address[0], + serverPort=bwc.conn.address[1], + serviceId=bwc.conn.service_id, + isServerSideError=isinstance(exc, OperationFailure), + ) + if bwc.publish: + assert bwc.start_time is not None + bwc._fail(request_id, failure, duration) + # Top-level error will be embedded in ClientBulkWriteException. + reply = {"error": exc} + finally: + bwc.start_time = datetime.datetime.now() + return result # type: ignore[return-value] + + def _execute_batch_unack( + self, + bwc: _ClientBulkWriteContext, + cmd: dict[str, Any], + ops: list[tuple[str, Mapping[str, Any]]], + ) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Executes a batch of bulkWrite server commands (unack).""" + request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops) + self.unack_write(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type] + return to_send_ops, to_send_ns + + def _execute_batch( + self, + bwc: _ClientBulkWriteContext, + cmd: dict[str, Any], + ops: list[tuple[str, Mapping[str, Any]]], + ) -> tuple[dict[str, Any], list[Mapping[str, Any]], list[Mapping[str, Any]]]: + """Executes a batch of bulkWrite server commands (ack).""" + request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops) + result = self.write_command(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type] + self.client._process_response(result, bwc.session) # type: ignore[arg-type] + return result, to_send_ops, to_send_ns # type: ignore[return-value] + + def _process_results_cursor( + self, + full_result: MutableMapping[str, Any], + result: MutableMapping[str, Any], + conn: Connection, + session: Optional[ClientSession], + ) -> None: + """Internal helper for processing the server reply command cursor.""" + if result.get("cursor"): + coll = Collection( + database=Database(self.client, "admin"), + name="$cmd.bulkWrite", + ) + cmd_cursor = CommandCursor( + coll, + result["cursor"], + conn.address, + session=session, + explicit_session=session is not None, + comment=self.comment, + ) + cmd_cursor._maybe_pin_connection(conn) + + # Iterate the cursor to get individual write results. + try: + for doc in cmd_cursor: + original_index = doc["idx"] + self.idx_offset + op_type, op = self.ops[original_index] + + if not doc["ok"]: + result["writeErrors"].append(doc) + if self.ordered: + return + + # Record individual write result. + if doc["ok"] and self.verbose_results: + if op_type == "insert": + inserted_id = op["document"]["_id"] + res = InsertOneResult(inserted_id, acknowledged=True) # type: ignore[assignment] + if op_type in ["update", "replace"]: + op_type = "update" + res = UpdateResult(doc, acknowledged=True, in_client_bulk=True) # type: ignore[assignment] + if op_type == "delete": + res = DeleteResult(doc, acknowledged=True) # type: ignore[assignment] + full_result[f"{op_type}Results"][original_index] = res + + except Exception as exc: + # Attempt to close the cursor, then raise top-level error. + if cmd_cursor.alive: + cmd_cursor.close() + result["error"] = _convert_client_bulk_exception(exc) + + def _execute_command( + self, + write_concern: WriteConcern, + session: Optional[ClientSession], + conn: Connection, + op_id: int, + retryable: bool, + full_result: MutableMapping[str, Any], + final_write_concern: Optional[WriteConcern] = None, + ) -> None: + """Internal helper for executing batches of bulkWrite commands.""" + db_name = "admin" + cmd_name = "bulkWrite" + listeners = self.client._event_listeners + + # Connection.command validates the session, but we use + # Connection.write_command + conn.validate_session(self.client, session) + + bwc = self.bulk_ctx_class( + db_name, + cmd_name, + conn, + op_id, + listeners, # type: ignore[arg-type] + session, + self.client.codec_options, + ) + + while self.idx_offset < self.total_ops: + # If this is the last possible batch, use the + # final write concern. + if self.total_ops - self.idx_offset <= bwc.max_write_batch_size: + write_concern = final_write_concern or write_concern + + # Construct the server command, specifying the relevant options. + cmd = {"bulkWrite": 1} + cmd["errorsOnly"] = not self.verbose_results + cmd["ordered"] = self.ordered # type: ignore[assignment] + not_in_transaction = session and not session.in_transaction + if not_in_transaction or not session: + _csot.apply_write_concern(cmd, write_concern) + if self.bypass_doc_val is not None: + cmd["bypassDocumentValidation"] = self.bypass_doc_val + if self.comment: + cmd["comment"] = self.comment # type: ignore[assignment] + if self.let: + cmd["let"] = self.let + + if session: + # Start a new retryable write unless one was already + # started for this command. + if retryable and not self.started_retryable_write: + session._start_retryable_write() + self.started_retryable_write = True + session._apply_to(cmd, retryable, ReadPreference.PRIMARY, conn) + conn.send_cluster_time(cmd, session, self.client) + conn.add_server_api(cmd) + # CSOT: apply timeout before encoding the command. + conn.apply_timeout(self.client, cmd) + ops = islice(self.ops, self.idx_offset, None) + + # Run as many ops as possible in one server command. + if write_concern.acknowledged: + raw_result, to_send_ops, _ = self._execute_batch(bwc, cmd, ops) # type: ignore[arg-type] + result = copy.deepcopy(raw_result) + + # Top-level server/network error. + if result.get("error"): + error = result["error"] + retryable_top_level_error = ( + isinstance(error.details, dict) + and error.details.get("code", 0) in _RETRYABLE_ERROR_CODES + ) + retryable_network_error = isinstance( + error, ConnectionFailure + ) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError)) + + # Synthesize the full bulk result without modifying the + # current one because this write operation may be retried. + if retryable and (retryable_top_level_error or retryable_network_error): + full = copy.deepcopy(full_result) + _merge_command(self.ops, self.idx_offset, full, result) + _throw_client_bulk_write_exception(full, self.verbose_results) + else: + _merge_command(self.ops, self.idx_offset, full_result, result) + _throw_client_bulk_write_exception(full_result, self.verbose_results) + + result["error"] = None + result["writeErrors"] = [] + if result.get("nErrors", 0) < len(to_send_ops): + full_result["anySuccessful"] = True + + # Top-level command error. + if not result["ok"]: + result["error"] = raw_result + _merge_command(self.ops, self.idx_offset, full_result, result) + break + + if retryable: + # Retryable writeConcernErrors halt the execution of this batch. + wce = result.get("writeConcernError", {}) + if wce.get("code", 0) in _RETRYABLE_ERROR_CODES: + # Synthesize the full bulk result without modifying the + # current one because this write operation may be retried. + full = copy.deepcopy(full_result) + _merge_command(self.ops, self.idx_offset, full, result) + _throw_client_bulk_write_exception(full, self.verbose_results) + + # Process the server reply as a command cursor. + self._process_results_cursor(full_result, result, conn, session) + + # Merge this batch's results with the full results. + _merge_command(self.ops, self.idx_offset, full_result, result) + + # We're no longer in a retry once a command succeeds. + self.retrying = False + self.started_retryable_write = False + + else: + to_send_ops, _ = self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type] + + self.idx_offset += len(to_send_ops) + + # We halt execution if we hit a top-level error, + # or an individual error in an ordered bulk write. + if full_result["error"] or (self.ordered and full_result["writeErrors"]): + break + + def execute_command( + self, + session: Optional[ClientSession], + operation: str, + ) -> MutableMapping[str, Any]: + """Execute commands with w=1 WriteConcern.""" + full_result: MutableMapping[str, Any] = { + "anySuccessful": False, + "error": None, + "writeErrors": [], + "writeConcernErrors": [], + "nInserted": 0, + "nUpserted": 0, + "nMatched": 0, + "nModified": 0, + "nDeleted": 0, + "insertResults": {}, + "updateResults": {}, + "deleteResults": {}, + } + op_id = _randint() + + def retryable_bulk( + session: Optional[ClientSession], + conn: Connection, + retryable: bool, + ) -> None: + if conn.max_wire_version < 25: + raise InvalidOperation( + "MongoClient.bulk_write requires MongoDB server version 8.0+." + ) + self._execute_command( + self.write_concern, + session, + conn, + op_id, + retryable, + full_result, + ) + + self.client._retryable_write( + self.is_retryable, + retryable_bulk, + session, + operation, + bulk=self, + operation_id=op_id, + ) + + if full_result["error"] or full_result["writeErrors"] or full_result["writeConcernErrors"]: + _throw_client_bulk_write_exception(full_result, self.verbose_results) + return full_result + + def execute_command_unack_unordered( + self, + conn: Connection, + ) -> None: + """Execute commands with OP_MSG and w=0 writeConcern, unordered.""" + db_name = "admin" + cmd_name = "bulkWrite" + listeners = self.client._event_listeners + op_id = _randint() + + bwc = self.bulk_ctx_class( + db_name, + cmd_name, + conn, + op_id, + listeners, # type: ignore[arg-type] + None, + self.client.codec_options, + ) + + while self.idx_offset < self.total_ops: + # Construct the server command, specifying the relevant options. + cmd = {"bulkWrite": 1} + cmd["errorsOnly"] = not self.verbose_results + cmd["ordered"] = self.ordered # type: ignore[assignment] + if self.bypass_doc_val is not None: + cmd["bypassDocumentValidation"] = self.bypass_doc_val + cmd["writeConcern"] = {"w": 0} # type: ignore[assignment] + if self.comment: + cmd["comment"] = self.comment # type: ignore[assignment] + if self.let: + cmd["let"] = self.let + + conn.add_server_api(cmd) + ops = islice(self.ops, self.idx_offset, None) + + # Run as many ops as possible in one server command. + to_send_ops, _ = self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type] + + self.idx_offset += len(to_send_ops) + + def execute_command_unack_ordered( + self, + conn: Connection, + ) -> None: + """Execute commands with OP_MSG and w=0 WriteConcern, ordered.""" + full_result: MutableMapping[str, Any] = { + "anySuccessful": False, + "error": None, + "writeErrors": [], + "writeConcernErrors": [], + "nInserted": 0, + "nUpserted": 0, + "nMatched": 0, + "nModified": 0, + "nDeleted": 0, + "insertResults": {}, + "updateResults": {}, + "deleteResults": {}, + } + # Ordered bulk writes have to be acknowledged so that we stop + # processing at the first error, even when the application + # specified unacknowledged writeConcern. + initial_write_concern = WriteConcern() + op_id = _randint() + try: + self._execute_command( + initial_write_concern, + None, + conn, + op_id, + False, + full_result, + self.write_concern, + ) + except OperationFailure: + pass + + def execute_no_results( + self, + conn: Connection, + ) -> None: + """Execute all operations, returning no results (w=0).""" + if self.uses_collation: + raise ConfigurationError("Collation is unsupported for unacknowledged writes.") + if self.uses_array_filters: + raise ConfigurationError("arrayFilters is unsupported for unacknowledged writes.") + # Cannot have both unacknowledged writes and bypass document validation. + if self.bypass_doc_val is not None: + raise OperationFailure( + "Cannot set bypass_document_validation with unacknowledged write concern" + ) + + if self.ordered: + return self.execute_command_unack_ordered(conn) + return self.execute_command_unack_unordered(conn) + + def execute( + self, + session: Optional[ClientSession], + operation: str, + ) -> Any: + """Execute operations.""" + if not self.ops: + raise InvalidOperation("No operations to execute") + if self.executed: + raise InvalidOperation("Bulk operations can only be executed once.") + self.executed = True + session = _validate_session_write_concern(session, self.write_concern) + + if not self.write_concern.acknowledged: + with self.client._conn_for_writes(session, operation) as connection: + if connection.max_wire_version < 25: + raise InvalidOperation( + "MongoClient.bulk_write requires MongoDB server version 8.0+." + ) + self.execute_no_results(connection) + return ClientBulkWriteResult(None, False, False) # type: ignore[arg-type] + + result = self.execute_command(session, operation) + return ClientBulkWriteResult( + result, + self.write_concern.acknowledged, + self.verbose_results, + ) diff --git a/pymongo/synchronous/client_session.py b/pymongo/synchronous/client_session.py index e07298b49..f1d680fc0 100644 --- a/pymongo/synchronous/client_session.py +++ b/pymongo/synchronous/client_session.py @@ -23,11 +23,11 @@ Causally Consistent Reads with client.start_session(causal_consistency=True) as session: collection = client.db.collection - await collection.update_one({"_id": 1}, {"$set": {"x": 10}}, session=session) + collection.update_one({"_id": 1}, {"$set": {"x": 10}}, session=session) secondary_c = collection.with_options(read_preference=ReadPreference.SECONDARY) # A secondary read waits for replication of the write. - await secondary_c.find_one({"_id": 1}, session=session) + secondary_c.find_one({"_id": 1}, session=session) If `causal_consistency` is True (the default), read operations that use the session are causally after previous read and write operations. Using a @@ -54,15 +54,15 @@ operation: orders = client.db.orders inventory = client.db.inventory with client.start_session() as session: - async with session.start_transaction(): - await orders.insert_one({"sku": "abc123", "qty": 100}, session=session) - await inventory.update_one( + with session.start_transaction(): + orders.insert_one({"sku": "abc123", "qty": 100}, session=session) + inventory.update_one( {"sku": "abc123", "qty": {"$gte": 100}}, {"$inc": {"qty": -100}}, session=session, ) -Upon normal completion of ``async with session.start_transaction()`` block, the +Upon normal completion of ``with session.start_transaction()`` block, the transaction automatically calls :meth:`ClientSession.commit_transaction`. If the block exits with an exception, the transaction automatically calls :meth:`ClientSession.abort_transaction`. @@ -114,8 +114,8 @@ replica set secondaries. # Each read using this session reads data from the same point in time. with client.start_session(snapshot=True) as session: - order = await orders.find_one({"sku": "abc123"}, session=session) - inventory = await inventory.find_one({"sku": "abc123"}, session=session) + order = orders.find_one({"sku": "abc123"}, session=session) + inventory = inventory.find_one({"sku": "abc123"}, session=session) Snapshot Reads Limitations ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -609,24 +609,24 @@ class ClientSession: This method starts a transaction on this session, executes ``callback`` once, and then commits the transaction. For example:: - async def callback(session): + def callback(session): orders = session.client.db.orders inventory = session.client.db.inventory - await orders.insert_one({"sku": "abc123", "qty": 100}, session=session) - await inventory.update_one({"sku": "abc123", "qty": {"$gte": 100}}, + orders.insert_one({"sku": "abc123", "qty": 100}, session=session) + inventory.update_one({"sku": "abc123", "qty": {"$gte": 100}}, {"$inc": {"qty": -100}}, session=session) with client.start_session() as session: - await session.with_transaction(callback) + session.with_transaction(callback) To pass arbitrary arguments to the ``callback``, wrap your callable with a ``lambda`` like this:: - async def callback(session, custom_arg, custom_kwarg=None): + def callback(session, custom_arg, custom_kwarg=None): # Transaction operations... with client.start_session() as session: - await session.with_transaction( + session.with_transaction( lambda s: callback(s, "custom_arg", custom_kwarg=1)) In the event of an exception, ``with_transaction`` may retry the commit diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 5803e34b2..54db3a56b 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -385,7 +385,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): __iter__ = None def __next__(self) -> NoReturn: - raise TypeError(f"'{type(self).__name__}' object is not iterable") + raise TypeError("'Collection' object is not iterable") next = __next__ @@ -423,19 +423,19 @@ class Collection(common.BaseObject, Generic[_DocumentType]): Performs an aggregation with an implicit initial ``$changeStream`` stage and returns a - :class:`~pymongo.synchronous.change_stream.CollectionChangeStream` cursor which + :class:`~pymongo.change_stream.CollectionChangeStream` cursor which iterates over changes on this collection. .. code-block:: python - async with db.collection.watch() as stream: - async for change in stream: + with db.collection.watch() as stream: + for change in stream: print(change) - The :class:`~pymongo.synchronous.change_stream.CollectionChangeStream` iterable + The :class:`~pymongo.change_stream.CollectionChangeStream` iterable blocks until the next change document is returned or an error is raised. If the - :meth:`~pymongo.synchronous.change_stream.CollectionChangeStream.next` method + :meth:`~pymongo.change_stream.CollectionChangeStream.next` method encounters a network error when retrieving a batch from the server, it will automatically attempt to recreate the cursor such that no change events are missed. Any error encountered during the resume @@ -444,8 +444,8 @@ class Collection(common.BaseObject, Generic[_DocumentType]): .. code-block:: python try: - async with db.collection.watch([{"$match": {"operationType": "insert"}}]) as stream: - async for insert_change in stream: + with db.coll.watch([{"$match": {"operationType": "insert"}}]) as stream: + for insert_change in stream: print(insert_change) except pymongo.errors.PyMongoError: # The ChangeStream encountered an unrecoverable error or the @@ -502,7 +502,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): command. :param show_expanded_events: Include expanded events such as DDL events like `dropIndexes`. - :return: A :class:`~pymongo.synchronous.change_stream.CollectionChangeStream` cursor. + :return: A :class:`~pymongo.change_stream.CollectionChangeStream` cursor. .. versionchanged:: 4.3 Added `show_expanded_events` parameter. @@ -818,12 +818,12 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> InsertOneResult: """Insert a single document. - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 0 - >>> result = await db.test.insert_one({'x': 1}) + >>> result = db.test.insert_one({'x': 1}) >>> result.inserted_id ObjectId('54f112defba522406c9cc208') - >>> await db.test.find_one({'x': 1}) + >>> db.test.find_one({'x': 1}) {'x': 1, '_id': ObjectId('54f112defba522406c9cc208')} :param document: The document to insert. Must be a mutable mapping @@ -884,12 +884,12 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> InsertManyResult: """Insert an iterable of documents. - >>> await db.test.count_documents({}) + >>> db.test.count_documents({}) 0 - >>> result = await db.test.insert_many([{'x': i} for i in range(2)]) - >>> await result.inserted_ids + >>> result = db.test.insert_many([{'x': i} for i in range(2)]) + >>> result.inserted_ids [ObjectId('54f113fffba522406c9cc20e'), ObjectId('54f113fffba522406c9cc20f')] - >>> await db.test.count_documents({}) + >>> db.test.count_documents({}) 2 :param documents: A iterable of documents to insert. @@ -1098,16 +1098,16 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> UpdateResult: """Replace a single document matching the filter. - >>> async for doc in db.test.find({}): + >>> for doc in db.test.find({}): ... print(doc) ... {'x': 1, '_id': ObjectId('54f4c5befba5220aa4d6dee7')} - >>> result = await db.test.replace_one({'x': 1}, {'y': 1}) + >>> result = db.test.replace_one({'x': 1}, {'y': 1}) >>> result.matched_count 1 >>> result.modified_count 1 - >>> async for doc in db.test.find({}): + >>> for doc in db.test.find({}): ... print(doc) ... {'y': 1, '_id': ObjectId('54f4c5befba5220aa4d6dee7')} @@ -1115,14 +1115,14 @@ class Collection(common.BaseObject, Generic[_DocumentType]): The *upsert* option can be used to insert a new document if a matching document does not exist. - >>> result = await db.test.replace_one({'x': 1}, {'x': 1}, True) + >>> result = db.test.replace_one({'x': 1}, {'x': 1}, True) >>> result.matched_count 0 >>> result.modified_count 0 >>> result.upserted_id ObjectId('54f11e5c8891e756a6e1abd4') - >>> await db.test.find_one({'x': 1}) + >>> db.test.find_one({'x': 1}) {'x': 1, '_id': ObjectId('54f11e5c8891e756a6e1abd4')} :param filter: A query that matches the document to replace. @@ -1201,18 +1201,18 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> UpdateResult: """Update a single document matching the filter. - >>> async for doc in db.test.find(): + >>> for doc in db.test.find(): ... print(doc) ... {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} - >>> result = await db.test.update_one({'x': 1}, {'$inc': {'x': 3}}) + >>> result = db.test.update_one({'x': 1}, {'$inc': {'x': 3}}) >>> result.matched_count 1 >>> result.modified_count 1 - >>> async for doc in db.test.find(): + >>> for doc in db.test.find(): ... print(doc) ... {'x': 4, '_id': 0} @@ -1222,14 +1222,14 @@ class Collection(common.BaseObject, Generic[_DocumentType]): If ``upsert=True`` and no documents match the filter, create a new document based on the filter criteria and update modifications. - >>> result = await db.test.update_one({'x': -10}, {'$inc': {'x': 3}}, upsert=True) + >>> result = db.test.update_one({'x': -10}, {'$inc': {'x': 3}}, upsert=True) >>> result.matched_count 0 >>> result.modified_count 0 >>> result.upserted_id ObjectId('626a678eeaa80587d4bb3fb7') - >>> await db.test.find_one(result.upserted_id) + >>> db.test.find_one(result.upserted_id) {'_id': ObjectId('626a678eeaa80587d4bb3fb7'), 'x': -7} :param filter: A query that matches the document to update. @@ -1314,18 +1314,18 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> UpdateResult: """Update one or more documents that match the filter. - >>> async for doc in db.test.find(): + >>> for doc in db.test.find(): ... print(doc) ... {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} - >>> result = await db.test.update_many({'x': 1}, {'$inc': {'x': 3}}) + >>> result = db.test.update_many({'x': 1}, {'$inc': {'x': 3}}) >>> result.matched_count 3 >>> result.modified_count 3 - >>> async for doc in db.test.find(): + >>> for doc in db.test.find(): ... print(doc) ... {'x': 4, '_id': 0} @@ -1417,8 +1417,8 @@ class Collection(common.BaseObject, Generic[_DocumentType]): The following two calls are equivalent: - >>> await db.foo.drop() - >>> await db.drop_collection("foo") + >>> db.foo.drop() + >>> db.drop_collection("foo") .. versionchanged:: 4.2 Added ``encrypted_fields`` parameter. @@ -1550,12 +1550,12 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> DeleteResult: """Delete a single document matching the filter. - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 3 - >>> result = await db.test.delete_one({'x': 1}) + >>> result = db.test.delete_one({'x': 1}) >>> result.deleted_count 1 - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 2 :param filter: A query that matches the document to delete. @@ -1615,12 +1615,12 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> DeleteResult: """Delete one or more documents matching the filter. - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 3 - >>> result = await db.test.delete_many({'x': 1}) + >>> result = db.test.delete_many({'x': 1}) >>> result.deleted_count 3 - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 0 :param filter: A query that matches the documents to delete. @@ -1694,7 +1694,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): :: code-block: python - >>> await collection.find_one(max_time_ms=100) + >>> collection.find_one(max_time_ms=100) """ if filter is not None and not isinstance(filter, abc.Mapping): @@ -1904,8 +1904,8 @@ class Collection(common.BaseObject, Generic[_DocumentType]): :mod:`bson` module. >>> import bson - >>> cursor = await db.test.find_raw_batches() - >>> async for batch in cursor: + >>> cursor = db.test.find_raw_batches() + >>> for batch in cursor: ... print(bson.decode_all(batch)) .. note:: find_raw_batches does not support auto encryption. @@ -2133,7 +2133,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): >>> index1 = IndexModel([("hello", DESCENDING), ... ("world", ASCENDING)], name="hello_world") >>> index2 = IndexModel([("goodbye", DESCENDING)]) - >>> await db.test.create_indexes([index1, index2]) + >>> db.test.create_indexes([index1, index2]) ["hello_world", "goodbye_-1"] :param indexes: A list of :class:`~pymongo.operations.IndexModel` @@ -2232,18 +2232,18 @@ class Collection(common.BaseObject, Generic[_DocumentType]): To create a single key ascending index on the key ``'mike'`` we just use a string argument:: - >>> await my_collection.create_index("mike") + >>> my_collection.create_index("mike") For a compound index on ``'mike'`` descending and ``'eliot'`` ascending we need to use a list of tuples:: - >>> await my_collection.create_index([("mike", pymongo.DESCENDING), + >>> my_collection.create_index([("mike", pymongo.DESCENDING), ... "eliot"]) All optional index creation parameters should be passed as keyword arguments to this method. For example:: - >>> await my_collection.create_index([("mike", pymongo.DESCENDING)], + >>> my_collection.create_index([("mike", pymongo.DESCENDING)], ... background=True) Valid options include, but are not limited to: @@ -2448,7 +2448,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> CommandCursor[MutableMapping[str, Any]]: """Get a cursor over the index documents for this collection. - >>> async for index in db.test.list_indexes(): + >>> for index in db.test.list_indexes(): ... print(index) ... SON([('v', 2), ('key', SON([('_id', 1)])), ('name', '_id_')]) @@ -2957,9 +2957,9 @@ class Collection(common.BaseObject, Generic[_DocumentType]): :mod:`bson` module. >>> import bson - >>> cursor = await db.test.aggregate_raw_batches([ + >>> cursor = db.test.aggregate_raw_batches([ ... {'$project': {'x': {'$multiply': [2, '$x']}}}]) - >>> async for batch in cursor: + >>> for batch in cursor: ... print(bson.decode_all(batch)) .. note:: aggregate_raw_batches does not support auto encryption. @@ -3217,28 +3217,28 @@ class Collection(common.BaseObject, Generic[_DocumentType]): ) -> _DocumentType: """Finds a single document and deletes it, returning the document. - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 2 - >>> await db.test.find_one_and_delete({'x': 1}) + >>> db.test.find_one_and_delete({'x': 1}) {'x': 1, '_id': ObjectId('54f4e12bfba5220aa4d6dee8')} - >>> await db.test.count_documents({'x': 1}) + >>> db.test.count_documents({'x': 1}) 1 If multiple documents match *filter*, a *sort* can be applied. - >>> async for doc in db.test.find({'x': 1}): + >>> for doc in db.test.find({'x': 1}): ... print(doc) ... {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} - >>> await db.test.find_one_and_delete( + >>> db.test.find_one_and_delete( ... {'x': 1}, sort=[('_id', pymongo.DESCENDING)]) {'x': 1, '_id': 2} The *projection* option can be used to limit the fields returned. - >>> await db.test.find_one_and_delete({'x': 1}, projection={'_id': False}) + >>> db.test.find_one_and_delete({'x': 1}, projection={'_id': False}) {'x': 1} :param filter: A query that matches the document to delete. @@ -3314,15 +3314,15 @@ class Collection(common.BaseObject, Generic[_DocumentType]): :meth:`find_one_and_update` by replacing the document matched by *filter*, rather than modifying the existing document. - >>> async for doc in db.test.find({}): + >>> for doc in db.test.find({}): ... print(doc) ... {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} - >>> await db.test.find_one_and_replace({'x': 1}, {'y': 1}) + >>> db.test.find_one_and_replace({'x': 1}, {'y': 1}) {'x': 1, '_id': 0} - >>> async for doc in db.test.find({}): + >>> for doc in db.test.find({}): ... print(doc) ... {'y': 1, '_id': 0} @@ -3418,13 +3418,13 @@ class Collection(common.BaseObject, Generic[_DocumentType]): """Finds a single document and updates it, returning either the original or the updated document. - >>> await db.test.find_one_and_update( + >>> db.test.find_one_and_update( ... {'_id': 665}, {'$inc': {'count': 1}, '$set': {'done': True}}) {'_id': 665, 'done': False, 'count': 25}} Returns ``None`` if no document matches the filter. - >>> await db.test.find_one_and_update( + >>> db.test.find_one_and_update( ... {'_exists': False}, {'$inc': {'count': 1}}) When the filter matches, by default :meth:`find_one_and_update` @@ -3434,7 +3434,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): option. >>> from pymongo import ReturnDocument - >>> await db.example.find_one_and_update( + >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... return_document=ReturnDocument.AFTER) @@ -3442,7 +3442,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): You can limit the fields returned with the *projection* option. - >>> await db.example.find_one_and_update( + >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... projection={'seq': True, '_id': False}, @@ -3452,9 +3452,9 @@ class Collection(common.BaseObject, Generic[_DocumentType]): The *upsert* option can be used to create the document if it doesn't already exist. - >>> await db.example.delete_many({}).deleted_count + >>> (db.example.delete_many({})).deleted_count 1 - >>> await db.example.find_one_and_update( + >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... projection={'seq': True, '_id': False}, @@ -3464,12 +3464,12 @@ class Collection(common.BaseObject, Generic[_DocumentType]): If multiple documents match *filter*, a *sort* can be applied. - >>> async for doc in db.test.find({'done': True}): + >>> for doc in db.test.find({'done': True}): ... print(doc) ... {'_id': 665, 'done': True, 'result': {'count': 26}} {'_id': 701, 'done': True, 'result': {'count': 17}} - >>> await db.test.find_one_and_update( + >>> db.test.find_one_and_update( ... {'done': True}, ... {'$set': {'final': True}}, ... sort=[('_id', pymongo.DESCENDING)]) diff --git a/pymongo/synchronous/command_cursor.py b/pymongo/synchronous/command_cursor.py index 1cd4d8694..86fa69dcb 100644 --- a/pymongo/synchronous/command_cursor.py +++ b/pymongo/synchronous/command_cursor.py @@ -164,7 +164,7 @@ class CommandCursor(Generic[_DocumentType]): Even if :attr:`alive` is ``True``, :meth:`next` can raise :exc:`StopIteration`. Best to use a for loop:: - async for doc in collection.aggregate(pipeline): + for doc in collection.aggregate(pipeline): print(doc) .. note:: :attr:`alive` can be True while iterating a cursor from @@ -382,11 +382,11 @@ class CommandCursor(Generic[_DocumentType]): self.close() def to_list(self) -> list[_DocumentType]: - """Converts the contents of this cursor to a list more efficiently than ``[doc async for doc in cursor]``. + """Converts the contents of this cursor to a list more efficiently than ``[doc for doc in cursor]``. To use:: - >>> await cursor.to_list() + >>> cursor.to_list() If the cursor is empty or has no more results, an empty list will be returned. diff --git a/pymongo/synchronous/cursor.py b/pymongo/synchronous/cursor.py index 652af606c..1595ce40b 100644 --- a/pymongo/synchronous/cursor.py +++ b/pymongo/synchronous/cursor.py @@ -527,7 +527,7 @@ class Cursor(Generic[_DocumentType]): def max_await_time_ms(self, max_await_time_ms: Optional[int]) -> Cursor[_DocumentType]: """Specifies a time limit for a getMore operation on a - :attr:`~pymongo.cursor_shared.CursorType.TAILABLE_AWAIT` cursor. For all other + :attr:`~pymongo.cursor.CursorType.TAILABLE_AWAIT` cursor. For all other types of cursor max_await_time_ms is ignored. Raises :exc:`TypeError` if `max_await_time_ms` is not an integer or @@ -712,27 +712,27 @@ class Cursor(Generic[_DocumentType]): Pass a field name and a direction, either :data:`~pymongo.ASCENDING` or :data:`~pymongo.DESCENDING`.:: - async for doc in collection.find().sort('field', pymongo.ASCENDING): + for doc in collection.find().sort('field', pymongo.ASCENDING): print(doc) To sort by multiple fields, pass a list of (key, direction) pairs. If just a name is given, :data:`~pymongo.ASCENDING` will be inferred:: - async for doc in collection.find().sort([ + for doc in collection.find().sort([ 'field1', ('field2', pymongo.DESCENDING)]): print(doc) Text search results can be sorted by relevance:: - cursor = await db.test.find( + cursor = db.test.find( {'$text': {'$search': 'some words'}}, {'score': {'$meta': 'textScore'}}) # Sort by 'score' field. cursor.sort([('score', {'$meta': 'textScore'})]) - async for doc in cursor: + for doc in cursor: print(doc) For more advanced text search functionality, see MongoDB's @@ -831,7 +831,7 @@ class Cursor(Generic[_DocumentType]): to the object currently being scanned. For example:: # Find all documents where field "a" is less than "b" plus "c". - async for doc in db.test.find().where('this.a < (this.b + this.c)'): + for doc in db.test.find().where('this.a < (this.b + this.c)'): print(doc) Raises :class:`TypeError` if `code` is not an instance of @@ -904,7 +904,7 @@ class Cursor(Generic[_DocumentType]): With regular cursors, simply use a for loop instead of :attr:`alive`:: - async for doc in collection.find(): + for doc in collection.find(): print(doc) .. note:: Even if :attr:`alive` is True, :meth:`next` can raise @@ -1285,11 +1285,11 @@ class Cursor(Generic[_DocumentType]): self.close() def to_list(self) -> list[_DocumentType]: - """Converts the contents of this cursor to a list more efficiently than ``[doc async for doc in cursor]``. + """Converts the contents of this cursor to a list more efficiently than ``[doc for doc in cursor]``. To use:: - >>> await cursor.to_list() + >>> cursor.to_list() If the cursor is empty or has no more results, an empty list will be returned. diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 3b3a91095..93a998528 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -337,21 +337,21 @@ class Database(common.BaseObject, Generic[_DocumentType]): Performs an aggregation with an implicit initial ``$changeStream`` stage and returns a - :class:`~pymongo.synchronous.change_stream.DatabaseChangeStream` cursor which + :class:`~pymongo.change_stream.DatabaseChangeStream` cursor which iterates over changes on all collections in this database. Introduced in MongoDB 4.0. .. code-block:: python - async with db.watch() as stream: - async for change in stream: + with db.watch() as stream: + for change in stream: print(change) - The :class:`~pymongo.synchronous.change_stream.DatabaseChangeStream` iterable + The :class:`~pymongo.change_stream.DatabaseChangeStream` iterable blocks until the next change document is returned or an error is raised. If the - :meth:`~pymongo.synchronous.change_stream.DatabaseChangeStream.next` method + :meth:`~pymongo.change_stream.DatabaseChangeStream.next` method encounters a network error when retrieving a batch from the server, it will automatically attempt to recreate the cursor such that no change events are missed. Any error encountered during the resume @@ -360,8 +360,8 @@ class Database(common.BaseObject, Generic[_DocumentType]): .. code-block:: python try: - async with db.watch([{"$match": {"operationType": "insert"}}]) as stream: - async for insert_change in stream: + with db.watch([{"$match": {"operationType": "insert"}}]) as stream: + for insert_change in stream: print(insert_change) except pymongo.errors.PyMongoError: # The ChangeStream encountered an unrecoverable error or the @@ -409,7 +409,7 @@ class Database(common.BaseObject, Generic[_DocumentType]): command. :param show_expanded_events: Include expanded events such as DDL events like `dropIndexes`. - :return: A :class:`~pymongo.synchronous.change_stream.DatabaseChangeStream` cursor. + :return: A :class:`~pymongo.change_stream.DatabaseChangeStream` cursor. .. versionchanged:: 4.3 Added `show_expanded_events` parameter. @@ -810,23 +810,23 @@ class Database(common.BaseObject, Generic[_DocumentType]): For example, a command like ``{buildinfo: 1}`` can be sent using: - >>> await db.command("buildinfo") + >>> db.command("buildinfo") OR - >>> await db.command({"buildinfo": 1}) + >>> db.command({"buildinfo": 1}) For a command where the value matters, like ``{count: collection_name}`` we can do: - >>> await db.command("count", collection_name) + >>> db.command("count", collection_name) OR - >>> await db.command({"count": collection_name}) + >>> db.command({"count": collection_name}) For commands that take additional arguments we can use kwargs. So ``{count: collection_name, query: query}`` becomes: - >>> await db.command("count", collection_name, query=query) + >>> db.command("count", collection_name, query=query) OR - >>> await db.command({"count": collection_name, "query": query}) + >>> db.command({"count": collection_name, "query": query}) :param command: document representing the command to be issued, or the name of the command (for simple commands only). diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index bd14311b5..41b4db4f1 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -62,6 +62,7 @@ from pymongo.client_options import ClientOptions from pymongo.errors import ( AutoReconnect, BulkWriteError, + ClientBulkWriteException, ConfigurationError, ConnectionFailure, InvalidOperation, @@ -76,12 +77,22 @@ from pymongo.lock import _HAS_REGISTER_AT_FORK, _create_lock, _release_locks from pymongo.logger import _CLIENT_LOGGER, _log_or_warn from pymongo.message import _CursorAddress, _GetMore, _Query from pymongo.monitoring import ConnectionClosedReason -from pymongo.operations import _Op +from pymongo.operations import ( + DeleteMany, + DeleteOne, + InsertOne, + ReplaceOne, + UpdateMany, + UpdateOne, + _Op, +) from pymongo.read_preferences import ReadPreference, _ServerMode +from pymongo.results import ClientBulkWriteResult from pymongo.server_selectors import writable_server_selector from pymongo.server_type import SERVER_TYPE from pymongo.synchronous import client_session, database, periodic_executor from pymongo.synchronous.change_stream import ChangeStream, ClusterChangeStream +from pymongo.synchronous.client_bulk import _ClientBulk from pymongo.synchronous.client_session import _EmptyServerSession from pymongo.synchronous.command_cursor import CommandCursor from pymongo.synchronous.settings import TopologySettings @@ -127,6 +138,15 @@ _ReadCall = Callable[ _IS_SYNC = True +_WriteOp = Union[ + InsertOne, + DeleteOne, + DeleteMany, + ReplaceOne, + UpdateOne, + UpdateMany, +] + class MongoClient(common.BaseObject, Generic[_DocumentType]): HOST = "localhost" @@ -1715,7 +1735,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): retryable: bool, func: _WriteCall[T], session: Optional[ClientSession], - bulk: Optional[_Bulk], + bulk: Optional[Union[_Bulk, _ClientBulk]], operation: str, operation_id: Optional[int] = None, ) -> T: @@ -1745,7 +1765,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): self, func: _WriteCall[T] | _ReadCall[T], session: Optional[ClientSession], - bulk: Optional[_Bulk], + bulk: Optional[Union[_Bulk, _ClientBulk]], operation: str, is_read: bool = False, address: Optional[_Address] = None, @@ -1828,7 +1848,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): func: _WriteCall[T], session: Optional[ClientSession], operation: str, - bulk: Optional[_Bulk] = None, + bulk: Optional[Union[_Bulk, _ClientBulk]] = None, operation_id: Optional[int] = None, ) -> T: """Execute an operation with consecutive retries if possible @@ -2193,10 +2213,134 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): session=session, ) + @_csot.apply + def bulk_write( + self, + models: Sequence[_WriteOp[_DocumentType]], + session: Optional[ClientSession] = None, + ordered: bool = True, + verbose_results: bool = False, + bypass_document_validation: Optional[bool] = None, + comment: Optional[Any] = None, + let: Optional[Mapping] = None, + write_concern: Optional[WriteConcern] = None, + ) -> ClientBulkWriteResult: + """Send a batch of write operations, potentially across multiple namespaces, to the server. + + Requests are passed as a list of write operation instances ( + :class:`~pymongo.operations.InsertOne`, + :class:`~pymongo.operations.UpdateOne`, + :class:`~pymongo.operations.UpdateMany`, + :class:`~pymongo.operations.ReplaceOne`, + :class:`~pymongo.operations.DeleteOne`, or + :class:`~pymongo.operations.DeleteMany`). + + >>> for doc in db.test.find({}): + ... print(doc) + ... + {'x': 1, '_id': ObjectId('54f62e60fba5226811f634ef')} + {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} + ... + >>> for doc in db.coll.find({}): + ... print(doc) + ... + {'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')} + ... + >>> # DeleteMany, UpdateOne, and UpdateMany are also available. + >>> from pymongo import InsertOne, DeleteOne, ReplaceOne + >>> models = [InsertOne(namespace="db.test", document={'y': 1}), + ... DeleteOne(namespace="db.test", filter={'x': 1}), + ... InsertOne(namespace="db.coll", document={'y': 2}), + ... ReplaceOne(namespace="db.test", filter={'w': 1}, replacement={'z': 1}, upsert=True)] + >>> result = client.bulk_write(models=models) + >>> result.inserted_count + 2 + >>> result.deleted_count + 1 + >>> result.modified_count + 0 + >>> result.upserted_ids + {3: ObjectId('54f62ee28891e756a6e1abd5')} + >>> for doc in db.test.find({}): + ... print(doc) + ... + {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} + {'y': 1, '_id': ObjectId('54f62ee2fba5226811f634f1')} + {'z': 1, '_id': ObjectId('54f62ee28891e756a6e1abd5')} + ... + >>> for doc in db.coll.find({}): + ... print(doc) + ... + {'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')} + {'y': 2, '_id': ObjectId('507f1f77bcf86cd799439012')} + + :param models: A list of write operation instances. + :param session: (optional) An instance of + :class:`~pymongo.client_session.ClientSession`. + :param ordered: If ``True`` (the default), requests will be + performed on the server serially, in the order provided. If an error + occurs all remaining operations are aborted. If ``False``, requests + will be still performed on the server serially, in the order provided, + but all operations will be attempted even if any errors occur. + :param verbose_results: If ``True``, detailed results for each + successful operation will be included in the returned + :class:`~pymongo.results.ClientBulkWriteResult`. Default is ``False``. + :param bypass_document_validation: (optional) If ``True``, allows the + write to opt-out of document level validation. Default is ``False``. + :param comment: (optional) A user-provided comment to attach to this + command. + :param let: (optional) Map of parameter names and values. Values must be + constant or closed expressions that do not reference document + fields. Parameters can then be accessed as variables in an + aggregate expression context (e.g. "$$var"). + :param write_concern: (optional) The write concern to use for this bulk write. + + :return: An instance of :class:`~pymongo.results.ClientBulkWriteResult`. + + .. seealso:: :ref:`writes-and-ids` + + .. note:: requires MongoDB server version 8.0+. + + .. versionadded:: 4.9 + """ + if self._options.auto_encryption_opts: + raise InvalidOperation( + "MongoClient.bulk_write does not currently support automatic encryption" + ) + + if session and session.in_transaction: + # Inherit the transaction write concern. + if write_concern: + raise InvalidOperation("Cannot set write concern after starting a transaction") + write_concern = session._transaction.opts.write_concern # type: ignore[union-attr] + else: + # Inherit the client's write concern if none is provided. + if not write_concern: + write_concern = self.write_concern + + common.validate_list("models", models) + + blk = _ClientBulk( + self, + write_concern=write_concern, # type: ignore[arg-type] + ordered=ordered, + bypass_document_validation=bypass_document_validation, + comment=comment, + let=let, + verbose_results=verbose_results, + ) + for model in models: + try: + model._add_to_client_bulk(blk) + except AttributeError: + raise TypeError(f"{model!r} is not a valid request") from None + + return blk.execute(session, _Op.BULK_WRITE) + def _retryable_error_doc(exc: PyMongoError) -> Optional[Mapping[str, Any]]: """Return the server response from PyMongo exception or None.""" - if isinstance(exc, BulkWriteError): + if isinstance(exc, (BulkWriteError, ClientBulkWriteException)): # Check the last writeConcernError to determine if this # BulkWriteError is retryable. wces = exc.details["writeConcernErrors"] @@ -2231,10 +2375,14 @@ def _add_retryable_write_error(exc: PyMongoError, max_wire_version: int, is_mong # Connection errors are always retryable except NotPrimaryError and WaitQueueTimeoutError which is # handled above. - if isinstance(exc, ConnectionFailure) and not isinstance( - exc, (NotPrimaryError, WaitQueueTimeoutError) + if isinstance(exc, ClientBulkWriteException): + exc_to_check = exc.error + else: + exc_to_check = exc + if isinstance(exc_to_check, ConnectionFailure) and not isinstance( + exc_to_check, (NotPrimaryError, WaitQueueTimeoutError) ): - exc._add_error_label("RetryableWriteError") + exc_to_check._add_error_label("RetryableWriteError") class _MongoClientErrorHandler: @@ -2279,6 +2427,8 @@ class _MongoClientErrorHandler: return self.handled = True if self.session: + if isinstance(exc_val, ClientBulkWriteException): + exc_val = exc_val.error if isinstance(exc_val, ConnectionFailure): if self.session.in_transaction: exc_val._add_error_label("TransientTransactionError") @@ -2290,7 +2440,7 @@ class _MongoClientErrorHandler: ): self.session._unpin() err_ctx = _ErrorContext( - exc_val, + exc_val, # type: ignore[arg-type] self.max_wire_version, self.sock_generation, self.completed_handshake, @@ -2317,7 +2467,7 @@ class _ClientConnectionRetryable(Generic[T]): self, mongo_client: MongoClient, func: _WriteCall[T] | _ReadCall[T], - bulk: Optional[_Bulk], + bulk: Optional[Union[_Bulk, _ClientBulk]], operation: str, is_read: bool = False, session: Optional[ClientSession] = None, @@ -2394,7 +2544,10 @@ class _ClientConnectionRetryable(Generic[T]): if not self._is_read: if not self._retryable: raise - retryable_write_error_exc = exc.has_error_label("RetryableWriteError") + if isinstance(exc, ClientBulkWriteException) and exc.error: + retryable_write_error_exc = exc.error.has_error_label("RetryableWriteError") + else: + retryable_write_error_exc = exc.has_error_label("RetryableWriteError") if retryable_write_error_exc: assert self._session self._session._unpin() diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index 8106c1922..e3d1f7bf2 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -48,6 +48,15 @@ def _sanitize(error: Exception) -> None: error.__cause__ = None +def _monotonic_duration(start: float) -> float: + """Return the duration since the given start time. + + Accounts for buggy platforms where time.monotonic() is not monotonic. + See PYTHON-4600. + """ + return max(0.0, time.monotonic() - start) + + class MonitorBase: def __init__(self, topology: Topology, name: str, interval: int, min_interval: float): """Base class to do periodic work on a background thread. @@ -247,7 +256,7 @@ class Monitor(MonitorBase): _sanitize(error) sd = self._server_description address = sd.address - duration = time.monotonic() - start + duration = _monotonic_duration(start) if self._publish: awaited = bool(self._stream and sd.is_server_type_known and sd.topology_version) assert self._listeners is not None @@ -317,7 +326,8 @@ class Monitor(MonitorBase): else: # New connection handshake or polling hello (MongoDB <4.4). response = conn._hello(cluster_time, None, None) - return response, time.monotonic() - start + duration = _monotonic_duration(start) + return response, duration class SrvMonitor(MonitorBase): @@ -441,7 +451,7 @@ class _RttMonitor(MonitorBase): raise Exception("_RttMonitor closed") start = time.monotonic() conn.hello() - return time.monotonic() - start + return _monotonic_duration(start) # Close monitors to cancel any in progress streaming checks before joining diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index b2c102ae0..8542f67bb 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -643,6 +643,7 @@ class Topology: :exc:`~.errors.InvalidOperation`. """ with self._lock: + old_td = self._description for server in self._servers.values(): server.close() @@ -662,9 +663,30 @@ class Topology: # Publish only after releasing the lock. if self._publish_tp: assert self._events is not None + self._description = TopologyDescription( + TOPOLOGY_TYPE.Unknown, + {}, + self._description.replica_set_name, + self._description.max_set_version, + self._description.max_election_id, + self._description._topology_settings, + ) + self._events.put( + ( + self._listeners.publish_topology_description_changed, + ( + old_td, + self._description, + self._topology_id, + ), + ) + ) self._events.put((self._listeners.publish_topology_closed, (self._topology_id,))) if self._publish_server or self._publish_tp: + # Make sure the events executor thread is fully closed before publishing the remaining events self.__events_executor.close() + self.__events_executor.join(1) + process_events_queue(weakref.ref(self._events)) # type: ignore[arg-type] @property def description(self) -> TopologyDescription: diff --git a/pymongo/typings.py b/pymongo/typings.py index 9f6d7b166..68962eb54 100644 --- a/pymongo/typings.py +++ b/pymongo/typings.py @@ -30,11 +30,13 @@ from bson.typings import _DocumentOut, _DocumentType, _DocumentTypeArg if TYPE_CHECKING: from pymongo.asynchronous.bulk import _AsyncBulk + from pymongo.asynchronous.client_bulk import _AsyncClientBulk from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection from pymongo.collation import Collation from pymongo.synchronous.bulk import _Bulk + from pymongo.synchronous.client_bulk import _ClientBulk from pymongo.synchronous.client_session import ClientSession from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection @@ -53,6 +55,7 @@ _AgnosticMongoClient = Union["AsyncMongoClient", "MongoClient"] _AgnosticConnection = Union["AsyncConnection", "Connection"] _AgnosticClientSession = Union["AsyncClientSession", "ClientSession"] _AgnosticBulk = Union["_AsyncBulk", "_Bulk"] +_AgnosticClientBulk = Union["_AsyncClientBulk", "_ClientBulk"] def strip_optional(elem: Optional[_T]) -> _T: diff --git a/requirements/docs.txt b/requirements/docs.txt index ce8682c07..16b274686 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -2,4 +2,5 @@ sphinx>=5.3,<8 sphinx_rtd_theme>=2,<3 readthedocs-sphinx-search~=0.3 sphinxcontrib-shellcheck>=1,<2 +sphinx-autobuild>=2020.9.1 furo==2023.9.10 diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py new file mode 100644 index 000000000..f55b3082b --- /dev/null +++ b/test/asynchronous/test_client_bulk_write.py @@ -0,0 +1,571 @@ +# Copyright 2024-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the client bulk write API.""" +from __future__ import annotations + +import sys + +sys.path[0:0] = [""] + +from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest +from test.utils import ( + OvertCommandListener, + async_rs_or_single_client, +) + +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts +from pymongo.errors import ( + ClientBulkWriteException, + DocumentTooLarge, + InvalidOperation, + NetworkTimeout, +) +from pymongo.monitoring import * +from pymongo.operations import * +from pymongo.write_concern import WriteConcern + +_IS_SYNC = False + + +class TestClientBulkWrite(AsyncIntegrationTest): + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_returns_error_if_no_namespace_provided(self): + client = await async_rs_or_single_client() + self.addAsyncCleanup(client.aclose) + + models = [InsertOne(document={"a": "b"})] + with self.assertRaises(InvalidOperation) as context: + await client.bulk_write(models=models) + self.assertIn( + "MongoClient.bulk_write requires a namespace to be provided for each write operation", + context.exception._message, + ) + + +# https://github.com/mongodb/specifications/tree/master/source/crud/tests +class TestClientBulkWriteCRUD(AsyncIntegrationTest): + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_batch_splits_if_num_operations_too_large(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"] + models = [] + for _ in range(max_write_batch_size + 1): + models.append(InsertOne(namespace="db.coll", document={"a": "b"})) + self.addAsyncCleanup(client.db["coll"].drop) + + result = await client.bulk_write(models=models) + self.assertEqual(result.inserted_count, max_write_batch_size + 1) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + first_event, second_event = bulk_write_events + self.assertEqual(len(first_event.command["ops"]), max_write_batch_size) + self.assertEqual(len(second_event.command["ops"]), 1) + self.assertEqual(first_event.operation_id, second_event.operation_id) + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_batch_splits_if_ops_payload_too_large(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"] + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + + models = [] + num_models = int(max_message_size_bytes / max_bson_object_size + 1) + b_repeated = "b" * (max_bson_object_size - 500) + for _ in range(num_models): + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + self.addAsyncCleanup(client.db["coll"].drop) + + result = await client.bulk_write(models=models) + self.assertEqual(result.inserted_count, num_models) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + first_event, second_event = bulk_write_events + self.assertEqual(len(first_event.command["ops"]), num_models - 1) + self.assertEqual(len(second_event.command["ops"]), 1) + self.assertEqual(first_event.operation_id, second_event.operation_id) + + @async_client_context.require_version_min(8, 0, 0, -24) + @async_client_context.require_failCommand_fail_point + async def test_collects_write_concern_errors_across_batches(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client( + event_listeners=[listener], + retryWrites=False, + ) + self.addAsyncCleanup(client.aclose) + max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"] + + fail_command = { + "configureFailPoint": "failCommand", + "mode": {"times": 2}, + "data": { + "failCommands": ["bulkWrite"], + "writeConcernError": {"code": 91, "errmsg": "Replication is being shut down"}, + }, + } + async with self.fail_point(fail_command): + models = [] + for _ in range(max_write_batch_size + 1): + models.append( + InsertOne( + namespace="db.coll", + document={"a": "b"}, + ) + ) + self.addAsyncCleanup(client.db["coll"].drop) + + with self.assertRaises(ClientBulkWriteException) as context: + await client.bulk_write(models=models) + self.assertEqual(len(context.exception.write_concern_errors), 2) # type: ignore[arg-type] + self.assertIsNotNone(context.exception.partial_result) + self.assertEqual( + context.exception.partial_result.inserted_count, max_write_batch_size + 1 + ) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_collects_write_errors_across_batches_unordered(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + collection = client.db["coll"] + self.addAsyncCleanup(collection.drop) + await collection.drop() + await collection.insert_one(document={"_id": 1}) + + max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"] + models = [] + for _ in range(max_write_batch_size + 1): + models.append( + InsertOne( + namespace="db.coll", + document={"_id": 1}, + ) + ) + + with self.assertRaises(ClientBulkWriteException) as context: + await client.bulk_write(models=models, ordered=False) + self.assertEqual(len(context.exception.write_errors), max_write_batch_size + 1) # type: ignore[arg-type] + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_collects_write_errors_across_batches_ordered(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + collection = client.db["coll"] + self.addAsyncCleanup(collection.drop) + await collection.drop() + await collection.insert_one(document={"_id": 1}) + + max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"] + models = [] + for _ in range(max_write_batch_size + 1): + models.append( + InsertOne( + namespace="db.coll", + document={"_id": 1}, + ) + ) + + with self.assertRaises(ClientBulkWriteException) as context: + await client.bulk_write(models=models, ordered=True) + self.assertEqual(len(context.exception.write_errors), 1) # type: ignore[arg-type] + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 1) + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_handles_cursor_requiring_getMore(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + collection = client.db["coll"] + self.addAsyncCleanup(collection.drop) + await collection.drop() + + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + models = [] + a_repeated = "a" * (max_bson_object_size // 2) + b_repeated = "b" * (max_bson_object_size // 2) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": a_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": b_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + + result = await client.bulk_write(models=models, verbose_results=True) + self.assertEqual(result.upserted_count, 2) + self.assertEqual(len(result.update_results), 2) + + get_more_event = False + for event in listener.started_events: + if event.command_name == "getMore": + get_more_event = True + self.assertTrue(get_more_event) + + @async_client_context.require_version_min(8, 0, 0, -24) + @async_client_context.require_no_standalone + async def test_handles_cursor_requiring_getMore_within_transaction(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + collection = client.db["coll"] + self.addAsyncCleanup(collection.drop) + await collection.drop() + + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + async with client.start_session() as session: + await session.start_transaction() + models = [] + a_repeated = "a" * (max_bson_object_size // 2) + b_repeated = "b" * (max_bson_object_size // 2) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": a_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": b_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + result = await client.bulk_write(models=models, session=session, verbose_results=True) + + self.assertEqual(result.upserted_count, 2) + self.assertEqual(len(result.update_results), 2) + + get_more_event = False + for event in listener.started_events: + if event.command_name == "getMore": + get_more_event = True + self.assertTrue(get_more_event) + + @async_client_context.require_version_min(8, 0, 0, -24) + @async_client_context.require_failCommand_fail_point + async def test_handles_getMore_error(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + collection = client.db["coll"] + self.addAsyncCleanup(collection.drop) + await collection.drop() + + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + fail_command = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": {"failCommands": ["getMore"], "errorCode": 8}, + } + async with self.fail_point(fail_command): + models = [] + a_repeated = "a" * (max_bson_object_size // 2) + b_repeated = "b" * (max_bson_object_size // 2) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": a_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": b_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + + with self.assertRaises(ClientBulkWriteException) as context: + await client.bulk_write(models=models, verbose_results=True) + self.assertIsNotNone(context.exception.error) + self.assertEqual(context.exception.error["code"], 8) + self.assertIsNotNone(context.exception.partial_result) + self.assertEqual(context.exception.partial_result.upserted_count, 2) + self.assertEqual(len(context.exception.partial_result.update_results), 1) + + get_more_event = False + kill_cursors_event = False + for event in listener.started_events: + if event.command_name == "getMore": + get_more_event = True + if event.command_name == "killCursors": + kill_cursors_event = True + self.assertTrue(get_more_event) + self.assertTrue(kill_cursors_event) + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_returns_error_if_unacknowledged_too_large_insert(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + b_repeated = "b" * max_bson_object_size + + # Insert document. + models_insert = [InsertOne(namespace="db.coll", document={"a": b_repeated})] + with self.assertRaises(DocumentTooLarge): + await client.bulk_write(models=models_insert, write_concern=WriteConcern(w=0)) + + # Replace document. + models_replace = [ReplaceOne(namespace="db.coll", filter={}, replacement={"a": b_repeated})] + with self.assertRaises(DocumentTooLarge): + await client.bulk_write(models=models_replace, write_concern=WriteConcern(w=0)) + + async def _setup_namespace_test_models(self): + max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"] + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + + ops_bytes = max_message_size_bytes - 1122 + num_models = ops_bytes // max_bson_object_size + remainder_bytes = ops_bytes % max_bson_object_size + + models = [] + b_repeated = "b" * (max_bson_object_size - 57) + for _ in range(num_models): + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + if remainder_bytes >= 217: + num_models += 1 + b_repeated = "b" * (remainder_bytes - 57) + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + return num_models, models + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_no_batch_splits_if_new_namespace_is_not_too_large(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + num_models, models = await self._setup_namespace_test_models() + models.append( + InsertOne( + namespace="db.coll", + document={"a": "b"}, + ) + ) + self.addAsyncCleanup(client.db["coll"].drop) + + # No batch splitting required. + result = await client.bulk_write(models=models) + self.assertEqual(result.inserted_count, num_models + 1) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + + self.assertEqual(len(bulk_write_events), 1) + event = bulk_write_events[0] + + self.assertEqual(len(event.command["ops"]), num_models + 1) + self.assertEqual(len(event.command["nsInfo"]), 1) + self.assertEqual(event.command["nsInfo"][0]["ns"], "db.coll") + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_batch_splits_if_new_namespace_is_too_large(self): + listener = OvertCommandListener() + client = await async_rs_or_single_client(event_listeners=[listener]) + self.addAsyncCleanup(client.aclose) + + num_models, models = await self._setup_namespace_test_models() + c_repeated = "c" * 200 + namespace = f"db.{c_repeated}" + models.append( + InsertOne( + namespace=namespace, + document={"a": "b"}, + ) + ) + self.addAsyncCleanup(client.db["coll"].drop) + self.addAsyncCleanup(client.db[c_repeated].drop) + + # Batch splitting required. + result = await client.bulk_write(models=models) + self.assertEqual(result.inserted_count, num_models + 1) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + + self.assertEqual(len(bulk_write_events), 2) + first_event, second_event = bulk_write_events + + self.assertEqual(len(first_event.command["ops"]), num_models) + self.assertEqual(len(first_event.command["nsInfo"]), 1) + self.assertEqual(first_event.command["nsInfo"][0]["ns"], "db.coll") + + self.assertEqual(len(second_event.command["ops"]), 1) + self.assertEqual(len(second_event.command["nsInfo"]), 1) + self.assertEqual(second_event.command["nsInfo"][0]["ns"], namespace) + + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_returns_error_if_no_writes_can_be_added_to_ops(self): + client = await async_rs_or_single_client() + self.addAsyncCleanup(client.aclose) + + max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"] + + # Document too large. + b_repeated = "b" * max_message_size_bytes + models = [InsertOne(namespace="db.coll", document={"a": b_repeated})] + with self.assertRaises(InvalidOperation) as context: + await client.bulk_write(models=models) + self.assertIn("cannot do an empty bulk write", context.exception._message) + + # Namespace too large. + c_repeated = "c" * max_message_size_bytes + namespace = f"db.{c_repeated}" + models = [InsertOne(namespace=namespace, document={"a": "b"})] + with self.assertRaises(InvalidOperation) as context: + await client.bulk_write(models=models) + self.assertIn("cannot do an empty bulk write", context.exception._message) + + @async_client_context.require_version_min(8, 0, 0, -24) + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + async def test_returns_error_if_auto_encryption_configured(self): + opts = AutoEncryptionOpts( + key_vault_namespace="db.coll", + kms_providers={"aws": {"accessKeyId": "foo", "secretAccessKey": "bar"}}, + ) + client = await async_rs_or_single_client(auto_encryption_opts=opts) + self.addAsyncCleanup(client.aclose) + + models = [InsertOne(namespace="db.coll", document={"a": "b"})] + with self.assertRaises(InvalidOperation) as context: + await client.bulk_write(models=models) + self.assertIn( + "bulk_write does not currently support automatic encryption", context.exception._message + ) + + +# https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/tests/README.md#11-multi-batch-bulkwrites +class TestClientBulkWriteTimeout(AsyncIntegrationTest): + @async_client_context.require_version_min(8, 0, 0, -24) + @async_client_context.require_failCommand_fail_point + async def test_timeout_in_multi_batch_bulk_write(self): + internal_client = await async_rs_or_single_client(timeoutMS=None) + self.addAsyncCleanup(internal_client.aclose) + + collection = internal_client.db["coll"] + self.addAsyncCleanup(collection.drop) + await collection.drop() + + max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"] + max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"] + fail_command = { + "configureFailPoint": "failCommand", + "mode": {"times": 2}, + "data": {"failCommands": ["bulkWrite"], "blockConnection": True, "blockTimeMS": 1010}, + } + async with self.fail_point(fail_command): + models = [] + num_models = int(max_message_size_bytes / max_bson_object_size + 1) + b_repeated = "b" * (max_bson_object_size - 500) + for _ in range(num_models): + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + + listener = OvertCommandListener() + client = await async_rs_or_single_client( + event_listeners=[listener], + readConcernLevel="majority", + readPreference="primary", + timeoutMS=2000, + w="majority", + ) + self.addAsyncCleanup(client.aclose) + with self.assertRaises(ClientBulkWriteException) as context: + await client.bulk_write(models=models) + self.assertIsInstance(context.exception.error, NetworkTimeout) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index 925584b89..833493ce3 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -1380,41 +1380,39 @@ class TestCursor(AsyncIntegrationTest): self.assertEqual("getMore", started[1].command_name) self.assertNotIn("$readPreference", started[1].command) + @async_client_context.require_version_min(4, 0) @async_client_context.require_replica_set async def test_to_list_tailable(self): oplog = self.client.local.oplog.rs last = await oplog.find().sort("$natural", pymongo.DESCENDING).limit(-1).next() ts = last["ts"] - + # Set maxAwaitTimeMS=1 to speed up the test and avoid blocking on the noop writer. c = oplog.find( {"ts": {"$gte": ts}}, cursor_type=pymongo.CursorType.TAILABLE_AWAIT, oplog_replay=True - ) - + ).max_await_time_ms(1) + self.addAsyncCleanup(c.close) docs = await c.to_list() - self.assertGreaterEqual(len(docs), 1) async def test_to_list_empty(self): c = self.db.does_not_exist.find() - docs = await c.to_list() - self.assertEqual([], docs) - @async_client_context.require_replica_set + @async_client_context.require_change_streams async def test_command_cursor_to_list(self): - c = await self.db.test.aggregate([{"$changeStream": {}}]) - + # Set maxAwaitTimeMS=1 to speed up the test. + c = await self.db.test.aggregate([{"$changeStream": {}}], maxAwaitTimeMS=1) + self.addAsyncCleanup(c.close) docs = await c.to_list() - self.assertGreaterEqual(len(docs), 0) - @async_client_context.require_replica_set + @async_client_context.require_change_streams async def test_command_cursor_to_list_empty(self): - c = await self.db.does_not_exist.aggregate([{"$changeStream": {}}]) - + # Set maxAwaitTimeMS=1 to speed up the test. + c = await self.db.does_not_exist.aggregate([{"$changeStream": {}}], maxAwaitTimeMS=1) + self.addAsyncCleanup(c.close) docs = await c.to_list() - self.assertEqual([], docs) diff --git a/test/asynchronous/test_database.py b/test/asynchronous/test_database.py index 0d848458d..c20a74d3d 100644 --- a/test/asynchronous/test_database.py +++ b/test/asynchronous/test_database.py @@ -116,10 +116,10 @@ class TestDatabaseNoConnect(unittest.TestCase): with self.assertRaises(TypeError): _ = db[0] # next fails - with self.assertRaisesRegex(TypeError, "'Database' object is not iterable"): + with self.assertRaisesRegex(TypeError, "'AsyncDatabase' object is not iterable"): _ = next(db) # .next() fails - with self.assertRaisesRegex(TypeError, "'Database' object is not iterable"): + with self.assertRaisesRegex(TypeError, "'AsyncDatabase' object is not iterable"): _ = db.next() # Do not implement typing.Iterable. self.assertNotIsInstance(db, Iterable) diff --git a/test/command_logging/unacknowledged-write.json b/test/command_logging/unacknowledged-write.json index dad0c0a36..0d33c020d 100644 --- a/test/command_logging/unacknowledged-write.json +++ b/test/command_logging/unacknowledged-write.json @@ -5,6 +5,7 @@ { "client": { "id": "client", + "useMultipleMongoses": false, "observeLogMessages": { "command": "debug" } diff --git a/test/command_monitoring/unacknowledged-client-bulkWrite.json b/test/command_monitoring/unacknowledged-client-bulkWrite.json new file mode 100644 index 000000000..1099b6a1e --- /dev/null +++ b/test/command_monitoring/unacknowledged-client-bulkWrite.json @@ -0,0 +1,218 @@ +{ + "description": "unacknowledged-client-bulkWrite", + "schemaVersion": "1.7", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ], + "uriOptions": { + "w": 0 + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "command-monitoring-tests.test" + }, + "tests": [ + { + "description": "A successful mixed client bulkWrite", + "operations": [ + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "command-monitoring-tests.test", + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "namespace": "command-monitoring-tests.test", + "filter": { + "_id": 3 + }, + "update": { + "$set": { + "x": 333 + } + } + } + } + ] + }, + "expectResult": { + "insertedCount": { + "$$unsetOrMatches": 0 + }, + "upsertedCount": { + "$$unsetOrMatches": 0 + }, + "matchedCount": { + "$$unsetOrMatches": 0 + }, + "modifiedCount": { + "$$unsetOrMatches": 0 + }, + "deletedCount": { + "$$unsetOrMatches": 0 + }, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 333 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ], + "expectEvents": [ + { + "client": "client", + "ignoreExtraEvents": true, + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 3 + }, + "updateMods": { + "$set": { + "x": 333 + } + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "command-monitoring-tests.test" + } + ] + } + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite", + "reply": { + "ok": 1, + "nInserted": { + "$$exists": false + }, + "nMatched": { + "$$exists": false + }, + "nModified": { + "$$exists": false + }, + "nUpserted": { + "$$exists": false + }, + "nDeleted": { + "$$exists": false + } + } + } + } + ] + } + ] + } + ] +} diff --git a/test/command_monitoring/unacknowledgedBulkWrite.json b/test/command_monitoring/unacknowledgedBulkWrite.json index 782cb84a5..78ddde767 100644 --- a/test/command_monitoring/unacknowledgedBulkWrite.json +++ b/test/command_monitoring/unacknowledgedBulkWrite.json @@ -5,6 +5,7 @@ { "client": { "id": "client", + "useMultipleMongoses": false, "observeEvents": [ "commandStartedEvent", "commandSucceededEvent", @@ -70,17 +71,7 @@ "object": "collection", "arguments": { "filter": {} - }, - "expectResult": [ - { - "_id": 1, - "x": 11 - }, - { - "_id": "unorderedBulkWriteInsertW0", - "x": 44 - } - ] + } } ], "expectEvents": [ diff --git a/test/command_monitoring/writeConcernError.json b/test/command_monitoring/writeConcernError.json index 1bfae9f95..455e5422b 100644 --- a/test/command_monitoring/writeConcernError.json +++ b/test/command_monitoring/writeConcernError.json @@ -152,4 +152,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/test/crud/unified/client-bulkWrite-delete-options.json b/test/crud/unified/client-bulkWrite-delete-options.json new file mode 100644 index 000000000..5bdf2b124 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-delete-options.json @@ -0,0 +1,267 @@ +{ + "description": "client bulkWrite delete options", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0", + "collation": { + "locale": "simple" + }, + "hint": "_id_" + }, + "tests": [ + { + "description": "client bulk write delete with collation", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "collation": { + "locale": "simple" + } + } + }, + { + "deleteMany": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "collation": { + "locale": "simple" + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 3, + "insertResults": {}, + "updateResults": {}, + "deleteResults": { + "0": { + "deletedCount": 1 + }, + "1": { + "deletedCount": 2 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "delete": 0, + "filter": { + "_id": 1 + }, + "collation": { + "locale": "simple" + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": { + "$gt": 1 + } + }, + "collation": { + "locale": "simple" + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [] + } + ] + }, + { + "description": "client bulk write delete with hint", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "hint": "_id_" + } + }, + { + "deleteMany": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "hint": "_id_" + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 3, + "insertResults": {}, + "updateResults": {}, + "deleteResults": { + "0": { + "deletedCount": 1 + }, + "1": { + "deletedCount": 2 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "delete": 0, + "filter": { + "_id": 1 + }, + "hint": "_id_", + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": { + "$gt": 1 + } + }, + "hint": "_id_", + "multi": true + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-errorResponse.json b/test/crud/unified/client-bulkWrite-errorResponse.json new file mode 100644 index 000000000..edf2339d8 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-errorResponse.json @@ -0,0 +1,68 @@ +{ + "description": "client bulkWrite errorResponse", + "schemaVersion": "1.12", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite operations support errorResponse assertions", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 8 + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1 + } + } + } + ] + }, + "expectError": { + "errorCode": 8, + "errorResponse": { + "code": 8 + } + } + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-errors.json b/test/crud/unified/client-bulkWrite-errors.json new file mode 100644 index 000000000..9f17f8533 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-errors.json @@ -0,0 +1,454 @@ +{ + "description": "client bulkWrite errors", + "schemaVersion": "1.21", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ], + "uriOptions": { + "retryWrites": false + }, + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0", + "writeConcernErrorCode": 91, + "writeConcernErrorMessage": "Replication is being shut down", + "undefinedVarCode": 17276 + }, + "tests": [ + { + "description": "an individual operation fails during an ordered bulkWrite", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 1, + "insertResults": {}, + "updateResults": {}, + "deleteResults": { + "0": { + "deletedCount": 1 + } + } + }, + "writeErrors": { + "1": { + "code": 17276 + } + } + } + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "an individual operation fails during an unordered bulkWrite", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": true, + "ordered": false + }, + "expectError": { + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 2, + "insertResults": {}, + "updateResults": {}, + "deleteResults": { + "0": { + "deletedCount": 1 + }, + "2": { + "deletedCount": 1 + } + } + }, + "writeErrors": { + "1": { + "code": 17276 + } + } + } + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "detailed results are omitted from error when verboseResults is false", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": false + }, + "expectError": { + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 1, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + }, + "writeErrors": { + "1": { + "code": 17276 + } + } + } + } + ] + }, + { + "description": "a top-level failure occurs during a bulkWrite", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 8 + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "x": 1 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "errorCode": 8 + } + } + ] + }, + { + "description": "a bulk write with only errors does not report a partial result", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "expectResult": { + "$$unsetOrMatches": {} + }, + "writeErrors": { + "0": { + "code": 17276 + } + } + } + } + ] + }, + { + "description": "a write concern error occurs during a bulkWrite", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 10 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 10 + } + }, + "updateResults": {}, + "deleteResults": {} + }, + "writeConcernErrors": [ + { + "code": 91, + "message": "Replication is being shut down" + } + ] + } + } + ] + }, + { + "description": "an empty list of write models is a client-side error", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [], + "verboseResults": true + }, + "expectError": { + "isClientError": true + } + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-mixed-namespaces.json b/test/crud/unified/client-bulkWrite-mixed-namespaces.json new file mode 100644 index 000000000..f90755dc8 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-mixed-namespaces.json @@ -0,0 +1,314 @@ +{ + "description": "client bulkWrite with mixed namespaces", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "db0" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "coll1" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "db1" + } + }, + { + "collection": { + "id": "collection2", + "database": "database1", + "collectionName": "coll2" + } + } + ], + "initialData": [ + { + "databaseName": "db0", + "collectionName": "coll0", + "documents": [] + }, + { + "databaseName": "db0", + "collectionName": "coll1", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + }, + { + "databaseName": "db1", + "collectionName": "coll2", + "documents": [ + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ], + "_yamlAnchors": { + "db0Coll0Namespace": "db0.coll0", + "db0Coll1Namespace": "db0.coll1", + "db1Coll2Namespace": "db1.coll2" + }, + "tests": [ + { + "description": "client bulkWrite with mixed namespaces", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "db0.coll0", + "document": { + "_id": 1 + } + } + }, + { + "insertOne": { + "namespace": "db0.coll0", + "document": { + "_id": 2 + } + } + }, + { + "updateOne": { + "namespace": "db0.coll1", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteOne": { + "namespace": "db1.coll2", + "filter": { + "_id": 3 + } + } + }, + { + "deleteOne": { + "namespace": "db0.coll1", + "filter": { + "_id": 2 + } + } + }, + { + "replaceOne": { + "namespace": "db1.coll2", + "filter": { + "_id": 4 + }, + "replacement": { + "x": 45 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 2, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 2, + "insertResults": { + "0": { + "insertedId": 1 + }, + "1": { + "insertedId": 2 + } + }, + "updateResults": { + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "5": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": { + "3": { + "deletedCount": 1 + }, + "4": { + "deletedCount": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1 + } + }, + { + "insert": 0, + "document": { + "_id": 2 + } + }, + { + "update": 1, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "delete": 2, + "filter": { + "_id": 3 + }, + "multi": false + }, + { + "delete": 1, + "filter": { + "_id": 2 + }, + "multi": false + }, + { + "update": 2, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 45 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "db0.coll0" + }, + { + "ns": "db0.coll1" + }, + { + "ns": "db1.coll2" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "db0", + "collectionName": "coll0", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + }, + { + "databaseName": "db0", + "collectionName": "coll1", + "documents": [ + { + "_id": 1, + "x": 12 + } + ] + }, + { + "databaseName": "db1", + "collectionName": "coll2", + "documents": [ + { + "_id": 4, + "x": 45 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-options.json b/test/crud/unified/client-bulkWrite-options.json new file mode 100644 index 000000000..a1e6af3bf --- /dev/null +++ b/test/crud/unified/client-bulkWrite-options.json @@ -0,0 +1,715 @@ +{ + "description": "client bulkWrite top-level options", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "writeConcernClient", + "uriOptions": { + "w": 1 + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0", + "comment": { + "bulk": "write" + }, + "let": { + "id1": 1, + "id2": 2 + }, + "writeConcern": { + "w": "majority" + } + }, + "tests": [ + { + "description": "client bulkWrite comment", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "comment": { + "bulk": "write" + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "comment": { + "bulk": "write" + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite bypassDocumentValidation", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "bypassDocumentValidation": true, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "bypassDocumentValidation": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite let", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id1" + ] + } + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + } + ], + "let": { + "id1": 1, + "id2": 2 + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 1, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": { + "1": { + "deletedCount": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "let": { + "id1": 1, + "id2": 2 + }, + "ops": [ + { + "update": 0, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id1" + ] + } + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 12 + } + ] + } + ] + }, + { + "description": "client bulkWrite bypassDocumentValidation: false is sent", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "bypassDocumentValidation": false, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "bypassDocumentValidation": false, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite writeConcern", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "writeConcern": { + "w": "majority" + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "writeConcern": { + "w": "majority" + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite inherits writeConcern from client", + "operations": [ + { + "object": "writeConcernClient", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "writeConcernClient", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "writeConcern": { + "w": 1 + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite writeConcern option overrides client writeConcern", + "operations": [ + { + "object": "writeConcernClient", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "writeConcern": { + "w": "majority" + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "writeConcernClient", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "writeConcern": { + "w": "majority" + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-ordered.json b/test/crud/unified/client-bulkWrite-ordered.json new file mode 100644 index 000000000..a55d6619b --- /dev/null +++ b/test/crud/unified/client-bulkWrite-ordered.json @@ -0,0 +1,290 @@ +{ + "description": "client bulkWrite with ordered option", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite with ordered: false", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "verboseResults": true, + "ordered": false + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 1 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": false, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1, + "x": 11 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ] + }, + { + "description": "client bulkWrite with ordered: true", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "verboseResults": true, + "ordered": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 1 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1, + "x": 11 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ] + }, + { + "description": "client bulkWrite defaults to ordered: true", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 1 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1, + "x": 11 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-results.json b/test/crud/unified/client-bulkWrite-results.json new file mode 100644 index 000000000..97a9e50b2 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-results.json @@ -0,0 +1,832 @@ +{ + "description": "client bulkWrite results", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + }, + { + "_id": 7, + "x": 77 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite with verboseResults: true returns detailed results", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "0": { + "insertedId": 8 + } + }, + "updateResults": { + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "2": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + }, + "3": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 4 + } + }, + "deleteResults": { + "4": { + "deletedCount": 1 + }, + "5": { + "deletedCount": 2 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + }, + { + "description": "client bulkWrite with verboseResults: false omits detailed results", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ], + "verboseResults": false + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + }, + { + "description": "client bulkWrite defaults to verboseResults: false", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ] + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-update-options.json b/test/crud/unified/client-bulkWrite-update-options.json new file mode 100644 index 000000000..93a2774e5 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-update-options.json @@ -0,0 +1,948 @@ +{ + "description": "client bulkWrite update options", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 3 + ] + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0", + "collation": { + "locale": "simple" + }, + "hint": "_id_" + }, + "tests": [ + { + "description": "client bulkWrite update with arrayFilters", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "array.$[i]": 4 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ] + } + }, + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$set": { + "array.$[i]": 5 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ] + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "1": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$set": { + "array.$[i]": 4 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ], + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$set": { + "array.$[i]": 5 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ], + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 4, + 4 + ] + }, + { + "_id": 2, + "array": [ + 1, + 5, + 5 + ] + }, + { + "_id": 3, + "array": [ + 1, + 5, + 5 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 3 + ] + } + ] + } + ] + }, + { + "description": "client bulkWrite update with collation", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "collation": { + "locale": "simple" + } + } + }, + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "collation": { + "locale": "simple" + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 4 + }, + "replacement": { + "array": [ + 1, + 2, + 6 + ] + }, + "collation": { + "locale": "simple" + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 4, + "modifiedCount": 4, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "1": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "collation": { + "locale": "simple" + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "collation": { + "locale": "simple" + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "array": [ + 1, + 2, + 6 + ] + }, + "collation": { + "locale": "simple" + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 4 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 6 + ] + } + ] + } + ] + }, + { + "description": "client bulkWrite update with hint", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "hint": "_id_" + } + }, + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "hint": "_id_" + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 4 + }, + "replacement": { + "array": [ + 1, + 2, + 6 + ] + }, + "hint": "_id_" + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 4, + "modifiedCount": 4, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "1": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "hint": "_id_", + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "hint": "_id_", + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "array": [ + 1, + 2, + 6 + ] + }, + "hint": "_id_", + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 4 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 6 + ] + } + ] + } + ] + }, + { + "description": "client bulkWrite update with upsert", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 5 + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "upsert": true + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 6 + }, + "replacement": { + "array": [ + 1, + 2, + 6 + ] + }, + "upsert": true + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 2, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 5 + }, + "1": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 6 + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 5 + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "upsert": true, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 6 + }, + "updateMods": { + "array": [ + 1, + 2, + 6 + ] + }, + "upsert": true, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 5, + "array": [ + 1, + 2, + 4 + ] + }, + { + "_id": 6, + "array": [ + 1, + 2, + 6 + ] + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-update-pipeline.json b/test/crud/unified/client-bulkWrite-update-pipeline.json new file mode 100644 index 000000000..57b6c9c1b --- /dev/null +++ b/test/crud/unified/client-bulkWrite-update-pipeline.json @@ -0,0 +1,257 @@ +{ + "description": "client bulkWrite update pipeline", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 1 + }, + { + "_id": 2, + "x": 2 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite updateOne with pipeline", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": [ + { + "$addFields": { + "foo": 1 + } + } + ] + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": [ + { + "$addFields": { + "foo": 1 + } + } + ], + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 1, + "foo": 1 + }, + { + "_id": 2, + "x": 2 + } + ] + } + ] + }, + { + "description": "client bulkWrite updateMany with pipeline", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": {}, + "update": [ + { + "$addFields": { + "foo": 1 + } + } + ] + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": {}, + "updateMods": [ + { + "$addFields": { + "foo": 1 + } + } + ], + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 1, + "foo": 1 + }, + { + "_id": 2, + "x": 2, + "foo": 1 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-update-validation.json b/test/crud/unified/client-bulkWrite-update-validation.json new file mode 100644 index 000000000..617e71133 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-update-validation.json @@ -0,0 +1,216 @@ +{ + "description": "client-bulkWrite-update-validation", + "schemaVersion": "1.1", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite replaceOne prohibits atomic modifiers", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "replacement": { + "$set": { + "x": 22 + } + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite updateOne requires atomic modifiers", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "x": 22 + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite updateMany requires atomic modifiers", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [ + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "x": 44 + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/test/data_lake/aggregate.json b/test/data_lake/aggregate.json deleted file mode 100644 index 99995bca4..000000000 --- a/test/data_lake/aggregate.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "collection_name": "driverdata", - "database_name": "test", - "tests": [ - { - "description": "Aggregate with pipeline (project, sort, limit)", - "operations": [ - { - "object": "collection", - "name": "aggregate", - "arguments": { - "pipeline": [ - { - "$project": { - "_id": 0 - } - }, - { - "$sort": { - "a": 1 - } - }, - { - "$limit": 2 - } - ] - }, - "result": [ - { - "a": 1, - "b": 2, - "c": 3 - }, - { - "a": 2, - "b": 3, - "c": 4 - } - ] - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "aggregate": "driverdata" - } - } - } - ] - } - ] -} diff --git a/test/data_lake/estimatedDocumentCount.json b/test/data_lake/estimatedDocumentCount.json deleted file mode 100644 index 997a3ab3f..000000000 --- a/test/data_lake/estimatedDocumentCount.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "collection_name": "driverdata", - "database_name": "test", - "tests": [ - { - "description": "estimatedDocumentCount succeeds", - "operations": [ - { - "object": "collection", - "name": "estimatedDocumentCount", - "result": 15 - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "count": "driverdata" - }, - "command_name": "count", - "database_name": "test" - } - } - ] - } - ] -} diff --git a/test/data_lake/getMore.json b/test/data_lake/getMore.json deleted file mode 100644 index e2e1d4788..000000000 --- a/test/data_lake/getMore.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "collection_name": "driverdata", - "database_name": "test", - "tests": [ - { - "description": "A successful find event with getMore", - "operations": [ - { - "object": "collection", - "name": "find", - "arguments": { - "filter": { - "a": { - "$gte": 2 - } - }, - "sort": { - "a": 1 - }, - "batchSize": 3, - "limit": 4 - } - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "find": "driverdata", - "filter": { - "a": { - "$gte": 2 - } - }, - "sort": { - "a": 1 - }, - "batchSize": 3, - "limit": 4 - }, - "command_name": "find", - "database_name": "test" - } - }, - { - "command_started_event": { - "command": { - "batchSize": 1 - }, - "command_name": "getMore", - "database_name": "cursors" - } - } - ] - } - ] -} diff --git a/test/data_lake/listCollections.json b/test/data_lake/listCollections.json deleted file mode 100644 index e419f7b3e..000000000 --- a/test/data_lake/listCollections.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "database_name": "test", - "tests": [ - { - "description": "ListCollections succeeds", - "operations": [ - { - "name": "listCollections", - "object": "database" - } - ], - "expectations": [ - { - "command_started_event": { - "command_name": "listCollections", - "database_name": "test", - "command": { - "listCollections": 1 - } - } - } - ] - } - ] -} diff --git a/test/data_lake/listDatabases.json b/test/data_lake/listDatabases.json deleted file mode 100644 index 6458148e4..000000000 --- a/test/data_lake/listDatabases.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "tests": [ - { - "description": "ListDatabases succeeds", - "operations": [ - { - "name": "listDatabases", - "object": "client" - } - ], - "expectations": [ - { - "command_started_event": { - "command_name": "listDatabases", - "database_name": "admin", - "command": { - "listDatabases": 1 - } - } - } - ] - } - ] -} diff --git a/test/data_lake/runCommand.json b/test/data_lake/runCommand.json deleted file mode 100644 index d81ff1a64..000000000 --- a/test/data_lake/runCommand.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "database_name": "test", - "tests": [ - { - "description": "ping succeeds using runCommand", - "operations": [ - { - "name": "runCommand", - "object": "database", - "command_name": "ping", - "arguments": { - "command": { - "ping": 1 - } - } - } - ], - "expectations": [ - { - "command_started_event": { - "command_name": "ping", - "database_name": "test", - "command": { - "ping": 1 - } - } - } - ] - } - ] -} diff --git a/test/data_lake/unified/aggregate.json b/test/data_lake/unified/aggregate.json new file mode 100644 index 000000000..68a3467c7 --- /dev/null +++ b/test/data_lake/unified/aggregate.json @@ -0,0 +1,84 @@ +{ + "description": "aggregate", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], + "tests": [ + { + "description": "Aggregate with pipeline (project, sort, limit)", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$project": { + "_id": 0 + } + }, + { + "$sort": { + "a": 1 + } + }, + { + "$limit": 2 + } + ] + }, + "expectResult": [ + { + "a": 1, + "b": 2, + "c": 3 + }, + { + "a": 2, + "b": 3, + "c": 4 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "driverdata" + }, + "commandName": "aggregate", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/test/data_lake/unified/estimatedDocumentCount.json b/test/data_lake/unified/estimatedDocumentCount.json new file mode 100644 index 000000000..b7515a441 --- /dev/null +++ b/test/data_lake/unified/estimatedDocumentCount.json @@ -0,0 +1,56 @@ +{ + "description": "estimatedDocumentCount", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], + "tests": [ + { + "description": "estimatedDocumentCount succeeds", + "operations": [ + { + "object": "collection0", + "name": "estimatedDocumentCount", + "expectResult": 15 + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "count": "driverdata" + }, + "commandName": "count", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/test/data_lake/find.json b/test/data_lake/unified/find.json similarity index 52% rename from test/data_lake/find.json rename to test/data_lake/unified/find.json index 8a3468a13..d0652dc72 100644 --- a/test/data_lake/find.json +++ b/test/data_lake/unified/find.json @@ -1,12 +1,36 @@ { - "collection_name": "driverdata", - "database_name": "test", + "description": "find", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], "tests": [ { "description": "Find with projection and sort", "operations": [ { - "object": "collection", + "object": "collection0", "name": "find", "arguments": { "filter": { @@ -22,7 +46,7 @@ }, "limit": 5 }, - "result": [ + "expectResult": [ { "a": 5, "b": 6, @@ -51,13 +75,20 @@ ] } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "find": "driverdata" + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "driverdata" + }, + "commandName": "find", + "databaseName": "test" + } } - } + ] } ] } diff --git a/test/data_lake/unified/getMore.json b/test/data_lake/unified/getMore.json new file mode 100644 index 000000000..109b6d3d8 --- /dev/null +++ b/test/data_lake/unified/getMore.json @@ -0,0 +1,95 @@ +{ + "description": "getMore", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], + "tests": [ + { + "description": "A successful find event with getMore", + "operations": [ + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "a": { + "$gte": 2 + } + }, + "sort": { + "a": 1 + }, + "batchSize": 3, + "limit": 4 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "driverdata", + "filter": { + "a": { + "$gte": 2 + } + }, + "sort": { + "a": 1 + }, + "batchSize": 3, + "limit": 4 + }, + "commandName": "find", + "databaseName": "test" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": { + "$$type": "string" + }, + "batchSize": 1 + }, + "commandName": "getMore", + "databaseName": "cursors" + } + } + ] + } + ] + } + ] +} diff --git a/test/data_lake/unified/listCollections.json b/test/data_lake/unified/listCollections.json new file mode 100644 index 000000000..642e7ed32 --- /dev/null +++ b/test/data_lake/unified/listCollections.json @@ -0,0 +1,48 @@ +{ + "description": "listCollections", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + } + ], + "tests": [ + { + "description": "ListCollections succeeds", + "operations": [ + { + "object": "database0", + "name": "listCollections" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1 + }, + "commandName": "listCollections", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/test/data_lake/unified/listDatabases.json b/test/data_lake/unified/listDatabases.json new file mode 100644 index 000000000..64506ee54 --- /dev/null +++ b/test/data_lake/unified/listDatabases.json @@ -0,0 +1,41 @@ +{ + "description": "listDatabases", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + } + ], + "tests": [ + { + "description": "ListCollections succeeds", + "operations": [ + { + "object": "client0", + "name": "listDatabases" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listDatabases": 1 + }, + "commandName": "listDatabases", + "databaseName": "admin" + } + } + ] + } + ] + } + ] +} diff --git a/test/data_lake/unified/runCommand.json b/test/data_lake/unified/runCommand.json new file mode 100644 index 000000000..325b6b3f3 --- /dev/null +++ b/test/data_lake/unified/runCommand.json @@ -0,0 +1,54 @@ +{ + "description": "runCommand", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + } + ], + "tests": [ + { + "description": "ping succeeds using runCommand", + "operations": [ + { + "object": "database0", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "commandName": "ping", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/test/discovery_and_monitoring/rs/compatible.json b/test/discovery_and_monitoring/rs/compatible.json index 444b13e9d..dfd5d57df 100644 --- a/test/discovery_and_monitoring/rs/compatible.json +++ b/test/discovery_and_monitoring/rs/compatible.json @@ -16,7 +16,7 @@ "b:27017" ], "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ], [ diff --git a/test/discovery_and_monitoring/rs/compatible_unknown.json b/test/discovery_and_monitoring/rs/compatible_unknown.json index cf92dd1ed..95e03ea95 100644 --- a/test/discovery_and_monitoring/rs/compatible_unknown.json +++ b/test/discovery_and_monitoring/rs/compatible_unknown.json @@ -16,7 +16,7 @@ "b:27017" ], "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/sharded/compatible.json b/test/discovery_and_monitoring/sharded/compatible.json index e531db97f..ceb0ec24c 100644 --- a/test/discovery_and_monitoring/sharded/compatible.json +++ b/test/discovery_and_monitoring/sharded/compatible.json @@ -23,7 +23,7 @@ "isWritablePrimary": true, "msg": "isdbgrid", "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/single/compatible.json b/test/discovery_and_monitoring/single/compatible.json index 302927598..493d9b748 100644 --- a/test/discovery_and_monitoring/single/compatible.json +++ b/test/discovery_and_monitoring/single/compatible.json @@ -11,7 +11,7 @@ "helloOk": true, "isWritablePrimary": true, "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/single/too_old_then_upgraded.json b/test/discovery_and_monitoring/single/too_old_then_upgraded.json index 58ae7d9de..c3dd98cf6 100644 --- a/test/discovery_and_monitoring/single/too_old_then_upgraded.json +++ b/test/discovery_and_monitoring/single/too_old_then_upgraded.json @@ -1,5 +1,5 @@ { - "description": "Standalone with default maxWireVersion of 0 is upgraded to one with maxWireVersion 6", + "description": "Standalone with default maxWireVersion of 0 is upgraded to one with maxWireVersion 21", "uri": "mongodb://a", "phases": [ { @@ -35,7 +35,7 @@ "helloOk": true, "isWritablePrimary": true, "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/unified/interruptInUse-pool-clear.json b/test/discovery_and_monitoring/unified/interruptInUse-pool-clear.json index 6fdef55b4..a20d79030 100644 --- a/test/discovery_and_monitoring/unified/interruptInUse-pool-clear.json +++ b/test/discovery_and_monitoring/unified/interruptInUse-pool-clear.json @@ -4,11 +4,11 @@ "runOnRequirements": [ { "minServerVersion": "4.9", + "serverless": "forbid", "topologies": [ "replicaset", "sharded" - ], - "serverless": "forbid" + ] } ], "createEntities": [ @@ -39,13 +39,6 @@ "client": { "id": "client", "useMultipleMongoses": false, - "uriOptions": { - "connectTimeoutMS": 500, - "heartbeatFrequencyMS": 500, - "appname": "interruptInUse", - "retryReads": false, - "minPoolSize": 0 - }, "observeEvents": [ "poolClearedEvent", "connectionClosedEvent", @@ -54,7 +47,14 @@ "commandFailedEvent", "connectionCheckedOutEvent", "connectionCheckedInEvent" - ] + ], + "uriOptions": { + "connectTimeoutMS": 500, + "heartbeatFrequencyMS": 500, + "appname": "interruptInUse", + "retryReads": false, + "minPoolSize": 0 + } } }, { @@ -83,7 +83,9 @@ "name": "insertOne", "object": "collection", "arguments": { - "document": { "_id" : 1 } + "document": { + "_id": 1 + } } }, { @@ -92,14 +94,16 @@ "arguments": { "thread": "thread1", "operation": { - "name": "find", - "object": "collection", - "arguments": { - "filter": { "$where": "sleep(2000) || true" } - }, - "expectError": { - "isError": true + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$where": "sleep(2000) || true" } + }, + "expectError": { + "isError": true + } } } }, @@ -107,6 +111,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { + "client": "setupClient", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -114,22 +119,22 @@ }, "data": { "failCommands": [ - "hello", "isMaster" + "hello", + "isMaster" ], "blockConnection": true, "blockTimeMS": 1500, "appName": "interruptInUse" } - }, - "client": "setupClient" + } } }, { - "name": "waitForThread", - "object": "testRunner", - "arguments": { - "thread": "thread1" - } + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread1" + } } ], "expectEvents": [ @@ -157,20 +162,20 @@ "commandName": "find" } } - ] - }, - { + ] + }, + { "client": "client", "eventType": "cmap", "events": [ { - "connectionCheckedOutEvent": { } + "connectionCheckedOutEvent": {} }, { - "connectionCheckedInEvent": { } + "connectionCheckedInEvent": {} }, { - "connectionCheckedOutEvent": { } + "connectionCheckedOutEvent": {} }, { "poolClearedEvent": { @@ -181,16 +186,22 @@ "connectionCheckedInEvent": {} }, { - "connectionClosedEvent": { } + "connectionClosedEvent": {} } - ] - } + ] + } ], - "outcome": [{ - "collectionName": "interruptInUse", - "databaseName": "sdam-tests", - "documents": [{ "_id": 1 }] - }] + "outcome": [ + { + "collectionName": "interruptInUse", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] }, { "description": "Error returned from connection pool clear with interruptInUseConnections=true is retryable", @@ -204,22 +215,22 @@ "client": { "id": "client", "useMultipleMongoses": false, + "observeEvents": [ + "poolClearedEvent", + "connectionClosedEvent", + "commandStartedEvent", + "commandFailedEvent", + "commandSucceededEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent" + ], "uriOptions": { "connectTimeoutMS": 500, "heartbeatFrequencyMS": 500, "appname": "interruptInUseRetryable", "retryReads": true, "minPoolSize": 0 - }, - "observeEvents": [ - "poolClearedEvent", - "connectionClosedEvent", - "commandFailedEvent", - "commandStartedEvent", - "commandSucceededEvent", - "connectionCheckedOutEvent", - "connectionCheckedInEvent" - ] + } } }, { @@ -248,7 +259,9 @@ "name": "insertOne", "object": "collection", "arguments": { - "document": { "_id" : 1 } + "document": { + "_id": 1 + } } }, { @@ -257,11 +270,13 @@ "arguments": { "thread": "thread1", "operation": { - "name": "find", - "object": "collection", - "arguments": { - "filter": { "$where": "sleep(2000) || true" } + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$where": "sleep(2000) || true" } + } } } }, @@ -269,6 +284,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { + "client": "setupClient", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -276,22 +292,22 @@ }, "data": { "failCommands": [ - "hello", "isMaster" + "hello", + "isMaster" ], "blockConnection": true, "blockTimeMS": 1500, "appName": "interruptInUseRetryable" } - }, - "client": "setupClient" + } } }, { - "name": "waitForThread", - "object": "testRunner", - "arguments": { - "thread": "thread1" - } + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread1" + } } ], "expectEvents": [ @@ -329,20 +345,20 @@ "commandName": "find" } } - ] - }, - { + ] + }, + { "client": "client", "eventType": "cmap", "events": [ { - "connectionCheckedOutEvent": { } + "connectionCheckedOutEvent": {} }, { - "connectionCheckedInEvent": { } + "connectionCheckedInEvent": {} }, { - "connectionCheckedOutEvent": { } + "connectionCheckedOutEvent": {} }, { "poolClearedEvent": { @@ -353,7 +369,7 @@ "connectionCheckedInEvent": {} }, { - "connectionClosedEvent": { } + "connectionClosedEvent": {} }, { "connectionCheckedOutEvent": {} @@ -361,14 +377,20 @@ { "connectionCheckedInEvent": {} } - ] - } + ] + } ], - "outcome": [{ - "collectionName": "interruptInUse", - "databaseName": "sdam-tests", - "documents": [{ "_id": 1 }] - }] + "outcome": [ + { + "collectionName": "interruptInUse", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] }, { "description": "Error returned from connection pool clear with interruptInUseConnections=true is retryable for write", @@ -382,22 +404,23 @@ "client": { "id": "client", "useMultipleMongoses": false, + "observeEvents": [ + "poolClearedEvent", + "connectionClosedEvent", + "commandStartedEvent", + "commandFailedEvent", + "commandSucceededEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent" + ], "uriOptions": { "connectTimeoutMS": 500, "heartbeatFrequencyMS": 500, "appname": "interruptInUseRetryableWrite", "retryWrites": true, "minPoolSize": 0 - }, - "observeEvents": [ - "poolClearedEvent", - "connectionClosedEvent", - "commandFailedEvent", - "commandStartedEvent", - "commandSucceededEvent", - "connectionCheckedOutEvent", - "connectionCheckedInEvent" - ]} + } + } }, { "database": { @@ -425,7 +448,9 @@ "name": "insertOne", "object": "collection", "arguments": { - "document": { "_id": 1 } + "document": { + "_id": 1 + } } }, { @@ -437,9 +462,15 @@ "name": "updateOne", "object": "collection", "arguments": { - "filter": { "$where": "sleep(2000) || true" }, - "update": [ { "$set": { "a": "bar" } } ] - } + "filter": { + "$where": "sleep(2000) || true" + }, + "update": { + "$set": { + "a": "bar" + } + } + } } } }, @@ -447,6 +478,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { + "client": "setupClient", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -454,22 +486,22 @@ }, "data": { "failCommands": [ - "hello", "isMaster" + "hello", + "isMaster" ], "blockConnection": true, "blockTimeMS": 1500, "appName": "interruptInUseRetryableWrite" } - }, - "client": "setupClient" + } } }, { - "name": "waitForThread", - "object": "testRunner", - "arguments": { - "thread": "thread1" - } + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread1" + } } ], "expectEvents": [ @@ -507,20 +539,20 @@ "commandName": "update" } } - ] - }, - { + ] + }, + { "client": "client", "eventType": "cmap", "events": [ { - "connectionCheckedOutEvent": { } + "connectionCheckedOutEvent": {} }, { - "connectionCheckedInEvent": { } + "connectionCheckedInEvent": {} }, { - "connectionCheckedOutEvent": { } + "connectionCheckedOutEvent": {} }, { "poolClearedEvent": { @@ -531,7 +563,7 @@ "connectionCheckedInEvent": {} }, { - "connectionClosedEvent": { } + "connectionClosedEvent": {} }, { "connectionCheckedOutEvent": {} @@ -539,14 +571,21 @@ { "connectionCheckedInEvent": {} } - ] - } + ] + } ], - "outcome": [{ - "collectionName": "interruptInUse", - "databaseName": "sdam-tests", - "documents": [{ "_id": 1, "a" : "bar"}] - }] + "outcome": [ + { + "collectionName": "interruptInUse", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1, + "a": "bar" + } + ] + } + ] } ] -} \ No newline at end of file +} diff --git a/test/discovery_and_monitoring/unified/loadbalanced-emit-topology-changed-before-close.json b/test/discovery_and_monitoring/unified/loadbalanced-emit-topology-changed-before-close.json new file mode 100644 index 000000000..30c065763 --- /dev/null +++ b/test/discovery_and_monitoring/unified/loadbalanced-emit-topology-changed-before-close.json @@ -0,0 +1,88 @@ +{ + "description": "loadbalanced-emit-topology-description-changed-before-close", + "schemaVersion": "1.20", + "runOnRequirements": [ + { + "topologies": [ + "load-balanced" + ], + "minServerVersion": "4.4" + } + ], + "tests": [ + { + "description": "Topology lifecycle", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "topologyDescriptionChangedEvent", + "topologyOpeningEvent", + "topologyClosedEvent" + ] + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "topologyDescriptionChangedEvent": {} + }, + "count": 2 + } + }, + { + "name": "close", + "object": "client" + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "sdam", + "events": [ + { + "topologyOpeningEvent": {} + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Unknown" + }, + "newDescription": {} + } + }, + { + "topologyDescriptionChangedEvent": { + "newDescription": { + "type": "LoadBalanced" + } + } + }, + { + "topologyDescriptionChangedEvent": { + "newDescription": { + "type": "Unknown" + } + } + }, + { + "topologyClosedEvent": {} + } + ] + } + ] + } + ] +} diff --git a/test/discovery_and_monitoring/unified/replicaset-emit-topology-changed-before-close.json b/test/discovery_and_monitoring/unified/replicaset-emit-topology-changed-before-close.json new file mode 100644 index 000000000..066a4ffee --- /dev/null +++ b/test/discovery_and_monitoring/unified/replicaset-emit-topology-changed-before-close.json @@ -0,0 +1,89 @@ +{ + "description": "replicaset-emit-topology-description-changed-before-close", + "schemaVersion": "1.20", + "runOnRequirements": [ + { + "topologies": [ + "replicaset" + ], + "minServerVersion": "4.4" + } + ], + "tests": [ + { + "description": "Topology lifecycle", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "topologyDescriptionChangedEvent", + "topologyOpeningEvent", + "topologyClosedEvent" + ] + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "topologyDescriptionChangedEvent": {} + }, + "count": 4 + } + }, + { + "name": "close", + "object": "client" + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "sdam", + "ignoreExtraEvents": false, + "events": [ + { + "topologyOpeningEvent": {} + }, + { + "topologyDescriptionChangedEvent": {} + }, + { + "topologyDescriptionChangedEvent": {} + }, + { + "topologyDescriptionChangedEvent": {} + }, + { + "topologyDescriptionChangedEvent": {} + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "ReplicaSetWithPrimary" + }, + "newDescription": { + "type": "Unknown" + } + } + }, + { + "topologyClosedEvent": {} + } + ] + } + ] + } + ] +} diff --git a/test/discovery_and_monitoring/unified/serverMonitoringMode.json b/test/discovery_and_monitoring/unified/serverMonitoringMode.json index 7d681b4f9..4b492f7d8 100644 --- a/test/discovery_and_monitoring/unified/serverMonitoringMode.json +++ b/test/discovery_and_monitoring/unified/serverMonitoringMode.json @@ -444,6 +444,69 @@ ] } ] + }, + { + "description": "poll waits after successful heartbeat", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "uriOptions": { + "serverMonitoringMode": "poll", + "heartbeatFrequencyMS": 1000000 + }, + "useMultipleMongoses": false, + "observeEvents": [ + "serverHeartbeatStartedEvent", + "serverHeartbeatSucceededEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "sdam-tests" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "wait", + "object": "testRunner", + "arguments": { + "ms": 500 + } + }, + { + "name": "assertEventCount", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatStartedEvent": {} + }, + "count": 1 + } + } + ] } ] } diff --git a/test/discovery_and_monitoring/unified/sharded-emit-topology-changed-before-close.json b/test/discovery_and_monitoring/unified/sharded-emit-topology-changed-before-close.json new file mode 100644 index 000000000..98fb58553 --- /dev/null +++ b/test/discovery_and_monitoring/unified/sharded-emit-topology-changed-before-close.json @@ -0,0 +1,108 @@ +{ + "description": "sharded-emit-topology-description-changed-before-close", + "schemaVersion": "1.20", + "runOnRequirements": [ + { + "topologies": [ + "sharded" + ], + "minServerVersion": "4.4" + } + ], + "tests": [ + { + "description": "Topology lifecycle", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "topologyDescriptionChangedEvent", + "topologyOpeningEvent", + "topologyClosedEvent" + ], + "useMultipleMongoses": true + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "topologyDescriptionChangedEvent": {} + }, + "count": 3 + } + }, + { + "name": "close", + "object": "client" + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "sdam", + "ignoreExtraEvents": false, + "events": [ + { + "topologyOpeningEvent": {} + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Unknown" + }, + "newDescription": { + "type": "Unknown" + } + } + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Unknown" + }, + "newDescription": { + "type": "Sharded" + } + } + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Sharded" + }, + "newDescription": { + "type": "Sharded" + } + } + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Sharded" + }, + "newDescription": { + "type": "Unknown" + } + } + }, + { + "topologyClosedEvent": {} + } + ] + } + ] + } + ] +} diff --git a/test/discovery_and_monitoring/unified/standalone-emit-topology-changed-before-close.json b/test/discovery_and_monitoring/unified/standalone-emit-topology-changed-before-close.json new file mode 100644 index 000000000..27b5444d5 --- /dev/null +++ b/test/discovery_and_monitoring/unified/standalone-emit-topology-changed-before-close.json @@ -0,0 +1,97 @@ +{ + "description": "standalone-emit-topology-description-changed-before-close", + "schemaVersion": "1.20", + "runOnRequirements": [ + { + "topologies": [ + "single" + ], + "minServerVersion": "4.4" + } + ], + "tests": [ + { + "description": "Topology lifecycle", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "topologyDescriptionChangedEvent", + "topologyOpeningEvent", + "topologyClosedEvent" + ] + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "topologyDescriptionChangedEvent": {} + }, + "count": 2 + } + }, + { + "name": "close", + "object": "client" + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "sdam", + "ignoreExtraEvents": false, + "events": [ + { + "topologyOpeningEvent": {} + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Unknown" + }, + "newDescription": { + "type": "Unknown" + } + } + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Unknown" + }, + "newDescription": { + "type": "Single" + } + } + }, + { + "topologyDescriptionChangedEvent": { + "previousDescription": { + "type": "Single" + }, + "newDescription": { + "type": "Unknown" + } + } + }, + { + "topologyClosedEvent": {} + } + ] + } + ] + } + ] +} diff --git a/test/read_write_concern/operation/default-write-concern-2.6.json b/test/read_write_concern/operation/default-write-concern-2.6.json index c623298cd..0d8f9c98a 100644 --- a/test/read_write_concern/operation/default-write-concern-2.6.json +++ b/test/read_write_concern/operation/default-write-concern-2.6.json @@ -1,53 +1,93 @@ { - "data": [ - { - "_id": 1, - "x": 11 - }, - { - "_id": 2, - "x": 22 - } - ], - "collection_name": "default_write_concern_coll", - "database_name": "default_write_concern_db", - "runOn": [ + "description": "default-write-concern-2.6", + "schemaVersion": "1.0", + "runOnRequirements": [ { "minServerVersion": "2.6" } ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default-write-concern-tests", + "databaseOptions": { + "writeConcern": {} + } + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll", + "collectionOptions": { + "writeConcern": {} + } + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], "tests": [ { "description": "DeleteOne omits default write concern", "operations": [ { "name": "deleteOne", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": {} }, - "result": { + "expectResult": { "deletedCount": 1 } } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "delete": "default_write_concern_coll", - "deletes": [ - { - "q": {}, - "limit": 1 + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "coll", + "deletes": [ + { + "q": {}, + "limit": 1 + } + ], + "writeConcern": { + "$$exists": false + } } - ], - "writeConcern": null + } } - } + ] } ] }, @@ -56,32 +96,36 @@ "operations": [ { "name": "deleteMany", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": {} }, - "result": { + "expectResult": { "deletedCount": 2 } } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "delete": "default_write_concern_coll", - "deletes": [ - { - "q": {}, - "limit": 0 + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "coll", + "deletes": [ + { + "q": {}, + "limit": 0 + } + ], + "writeConcern": { + "$$exists": false + } } - ], - "writeConcern": null + } } - } + ] } ] }, @@ -90,30 +134,24 @@ "operations": [ { "name": "bulkWrite", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "ordered": true, "requests": [ { - "name": "deleteMany", - "arguments": { + "deleteMany": { "filter": {} } }, { - "name": "insertOne", - "arguments": { + "insertOne": { "document": { "_id": 1 } } }, { - "name": "updateOne", - "arguments": { + "updateOne": { "filter": { "_id": 1 }, @@ -125,16 +163,14 @@ } }, { - "name": "insertOne", - "arguments": { + "insertOne": { "document": { "_id": 2 } } }, { - "name": "replaceOne", - "arguments": { + "replaceOne": { "filter": { "_id": 1 }, @@ -144,16 +180,14 @@ } }, { - "name": "insertOne", - "arguments": { + "insertOne": { "document": { "_id": 3 } } }, { - "name": "updateMany", - "arguments": { + "updateMany": { "filter": { "_id": 1 }, @@ -165,8 +199,7 @@ } }, { - "name": "deleteOne", - "arguments": { + "deleteOne": { "filter": { "_id": 3 } @@ -176,10 +209,177 @@ } } ], - "outcome": { - "collection": { - "name": "default_write_concern_coll", - "data": [ + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "coll", + "deletes": [ + { + "q": {}, + "limit": 0 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 1 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "update": "coll", + "updates": [ + { + "q": { + "_id": 1 + }, + "u": { + "$set": { + "x": 1 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 2 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "update": "coll", + "updates": [ + { + "q": { + "_id": 1 + }, + "u": { + "x": 2 + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 3 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "update": "coll", + "updates": [ + { + "q": { + "_id": 1 + }, + "u": { + "$set": { + "x": 3 + } + }, + "multi": true, + "upsert": { + "$$unsetOrMatches": false + } + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "delete": "coll", + "deletes": [ + { + "q": { + "_id": 3 + }, + "limit": 1 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ { "_id": 1, "x": 3 @@ -189,136 +389,6 @@ } ] } - }, - "expectations": [ - { - "command_started_event": { - "command": { - "delete": "default_write_concern_coll", - "deletes": [ - { - "q": {}, - "limit": 0 - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "insert": "default_write_concern_coll", - "documents": [ - { - "_id": 1 - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "update": "default_write_concern_coll", - "updates": [ - { - "q": { - "_id": 1 - }, - "u": { - "$set": { - "x": 1 - } - } - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "insert": "default_write_concern_coll", - "documents": [ - { - "_id": 2 - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "update": "default_write_concern_coll", - "updates": [ - { - "q": { - "_id": 1 - }, - "u": { - "x": 2 - } - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "insert": "default_write_concern_coll", - "documents": [ - { - "_id": 3 - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "update": "default_write_concern_coll", - "updates": [ - { - "q": { - "_id": 1 - }, - "u": { - "$set": { - "x": 3 - } - }, - "multi": true - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "delete": "default_write_concern_coll", - "deletes": [ - { - "q": { - "_id": 3 - }, - "limit": 1 - } - ], - "writeConcern": null - } - } - } ] }, { @@ -326,10 +396,7 @@ "operations": [ { "name": "insertOne", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "document": { "_id": 3 @@ -338,10 +405,7 @@ }, { "name": "insertMany", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "documents": [ { @@ -354,10 +418,51 @@ } } ], - "outcome": { - "collection": { - "name": "default_write_concern_coll", - "data": [ + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 3 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 4 + }, + { + "_id": 5 + } + ], + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ { "_id": 1, "x": 11 @@ -377,37 +482,6 @@ } ] } - }, - "expectations": [ - { - "command_started_event": { - "command": { - "insert": "default_write_concern_coll", - "documents": [ - { - "_id": 3 - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "insert": "default_write_concern_coll", - "documents": [ - { - "_id": 4 - }, - { - "_id": 5 - } - ], - "writeConcern": null - } - } - } ] }, { @@ -415,10 +489,7 @@ "operations": [ { "name": "updateOne", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": { "_id": 1 @@ -432,10 +503,7 @@ }, { "name": "updateMany", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": { "_id": 2 @@ -449,10 +517,7 @@ }, { "name": "replaceOne", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": { "_id": 2 @@ -463,10 +528,98 @@ } } ], - "outcome": { - "collection": { - "name": "default_write_concern_coll", - "data": [ + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll", + "updates": [ + { + "q": { + "_id": 1 + }, + "u": { + "$set": { + "x": 1 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "update": "coll", + "updates": [ + { + "q": { + "_id": 2 + }, + "u": { + "$set": { + "x": 2 + } + }, + "multi": true, + "upsert": { + "$$unsetOrMatches": false + } + } + ], + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "update": "coll", + "updates": [ + { + "q": { + "_id": 2 + }, + "u": { + "x": 3 + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ { "_id": 1, "x": 1 @@ -477,67 +630,6 @@ } ] } - }, - "expectations": [ - { - "command_started_event": { - "command": { - "update": "default_write_concern_coll", - "updates": [ - { - "q": { - "_id": 1 - }, - "u": { - "$set": { - "x": 1 - } - } - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "update": "default_write_concern_coll", - "updates": [ - { - "q": { - "_id": 2 - }, - "u": { - "$set": { - "x": 2 - } - }, - "multi": true - } - ], - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "update": "default_write_concern_coll", - "updates": [ - { - "q": { - "_id": 2 - }, - "u": { - "x": 3 - } - } - ], - "writeConcern": null - } - } - } ] } ] diff --git a/test/read_write_concern/operation/default-write-concern-3.2.json b/test/read_write_concern/operation/default-write-concern-3.2.json index 04dd231f0..166a18491 100644 --- a/test/read_write_concern/operation/default-write-concern-3.2.json +++ b/test/read_write_concern/operation/default-write-concern-3.2.json @@ -1,31 +1,64 @@ { - "data": [ - { - "_id": 1, - "x": 11 - }, - { - "_id": 2, - "x": 22 - } - ], - "collection_name": "default_write_concern_coll", - "database_name": "default_write_concern_db", - "runOn": [ + "description": "default-write-concern-3.2", + "schemaVersion": "1.0", + "runOnRequirements": [ { "minServerVersion": "3.2" } ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default-write-concern-tests", + "databaseOptions": { + "writeConcern": {} + } + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll", + "collectionOptions": { + "writeConcern": {} + } + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], "tests": [ { "description": "findAndModify operations omit default write concern", "operations": [ { "name": "findOneAndUpdate", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": { "_id": 1 @@ -39,10 +72,7 @@ }, { "name": "findOneAndReplace", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": { "_id": 2 @@ -54,10 +84,7 @@ }, { "name": "findOneAndDelete", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "filter": { "_id": 2 @@ -65,60 +92,72 @@ } } ], - "outcome": { - "collection": { - "name": "default_write_concern_coll", - "data": [ + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "findAndModify": "coll", + "query": { + "_id": 1 + }, + "update": { + "$set": { + "x": 1 + } + }, + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "findAndModify": "coll", + "query": { + "_id": 2 + }, + "update": { + "x": 2 + }, + "writeConcern": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "findAndModify": "coll", + "query": { + "_id": 2 + }, + "remove": true, + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ { "_id": 1, "x": 1 } ] } - }, - "expectations": [ - { - "command_started_event": { - "command": { - "findAndModify": "default_write_concern_coll", - "query": { - "_id": 1 - }, - "update": { - "$set": { - "x": 1 - } - }, - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "findAndModify": "default_write_concern_coll", - "query": { - "_id": 2 - }, - "update": { - "x": 2 - }, - "writeConcern": null - } - } - }, - { - "command_started_event": { - "command": { - "findAndModify": "default_write_concern_coll", - "query": { - "_id": 2 - }, - "remove": true, - "writeConcern": null - } - } - } ] } ] diff --git a/test/read_write_concern/operation/default-write-concern-3.4.json b/test/read_write_concern/operation/default-write-concern-3.4.json index 6519f6f08..e18cdfc0c 100644 --- a/test/read_write_concern/operation/default-write-concern-3.4.json +++ b/test/read_write_concern/operation/default-write-concern-3.4.json @@ -1,30 +1,68 @@ { - "data": [ - { - "_id": 1, - "x": 11 - }, - { - "_id": 2, - "x": 22 - } - ], - "collection_name": "default_write_concern_coll", - "database_name": "default_write_concern_db", - "runOn": [ + "description": "default-write-concern-3.4", + "schemaVersion": "1.4", + "runOnRequirements": [ { "minServerVersion": "3.4" } ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default-write-concern-tests", + "databaseOptions": { + "writeConcern": {} + } + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll", + "collectionOptions": { + "writeConcern": {} + } + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], "tests": [ { "description": "Aggregate with $out omits default write concern", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "name": "aggregate", "arguments": { "pipeline": [ @@ -42,77 +80,89 @@ } } ], - "outcome": { - "collection": { - "name": "other_collection_name", - "data": [ + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "coll", + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$out": "other_collection_name" + } + ], + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "other_collection_name", + "databaseName": "default-write-concern-tests", + "documents": [ { "_id": 2, "x": 22 } ] } - }, - "expectations": [ - { - "command_started_event": { - "command": { - "aggregate": "default_write_concern_coll", - "pipeline": [ - { - "$match": { - "_id": { - "$gt": 1 - } - } - }, - { - "$out": "other_collection_name" - } - ], - "writeConcern": null - } - } - } ] }, { "description": "RunCommand with a write command omits default write concern (runCommand should never inherit write concern)", "operations": [ { - "object": "database", - "databaseOptions": { - "writeConcern": {} - }, + "object": "database0", "name": "runCommand", - "command_name": "delete", "arguments": { "command": { - "delete": "default_write_concern_coll", + "delete": "coll", "deletes": [ { "q": {}, "limit": 1 } ] - } + }, + "commandName": "delete" } } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "delete": "default_write_concern_coll", - "deletes": [ - { - "q": {}, - "limit": 1 + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "coll", + "deletes": [ + { + "q": {}, + "limit": 1 + } + ], + "writeConcern": { + "$$exists": false + } } - ], - "writeConcern": null + } } - } + ] } ] }, @@ -120,10 +170,7 @@ "description": "CreateIndex and dropIndex omits default write concern", "operations": [ { - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "name": "createIndex", "arguments": { "keys": { @@ -132,53 +179,61 @@ } }, { - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "name": "dropIndex", "arguments": { "name": "x_1" } } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "createIndexes": "default_write_concern_coll", - "indexes": [ - { - "name": "x_1", - "key": { - "x": 1 + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "createIndexes": "coll", + "indexes": [ + { + "name": "x_1", + "key": { + "x": 1 + } + } + ], + "writeConcern": { + "$$exists": false } } - ], - "writeConcern": null + } + }, + { + "commandStartedEvent": { + "command": { + "dropIndexes": "coll", + "index": "x_1", + "writeConcern": { + "$$exists": false + } + } + } } - } - }, - { - "command_started_event": { - "command": { - "dropIndexes": "default_write_concern_coll", - "index": "x_1", - "writeConcern": null - } - } + ] } ] }, { "description": "MapReduce omits default write concern", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "mapReduce", - "object": "collection", - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "arguments": { "map": { "$code": "function inc() { return emit(0, this.x + 1) }" @@ -192,23 +247,30 @@ } } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "mapReduce": "default_write_concern_coll", - "map": { - "$code": "function inc() { return emit(0, this.x + 1) }" - }, - "reduce": { - "$code": "function sum(key, values) { return values.reduce((acc, x) => acc + x); }" - }, - "out": { - "inline": 1 - }, - "writeConcern": null + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "mapReduce": "coll", + "map": { + "$code": "function inc() { return emit(0, this.x + 1) }" + }, + "reduce": { + "$code": "function sum(key, values) { return values.reduce((acc, x) => acc + x); }" + }, + "out": { + "inline": 1 + }, + "writeConcern": { + "$$exists": false + } + } + } } - } + ] } ] } diff --git a/test/read_write_concern/operation/default-write-concern-4.2.json b/test/read_write_concern/operation/default-write-concern-4.2.json index fef192d1a..e8bb78d91 100644 --- a/test/read_write_concern/operation/default-write-concern-4.2.json +++ b/test/read_write_concern/operation/default-write-concern-4.2.json @@ -1,33 +1,63 @@ { - "data": [ - { - "_id": 1, - "x": 11 - }, - { - "_id": 2, - "x": 22 - } - ], - "collection_name": "default_write_concern_coll", - "database_name": "default_write_concern_db", - "runOn": [ + "description": "default-write-concern-4.2", + "schemaVersion": "1.0", + "runOnRequirements": [ { "minServerVersion": "4.2" } ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default-write-concern-tests", + "databaseOptions": { + "writeConcern": {} + } + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll", + "collectionOptions": { + "writeConcern": {} + } + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "default-write-concern-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], "tests": [ { "description": "Aggregate with $merge omits default write concern", "operations": [ { - "object": "collection", - "databaseOptions": { - "writeConcern": {} - }, - "collectionOptions": { - "writeConcern": {} - }, + "object": "collection0", "name": "aggregate", "arguments": { "pipeline": [ @@ -47,41 +77,49 @@ } } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "aggregate": "default_write_concern_coll", - "pipeline": [ - { - "$match": { - "_id": { - "$gt": 1 + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "coll", + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$merge": { + "into": "other_collection_name" + } } - } - }, - { - "$merge": { - "into": "other_collection_name" + ], + "writeConcern": { + "$$exists": false } } - ], - "writeConcern": null + } } - } + ] } ], - "outcome": { - "collection": { - "name": "other_collection_name", - "data": [ + "outcome": [ + { + "collectionName": "other_collection_name", + "databaseName": "default-write-concern-tests", + "documents": [ { "_id": 2, "x": 22 } ] } - } + ] } ] } diff --git a/test/retryable_writes/unified/client-bulkWrite-clientErrors.json b/test/retryable_writes/unified/client-bulkWrite-clientErrors.json new file mode 100644 index 000000000..e2c0fb9c0 --- /dev/null +++ b/test/retryable_writes/unified/client-bulkWrite-clientErrors.json @@ -0,0 +1,350 @@ +{ + "description": "client bulkWrite retryable writes with client errors", + "schemaVersion": "1.21", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ], + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "retryable-writes-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite with one network error succeeds after retry", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "closeConnection": true + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "client bulkWrite with two network errors fails after retry", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "closeConnection": true + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "isClientError": true, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/test/retryable_writes/unified/client-bulkWrite-serverErrors.json b/test/retryable_writes/unified/client-bulkWrite-serverErrors.json new file mode 100644 index 000000000..4a0b210eb --- /dev/null +++ b/test/retryable_writes/unified/client-bulkWrite-serverErrors.json @@ -0,0 +1,872 @@ +{ + "description": "client bulkWrite retryable writes", + "schemaVersion": "1.21", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ], + "useMultipleMongoses": false + } + }, + { + "client": { + "id": "clientRetryWritesFalse", + "uriOptions": { + "retryWrites": false + }, + "observeEvents": [ + "commandStartedEvent" + ], + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "retryable-writes-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite with no multi: true operations succeeds after retryable top-level error", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "replaceOne": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 2 + }, + "replacement": { + "x": 222 + } + } + }, + { + "deleteOne": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 1, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": { + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": { + "3": { + "deletedCount": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 222 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "client bulkWrite with multi: true operations fails after retryable top-level error", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateMany": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteMany": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ] + }, + "expectError": { + "errorCode": 189, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": true + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite with no multi: true operations succeeds after retryable writeConcernError", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableWriteError" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "replaceOne": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 2 + }, + "replacement": { + "x": 222 + } + } + }, + { + "deleteOne": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 1, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": { + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": { + "3": { + "deletedCount": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite with multi: true operations fails after retryable writeConcernError", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableWriteError" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateMany": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteMany": { + "namespace": "retryable-writes-tests.coll0", + "filter": { + "_id": 3 + } + } + } + ] + }, + "expectError": { + "writeConcernErrors": [ + { + "code": 91, + "message": "Replication is being shut down" + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": true + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite with retryWrites: false does not retry", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "clientRetryWritesFalse", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "object": "clientRetryWritesFalse", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + } + ] + }, + "expectError": { + "errorCode": 189, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "expectEvents": [ + { + "client": "clientRetryWritesFalse", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/test/retryable_writes/unified/handshakeError.json b/test/retryable_writes/unified/handshakeError.json index ef06fb1e3..aa677494c 100644 --- a/test/retryable_writes/unified/handshakeError.json +++ b/test/retryable_writes/unified/handshakeError.json @@ -53,6 +53,222 @@ } ], "tests": [ + { + "description": "client.clientBulkWrite succeeds after retryable handshake network error", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "ping", + "saslContinue" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-handshake-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + } + ] + }, + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "databaseName": "retryable-writes-handshake-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite succeeds after retryable handshake server error (ShutdownInProgress)", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "ping", + "saslContinue" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-handshake-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + } + ] + }, + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "databaseName": "retryable-writes-handshake-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, { "description": "collection.insertOne succeeds after retryable handshake network error", "operations": [ diff --git a/test/server_selection_logging/operation-id.json b/test/server_selection_logging/operation-id.json index 23af7a8a2..5383b6633 100644 --- a/test/server_selection_logging/operation-id.json +++ b/test/server_selection_logging/operation-id.json @@ -47,6 +47,9 @@ } } ], + "_yamlAnchors": { + "namespace": "logging-tests.server-selection" + }, "tests": [ { "description": "Successful bulkWrite operation: log messages have operationIds", @@ -224,6 +227,190 @@ ] } ] + }, + { + "description": "Successful client bulkWrite operation: log messages have operationIds", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "topologyDescriptionChangedEvent": {} + }, + "count": 2 + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "logging-tests.server-selection", + "document": { + "x": 1 + } + } + } + ] + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection started", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + }, + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection succeeded", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "Failed client bulkWrite operation: log messages have operationIds", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "hello", + "ismaster" + ], + "appName": "loggingClient", + "closeConnection": true + } + } + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverDescriptionChangedEvent": { + "newDescription": { + "type": "Unknown" + } + } + }, + "count": 1 + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "logging-tests.server-selection", + "document": { + "x": 1 + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection started", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + }, + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Waiting for suitable server to become available", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + }, + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection failed", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + } + ] + } + ] } ] } diff --git a/test/test_change_stream.py b/test/test_change_stream.py index e00aaa640..b71f5613d 100644 --- a/test/test_change_stream.py +++ b/test/test_change_stream.py @@ -1020,21 +1020,32 @@ class TestCollectionChangeStream(TestChangeStreamBase, APITestsMixin, ProseSpecT self.assertEqual(change["ns"]["coll"], self.watched_collection().name) self.assertEqual(change["fullDocument"], raw_doc) + @client_context.require_version_min(4, 0) # Needed for start_at_operation_time. def test_uuid_representations(self): """Test with uuid document _ids and different uuid_representation.""" + optime = self.db.command("ping")["operationTime"] + self.watched_collection().insert_many( + [ + {"_id": Binary(uuid.uuid4().bytes, id_subtype)} + for id_subtype in (STANDARD, PYTHON_LEGACY) + ] + ) for uuid_representation in ALL_UUID_REPRESENTATIONS: - for id_subtype in (STANDARD, PYTHON_LEGACY): - options = self.watched_collection().codec_options.with_options( - uuid_representation=uuid_representation - ) - coll = self.watched_collection(codec_options=options) - with coll.watch() as change_stream: - coll.insert_one({"_id": Binary(uuid.uuid4().bytes, id_subtype)}) - _ = change_stream.next() - resume_token = change_stream.resume_token + options = self.watched_collection().codec_options.with_options( + uuid_representation=uuid_representation + ) + coll = self.watched_collection(codec_options=options) + with coll.watch(start_at_operation_time=optime, max_await_time_ms=1) as change_stream: + _ = change_stream.next() + resume_token_1 = change_stream.resume_token + _ = change_stream.next() + resume_token_2 = change_stream.resume_token - # Should not error. - coll.watch(resume_after=resume_token) + # Should not error. + with coll.watch(resume_after=resume_token_1): + pass + with coll.watch(resume_after=resume_token_2): + pass def test_document_id_order(self): """Test with document _ids that need their order preserved.""" @@ -1053,7 +1064,8 @@ class TestCollectionChangeStream(TestChangeStreamBase, APITestsMixin, ProseSpecT # The resume token is always a document. self.assertIsInstance(resume_token, document_class) # Should not error. - coll.watch(resume_after=resume_token) + with coll.watch(resume_after=resume_token): + pass coll.delete_many({}) def test_read_concern(self): diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py new file mode 100644 index 000000000..facf2971a --- /dev/null +++ b/test/test_client_bulk_write.py @@ -0,0 +1,571 @@ +# Copyright 2024-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the client bulk write API.""" +from __future__ import annotations + +import sys + +sys.path[0:0] = [""] + +from test import IntegrationTest, client_context, unittest +from test.utils import ( + OvertCommandListener, + rs_or_single_client, +) + +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts +from pymongo.errors import ( + ClientBulkWriteException, + DocumentTooLarge, + InvalidOperation, + NetworkTimeout, +) +from pymongo.monitoring import * +from pymongo.operations import * +from pymongo.write_concern import WriteConcern + +_IS_SYNC = True + + +class TestClientBulkWrite(IntegrationTest): + @client_context.require_version_min(8, 0, 0, -24) + def test_returns_error_if_no_namespace_provided(self): + client = rs_or_single_client() + self.addCleanup(client.close) + + models = [InsertOne(document={"a": "b"})] + with self.assertRaises(InvalidOperation) as context: + client.bulk_write(models=models) + self.assertIn( + "MongoClient.bulk_write requires a namespace to be provided for each write operation", + context.exception._message, + ) + + +# https://github.com/mongodb/specifications/tree/master/source/crud/tests +class TestClientBulkWriteCRUD(IntegrationTest): + @client_context.require_version_min(8, 0, 0, -24) + def test_batch_splits_if_num_operations_too_large(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + max_write_batch_size = (client_context.hello)["maxWriteBatchSize"] + models = [] + for _ in range(max_write_batch_size + 1): + models.append(InsertOne(namespace="db.coll", document={"a": "b"})) + self.addCleanup(client.db["coll"].drop) + + result = client.bulk_write(models=models) + self.assertEqual(result.inserted_count, max_write_batch_size + 1) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + first_event, second_event = bulk_write_events + self.assertEqual(len(first_event.command["ops"]), max_write_batch_size) + self.assertEqual(len(second_event.command["ops"]), 1) + self.assertEqual(first_event.operation_id, second_event.operation_id) + + @client_context.require_version_min(8, 0, 0, -24) + def test_batch_splits_if_ops_payload_too_large(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"] + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + + models = [] + num_models = int(max_message_size_bytes / max_bson_object_size + 1) + b_repeated = "b" * (max_bson_object_size - 500) + for _ in range(num_models): + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + self.addCleanup(client.db["coll"].drop) + + result = client.bulk_write(models=models) + self.assertEqual(result.inserted_count, num_models) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + first_event, second_event = bulk_write_events + self.assertEqual(len(first_event.command["ops"]), num_models - 1) + self.assertEqual(len(second_event.command["ops"]), 1) + self.assertEqual(first_event.operation_id, second_event.operation_id) + + @client_context.require_version_min(8, 0, 0, -24) + @client_context.require_failCommand_fail_point + def test_collects_write_concern_errors_across_batches(self): + listener = OvertCommandListener() + client = rs_or_single_client( + event_listeners=[listener], + retryWrites=False, + ) + self.addCleanup(client.close) + max_write_batch_size = (client_context.hello)["maxWriteBatchSize"] + + fail_command = { + "configureFailPoint": "failCommand", + "mode": {"times": 2}, + "data": { + "failCommands": ["bulkWrite"], + "writeConcernError": {"code": 91, "errmsg": "Replication is being shut down"}, + }, + } + with self.fail_point(fail_command): + models = [] + for _ in range(max_write_batch_size + 1): + models.append( + InsertOne( + namespace="db.coll", + document={"a": "b"}, + ) + ) + self.addCleanup(client.db["coll"].drop) + + with self.assertRaises(ClientBulkWriteException) as context: + client.bulk_write(models=models) + self.assertEqual(len(context.exception.write_concern_errors), 2) # type: ignore[arg-type] + self.assertIsNotNone(context.exception.partial_result) + self.assertEqual( + context.exception.partial_result.inserted_count, max_write_batch_size + 1 + ) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + @client_context.require_version_min(8, 0, 0, -24) + def test_collects_write_errors_across_batches_unordered(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + collection = client.db["coll"] + self.addCleanup(collection.drop) + collection.drop() + collection.insert_one(document={"_id": 1}) + + max_write_batch_size = (client_context.hello)["maxWriteBatchSize"] + models = [] + for _ in range(max_write_batch_size + 1): + models.append( + InsertOne( + namespace="db.coll", + document={"_id": 1}, + ) + ) + + with self.assertRaises(ClientBulkWriteException) as context: + client.bulk_write(models=models, ordered=False) + self.assertEqual(len(context.exception.write_errors), max_write_batch_size + 1) # type: ignore[arg-type] + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) + + @client_context.require_version_min(8, 0, 0, -24) + def test_collects_write_errors_across_batches_ordered(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + collection = client.db["coll"] + self.addCleanup(collection.drop) + collection.drop() + collection.insert_one(document={"_id": 1}) + + max_write_batch_size = (client_context.hello)["maxWriteBatchSize"] + models = [] + for _ in range(max_write_batch_size + 1): + models.append( + InsertOne( + namespace="db.coll", + document={"_id": 1}, + ) + ) + + with self.assertRaises(ClientBulkWriteException) as context: + client.bulk_write(models=models, ordered=True) + self.assertEqual(len(context.exception.write_errors), 1) # type: ignore[arg-type] + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 1) + + @client_context.require_version_min(8, 0, 0, -24) + def test_handles_cursor_requiring_getMore(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + collection = client.db["coll"] + self.addCleanup(collection.drop) + collection.drop() + + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + models = [] + a_repeated = "a" * (max_bson_object_size // 2) + b_repeated = "b" * (max_bson_object_size // 2) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": a_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": b_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + + result = client.bulk_write(models=models, verbose_results=True) + self.assertEqual(result.upserted_count, 2) + self.assertEqual(len(result.update_results), 2) + + get_more_event = False + for event in listener.started_events: + if event.command_name == "getMore": + get_more_event = True + self.assertTrue(get_more_event) + + @client_context.require_version_min(8, 0, 0, -24) + @client_context.require_no_standalone + def test_handles_cursor_requiring_getMore_within_transaction(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + collection = client.db["coll"] + self.addCleanup(collection.drop) + collection.drop() + + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + with client.start_session() as session: + session.start_transaction() + models = [] + a_repeated = "a" * (max_bson_object_size // 2) + b_repeated = "b" * (max_bson_object_size // 2) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": a_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": b_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + result = client.bulk_write(models=models, session=session, verbose_results=True) + + self.assertEqual(result.upserted_count, 2) + self.assertEqual(len(result.update_results), 2) + + get_more_event = False + for event in listener.started_events: + if event.command_name == "getMore": + get_more_event = True + self.assertTrue(get_more_event) + + @client_context.require_version_min(8, 0, 0, -24) + @client_context.require_failCommand_fail_point + def test_handles_getMore_error(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + collection = client.db["coll"] + self.addCleanup(collection.drop) + collection.drop() + + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + fail_command = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": {"failCommands": ["getMore"], "errorCode": 8}, + } + with self.fail_point(fail_command): + models = [] + a_repeated = "a" * (max_bson_object_size // 2) + b_repeated = "b" * (max_bson_object_size // 2) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": a_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + models.append( + UpdateOne( + namespace="db.coll", + filter={"_id": b_repeated}, + update={"$set": {"x": 1}}, + upsert=True, + ) + ) + + with self.assertRaises(ClientBulkWriteException) as context: + client.bulk_write(models=models, verbose_results=True) + self.assertIsNotNone(context.exception.error) + self.assertEqual(context.exception.error["code"], 8) + self.assertIsNotNone(context.exception.partial_result) + self.assertEqual(context.exception.partial_result.upserted_count, 2) + self.assertEqual(len(context.exception.partial_result.update_results), 1) + + get_more_event = False + kill_cursors_event = False + for event in listener.started_events: + if event.command_name == "getMore": + get_more_event = True + if event.command_name == "killCursors": + kill_cursors_event = True + self.assertTrue(get_more_event) + self.assertTrue(kill_cursors_event) + + @client_context.require_version_min(8, 0, 0, -24) + def test_returns_error_if_unacknowledged_too_large_insert(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + b_repeated = "b" * max_bson_object_size + + # Insert document. + models_insert = [InsertOne(namespace="db.coll", document={"a": b_repeated})] + with self.assertRaises(DocumentTooLarge): + client.bulk_write(models=models_insert, write_concern=WriteConcern(w=0)) + + # Replace document. + models_replace = [ReplaceOne(namespace="db.coll", filter={}, replacement={"a": b_repeated})] + with self.assertRaises(DocumentTooLarge): + client.bulk_write(models=models_replace, write_concern=WriteConcern(w=0)) + + def _setup_namespace_test_models(self): + max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"] + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + + ops_bytes = max_message_size_bytes - 1122 + num_models = ops_bytes // max_bson_object_size + remainder_bytes = ops_bytes % max_bson_object_size + + models = [] + b_repeated = "b" * (max_bson_object_size - 57) + for _ in range(num_models): + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + if remainder_bytes >= 217: + num_models += 1 + b_repeated = "b" * (remainder_bytes - 57) + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + return num_models, models + + @client_context.require_version_min(8, 0, 0, -24) + def test_no_batch_splits_if_new_namespace_is_not_too_large(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + num_models, models = self._setup_namespace_test_models() + models.append( + InsertOne( + namespace="db.coll", + document={"a": "b"}, + ) + ) + self.addCleanup(client.db["coll"].drop) + + # No batch splitting required. + result = client.bulk_write(models=models) + self.assertEqual(result.inserted_count, num_models + 1) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + + self.assertEqual(len(bulk_write_events), 1) + event = bulk_write_events[0] + + self.assertEqual(len(event.command["ops"]), num_models + 1) + self.assertEqual(len(event.command["nsInfo"]), 1) + self.assertEqual(event.command["nsInfo"][0]["ns"], "db.coll") + + @client_context.require_version_min(8, 0, 0, -24) + def test_batch_splits_if_new_namespace_is_too_large(self): + listener = OvertCommandListener() + client = rs_or_single_client(event_listeners=[listener]) + self.addCleanup(client.close) + + num_models, models = self._setup_namespace_test_models() + c_repeated = "c" * 200 + namespace = f"db.{c_repeated}" + models.append( + InsertOne( + namespace=namespace, + document={"a": "b"}, + ) + ) + self.addCleanup(client.db["coll"].drop) + self.addCleanup(client.db[c_repeated].drop) + + # Batch splitting required. + result = client.bulk_write(models=models) + self.assertEqual(result.inserted_count, num_models + 1) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + + self.assertEqual(len(bulk_write_events), 2) + first_event, second_event = bulk_write_events + + self.assertEqual(len(first_event.command["ops"]), num_models) + self.assertEqual(len(first_event.command["nsInfo"]), 1) + self.assertEqual(first_event.command["nsInfo"][0]["ns"], "db.coll") + + self.assertEqual(len(second_event.command["ops"]), 1) + self.assertEqual(len(second_event.command["nsInfo"]), 1) + self.assertEqual(second_event.command["nsInfo"][0]["ns"], namespace) + + @client_context.require_version_min(8, 0, 0, -24) + def test_returns_error_if_no_writes_can_be_added_to_ops(self): + client = rs_or_single_client() + self.addCleanup(client.close) + + max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"] + + # Document too large. + b_repeated = "b" * max_message_size_bytes + models = [InsertOne(namespace="db.coll", document={"a": b_repeated})] + with self.assertRaises(InvalidOperation) as context: + client.bulk_write(models=models) + self.assertIn("cannot do an empty bulk write", context.exception._message) + + # Namespace too large. + c_repeated = "c" * max_message_size_bytes + namespace = f"db.{c_repeated}" + models = [InsertOne(namespace=namespace, document={"a": "b"})] + with self.assertRaises(InvalidOperation) as context: + client.bulk_write(models=models) + self.assertIn("cannot do an empty bulk write", context.exception._message) + + @client_context.require_version_min(8, 0, 0, -24) + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + def test_returns_error_if_auto_encryption_configured(self): + opts = AutoEncryptionOpts( + key_vault_namespace="db.coll", + kms_providers={"aws": {"accessKeyId": "foo", "secretAccessKey": "bar"}}, + ) + client = rs_or_single_client(auto_encryption_opts=opts) + self.addCleanup(client.close) + + models = [InsertOne(namespace="db.coll", document={"a": "b"})] + with self.assertRaises(InvalidOperation) as context: + client.bulk_write(models=models) + self.assertIn( + "bulk_write does not currently support automatic encryption", context.exception._message + ) + + +# https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/tests/README.md#11-multi-batch-bulkwrites +class TestClientBulkWriteTimeout(IntegrationTest): + @client_context.require_version_min(8, 0, 0, -24) + @client_context.require_failCommand_fail_point + def test_timeout_in_multi_batch_bulk_write(self): + internal_client = rs_or_single_client(timeoutMS=None) + self.addCleanup(internal_client.close) + + collection = internal_client.db["coll"] + self.addCleanup(collection.drop) + collection.drop() + + max_bson_object_size = (client_context.hello)["maxBsonObjectSize"] + max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"] + fail_command = { + "configureFailPoint": "failCommand", + "mode": {"times": 2}, + "data": {"failCommands": ["bulkWrite"], "blockConnection": True, "blockTimeMS": 1010}, + } + with self.fail_point(fail_command): + models = [] + num_models = int(max_message_size_bytes / max_bson_object_size + 1) + b_repeated = "b" * (max_bson_object_size - 500) + for _ in range(num_models): + models.append( + InsertOne( + namespace="db.coll", + document={"a": b_repeated}, + ) + ) + + listener = OvertCommandListener() + client = rs_or_single_client( + event_listeners=[listener], + readConcernLevel="majority", + readPreference="primary", + timeoutMS=2000, + w="majority", + ) + self.addCleanup(client.close) + with self.assertRaises(ClientBulkWriteException) as context: + client.bulk_write(models=models) + self.assertIsInstance(context.exception.error, NetworkTimeout) + + bulk_write_events = [] + for event in listener.started_events: + if event.command_name == "bulkWrite": + bulk_write_events.append(event) + self.assertEqual(len(bulk_write_events), 2) diff --git a/test/test_cursor.py b/test/test_cursor.py index 12cb0cd57..e995bd529 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -232,7 +232,7 @@ class TestCursor(IntegrationTest): listener = AllowListEventListener("find", "getMore") coll = (rs_or_single_client(event_listeners=[listener]))[self.db.name].pymongo_test - # Tailable_await defaults. + # Tailable_defaults. coll.find(cursor_type=CursorType.TAILABLE_AWAIT).to_list() # find self.assertFalse("maxTimeMS" in listener.started_events[0].command) @@ -240,7 +240,7 @@ class TestCursor(IntegrationTest): self.assertFalse("maxTimeMS" in listener.started_events[1].command) listener.reset() - # Tailable_await with max_await_time_ms set. + # Tailable_with max_await_time_ms set. coll.find(cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms(99).to_list() # find self.assertEqual("find", listener.started_events[0].command_name) @@ -251,7 +251,7 @@ class TestCursor(IntegrationTest): self.assertEqual(99, listener.started_events[1].command["maxTimeMS"]) listener.reset() - # Tailable_await with max_time_ms and make sure list() works on synchronous cursors + # Tailable_with max_time_ms and make sure list() works on synchronous cursors if _IS_SYNC: list(coll.find(cursor_type=CursorType.TAILABLE_AWAIT).max_time_ms(99)) # type: ignore[call-overload] else: @@ -265,7 +265,7 @@ class TestCursor(IntegrationTest): self.assertFalse("maxTimeMS" in listener.started_events[1].command) listener.reset() - # Tailable_await with both max_time_ms and max_await_time_ms + # Tailable_with both max_time_ms and max_await_time_ms ( coll.find(cursor_type=CursorType.TAILABLE_AWAIT) .max_time_ms(99) @@ -1371,41 +1371,39 @@ class TestCursor(IntegrationTest): self.assertEqual("getMore", started[1].command_name) self.assertNotIn("$readPreference", started[1].command) + @client_context.require_version_min(4, 0) @client_context.require_replica_set def test_to_list_tailable(self): oplog = self.client.local.oplog.rs last = oplog.find().sort("$natural", pymongo.DESCENDING).limit(-1).next() ts = last["ts"] - + # Set maxAwaitTimeMS=1 to speed up the test and avoid blocking on the noop writer. c = oplog.find( {"ts": {"$gte": ts}}, cursor_type=pymongo.CursorType.TAILABLE_AWAIT, oplog_replay=True - ) - + ).max_await_time_ms(1) + self.addCleanup(c.close) docs = c.to_list() - self.assertGreaterEqual(len(docs), 1) def test_to_list_empty(self): c = self.db.does_not_exist.find() - docs = c.to_list() - self.assertEqual([], docs) - @client_context.require_replica_set + @client_context.require_change_streams def test_command_cursor_to_list(self): - c = self.db.test.aggregate([{"$changeStream": {}}]) - + # Set maxAwaitTimeMS=1 to speed up the test. + c = self.db.test.aggregate([{"$changeStream": {}}], maxAwaitTimeMS=1) + self.addCleanup(c.close) docs = c.to_list() - self.assertGreaterEqual(len(docs), 0) - @client_context.require_replica_set + @client_context.require_change_streams def test_command_cursor_to_list_empty(self): - c = self.db.does_not_exist.aggregate([{"$changeStream": {}}]) - + # Set maxAwaitTimeMS=1 to speed up the test. + c = self.db.does_not_exist.aggregate([{"$changeStream": {}}], maxAwaitTimeMS=1) + self.addCleanup(c.close) docs = c.to_list() - self.assertEqual([], docs) diff --git a/test/test_custom_types.py b/test/test_custom_types.py index 7daf83244..c30c62b1b 100644 --- a/test/test_custom_types.py +++ b/test/test_custom_types.py @@ -764,9 +764,7 @@ class TestGridFileCustomType(IntegrationTest): db.fs, _id=5, filename="my_file", - contentType="text/html", chunkSize=1000, - aliases=["foo"], metadata={"foo": "red", "bar": "blue"}, bar=3, baz="hello", @@ -780,13 +778,10 @@ class TestGridFileCustomType(IntegrationTest): self.assertEqual("my_file", two.filename) self.assertEqual(5, two._id) self.assertEqual(11, two.length) - self.assertEqual("text/html", two.content_type) self.assertEqual(1000, two.chunk_size) self.assertTrue(isinstance(two.upload_date, datetime.datetime)) - self.assertEqual(["foo"], two.aliases) self.assertEqual({"foo": "red", "bar": "blue"}, two.metadata) self.assertEqual(3, two.bar) - self.assertEqual(None, two.md5) for attr in [ "_id", @@ -805,7 +800,9 @@ class TestGridFileCustomType(IntegrationTest): class ChangeStreamsWCustomTypesTestMixin: @no_type_check def change_stream(self, *args, **kwargs): - return self.watched_target.watch(*args, **kwargs) + stream = self.watched_target.watch(*args, max_await_time_ms=1, **kwargs) + self.addCleanup(stream.close) + return stream @no_type_check def insert_and_check(self, change_stream, insert_doc, expected_doc): diff --git a/test/test_data_lake.py b/test/test_data_lake.py index a11bd9b9c..8ba83ab19 100644 --- a/test/test_data_lake.py +++ b/test/test_data_lake.py @@ -17,16 +17,16 @@ from __future__ import annotations import os import sys +from pathlib import Path import pytest sys.path[0:0] = [""] from test import IntegrationTest, client_context, unittest -from test.crud_v2_format import TestCrudV2 +from test.unified_format import generate_test_classes from test.utils import ( OvertCommandListener, - SpecTestCreator, rs_client_noauth, rs_or_single_client, ) @@ -100,30 +100,11 @@ class TestDataLakeProse(IntegrationTest): client[self.TEST_DB][self.TEST_COLLECTION].find_one() -class DataLakeTestSpec(TestCrudV2): - # Default test database and collection names. - TEST_DB = "test" - TEST_COLLECTION = "driverdata" +# Location of JSON test specifications. +TEST_PATH = Path(__file__).parent / "data_lake/unified" - @classmethod - @client_context.require_data_lake - def setUpClass(cls): - super().setUpClass() - - def setup_scenario(self, scenario_def): - # Spec tests MUST NOT insert data/drop collection for - # data lake testing. - pass - - -def create_test(scenario_def, test, name): - def run_scenario(self): - self.run_scenario(scenario_def, test) - - return run_scenario - - -SpecTestCreator(create_test, DataLakeTestSpec, _TEST_PATH).create_tests() +# Generate unified tests. +globals().update(generate_test_classes(TEST_PATH, module=__name__)) if __name__ == "__main__": diff --git a/test/test_examples.py b/test/test_examples.py index e003d8459..02b178586 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -747,6 +747,7 @@ class TestSampleShellCommands(IntegrationTest): done = False def insert_docs(): + nonlocal done while not done: db.inventory.insert_one({"username": "alice"}) db.inventory.delete_one({"username": "alice"}) @@ -760,17 +761,20 @@ class TestSampleShellCommands(IntegrationTest): cursor = db.inventory.watch() next(cursor) # End Changestream Example 1 + cursor.close() # Start Changestream Example 2 cursor = db.inventory.watch(full_document="updateLookup") next(cursor) # End Changestream Example 2 + cursor.close() # Start Changestream Example 3 resume_token = cursor.resume_token cursor = db.inventory.watch(resume_after=resume_token) next(cursor) # End Changestream Example 3 + cursor.close() # Start Changestream Example 4 pipeline = [ @@ -780,6 +784,7 @@ class TestSampleShellCommands(IntegrationTest): cursor = db.inventory.watch(pipeline=pipeline) next(cursor) # End Changestream Example 4 + cursor.close() finally: done = True t.join() diff --git a/test/test_pooling.py b/test/test_pooling.py index aa32f9f77..cd8a61735 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -416,7 +416,8 @@ class TestPooling(_TestPoolingBase): @client_context.require_failCommand_fail_point def test_csot_timeout_message(self): client = rs_or_single_client(appName="connectionTimeoutApp") - # Mock a connection failing due to timeout. + self.addCleanup(client.close) + # Mock an operation failing due to pymongo.timeout(). mock_connection_timeout = { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -440,8 +441,8 @@ class TestPooling(_TestPoolingBase): @client_context.require_failCommand_fail_point def test_socket_timeout_message(self): client = rs_or_single_client(socketTimeoutMS=500, appName="connectionTimeoutApp") - - # Mock a connection failing due to timeout. + self.addCleanup(client.close) + # Mock an operation failing due to socketTimeoutMS. mock_connection_timeout = { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -469,7 +470,7 @@ class TestPooling(_TestPoolingBase): 4, 9, 0 ) # configureFailPoint does not allow failure on handshake before 4.9, fixed in SERVER-49336 def test_connection_timeout_message(self): - # Mock a connection failing due to timeout. + # Mock a connection creation failing due to timeout. mock_connection_timeout = { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -481,9 +482,18 @@ class TestPooling(_TestPoolingBase): }, } + client = rs_or_single_client( + connectTimeoutMS=500, + socketTimeoutMS=500, + appName="connectionTimeoutApp", + heartbeatFrequencyMS=1000000, + ) + self.addCleanup(client.close) + client.admin.command("ping") + pool = get_pool(client) + pool.reset_without_pause() with self.fail_point(mock_connection_timeout): with self.assertRaises(Exception) as error: - client = rs_or_single_client(connectTimeoutMS=500, appName="connectionTimeoutApp") client.admin.command("ping") self.assertTrue( diff --git a/test/test_read_write_concern_spec.py b/test/test_read_write_concern_spec.py index 34aa1f754..3e37e8f9a 100644 --- a/test/test_read_write_concern_spec.py +++ b/test/test_read_write_concern_spec.py @@ -23,14 +23,13 @@ import warnings sys.path[0:0] = [""] from test import IntegrationTest, client_context, unittest +from test.unified_format import generate_test_classes from test.utils import ( EventListener, - SpecTestCreator, disable_replication, enable_replication, rs_or_single_client, ) -from test.utils_spec_runner import SpecRunner from pymongo import DESCENDING from pymongo.errors import ( @@ -321,25 +320,15 @@ def create_tests(): create_tests() -class TestOperation(SpecRunner): - # Location of JSON test specifications. - TEST_PATH = os.path.join(_TEST_PATH, "operation") - - def get_outcome_coll_name(self, outcome, collection): - """Spec says outcome has an optional 'collection.name'.""" - return outcome["collection"].get("name", collection.name) - - -def create_operation_test(scenario_def, test, name): - @client_context.require_test_commands - def run_scenario(self): - self.run_scenario(scenario_def, test) - - return run_scenario - - -test_creator = SpecTestCreator(create_operation_test, TestOperation, TestOperation.TEST_PATH) -test_creator.create_tests() +# Generate unified tests. +# PyMongo does not support MapReduce. +globals().update( + generate_test_classes( + os.path.join(_TEST_PATH, "operation"), + module=__name__, + expected_failures=["MapReduce .*"], + ) +) if __name__ == "__main__": diff --git a/test/test_retryable_reads_unified.py b/test/test_retryable_reads_unified.py index d33abd596..3f8740cf4 100644 --- a/test/test_retryable_reads_unified.py +++ b/test/test_retryable_reads_unified.py @@ -27,7 +27,7 @@ from test.unified_format import generate_test_classes TEST_PATH = Path(__file__).parent / "retryable_reads/unified" # Generate unified tests. -# PyMongo does not supportMapReduce, ListDatabaseObjects or ListCollectionObjects. +# PyMongo does not support MapReduce, ListDatabaseObjects or ListCollectionObjects. globals().update( generate_test_classes( TEST_PATH, diff --git a/test/test_sdam_monitoring_spec.py b/test/test_sdam_monitoring_spec.py index 5faee9b10..8e0a3cbbb 100644 --- a/test/test_sdam_monitoring_spec.py +++ b/test/test_sdam_monitoring_spec.py @@ -122,7 +122,7 @@ def compare_events(expected_dict, actual): elif expected_type == "topology_opening_event": if not isinstance(actual, monitoring.TopologyOpenedEvent): - return False, "Expected TopologyOpeningEvent, got %s" % (actual.__class__) + return False, "Expected TopologyOpenedEvent, got %s" % (actual.__class__) elif expected_type == "topology_description_changed_event": if not isinstance(actual, monitoring.TopologyDescriptionChangedEvent): @@ -179,7 +179,7 @@ class TestAllScenarios(IntegrationTest): def create_test(scenario_def): def run_scenario(self): - with client_knobs(events_queue_frequency=0.1): + with client_knobs(events_queue_frequency=0.05, min_heartbeat_interval=0.05): _run_scenario(self) def _run_scenario(self): @@ -216,7 +216,7 @@ def create_test(scenario_def): ) # Wait some time to catch possible lagging extra events. - time.sleep(0.5) + wait_until(lambda: topology._events.empty(), "publish lagging events") i = 0 while i < expected_len: @@ -273,7 +273,9 @@ class TestSdamMonitoring(IntegrationTest): def setUpClass(cls): super().setUpClass() # Speed up the tests by decreasing the event publish frequency. - cls.knobs = client_knobs(events_queue_frequency=0.1) + cls.knobs = client_knobs( + events_queue_frequency=0.1, heartbeat_frequency=0.1, min_heartbeat_interval=0.1 + ) cls.knobs.enable() cls.listener = ServerAndTopologyEventListener() retry_writes = client_context.supports_transactions() diff --git a/test/transactions/unified/client-bulkWrite.json b/test/transactions/unified/client-bulkWrite.json new file mode 100644 index 000000000..f8f1d9716 --- /dev/null +++ b/test/transactions/unified/client-bulkWrite.json @@ -0,0 +1,592 @@ +{ + "description": "client bulkWrite transactions", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + }, + { + "client": { + "id": "client_with_wmajority", + "uriOptions": { + "w": "majority" + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "session": { + "id": "session_with_wmajority", + "client": "client_with_wmajority" + } + } + ], + "_yamlAnchors": { + "namespace": "transaction-tests.coll0" + }, + "initialData": [ + { + "databaseName": "transaction-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + }, + { + "_id": 7, + "x": 77 + } + ] + } + ], + "tests": [ + { + "description": "client bulkWrite in a transaction", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "transaction-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": "transaction-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": "transaction-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": "transaction-tests.coll0", + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": "transaction-tests.coll0", + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": "transaction-tests.coll0", + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "0": { + "insertedId": 8 + } + }, + "updateResults": { + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + }, + "2": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + }, + "3": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 4 + } + }, + "deleteResults": { + "4": { + "deletedCount": 1 + }, + "5": { + "deletedCount": 2 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "transaction-tests.coll0" + } + ] + } + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction", + "databaseName": "admin", + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + }, + { + "description": "client writeConcern ignored for client bulkWrite in transaction", + "operations": [ + { + "object": "session_with_wmajority", + "name": "startTransaction", + "arguments": { + "writeConcern": { + "w": 1 + } + } + }, + { + "object": "client_with_wmajority", + "name": "clientBulkWrite", + "arguments": { + "session": "session_with_wmajority", + "models": [ + { + "insertOne": { + "namespace": "transaction-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + }, + { + "object": "session_with_wmajority", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client_with_wmajority", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "lsid": { + "$$sessionLsid": "session_with_wmajority" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + } + ], + "nsInfo": [ + { + "ns": "transaction-tests.coll0" + } + ] + } + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session_with_wmajority" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "w": 1 + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + }, + { + "_id": 7, + "x": 77 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + }, + { + "description": "client bulkWrite with writeConcern in a transaction causes a transaction error", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "session": "session0", + "writeConcern": { + "w": 1 + }, + "models": [ + { + "insertOne": { + "namespace": "transaction-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isClientError": true, + "errorContains": "Cannot set write concern after starting a transaction" + } + } + ] + } + ] +} diff --git a/test/transactions/unified/mongos-pin-auto.json b/test/transactions/unified/mongos-pin-auto.json index 93eac8bb7..27db52040 100644 --- a/test/transactions/unified/mongos-pin-auto.json +++ b/test/transactions/unified/mongos-pin-auto.json @@ -2004,6 +2004,104 @@ } ] }, + { + "description": "remain pinned after non-transient Interrupted error on clientBulkWrite bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 11601 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "database0.collection0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "errorLabelsOmit": [ + "TransientTransactionError" + ] + } + }, + { + "object": "testRunner", + "name": "assertSessionPinned", + "arguments": { + "session": "session0" + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ] + }, { "description": "unpin after transient connection error on insertOne insert", "operations": [ @@ -5175,6 +5273,202 @@ ] } ] + }, + { + "description": "unpin after transient connection error on clientBulkWrite bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "closeConnection": true + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "database0.collection0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "errorLabelsContain": [ + "TransientTransactionError" + ] + } + }, + { + "object": "testRunner", + "name": "assertSessionUnpinned", + "arguments": { + "session": "session0" + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ] + }, + { + "description": "unpin after transient ShutdownInProgress error on clientBulkWrite bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 91 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "database0.collection0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "errorLabelsContain": [ + "TransientTransactionError" + ] + } + }, + { + "object": "testRunner", + "name": "assertSessionUnpinned", + "arguments": { + "session": "session0" + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ] } ] } diff --git a/test/unified_format.py b/test/unified_format.py index df9e2af7f..0322d83cc 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -33,6 +33,7 @@ from collections import abc, defaultdict from test import ( IntegrationTest, client_context, + client_knobs, unittest, ) from test.helpers import ( @@ -73,6 +74,7 @@ from pymongo import ASCENDING, CursorType, MongoClient, _csot from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.errors import ( BulkWriteError, + ClientBulkWriteException, ConfigurationError, ConnectionFailure, EncryptionError, @@ -117,10 +119,18 @@ from pymongo.monitoring import ( _ServerEvent, _ServerHeartbeatEvent, ) -from pymongo.operations import SearchIndexModel +from pymongo.operations import ( + DeleteMany, + DeleteOne, + InsertOne, + ReplaceOne, + SearchIndexModel, + UpdateMany, + UpdateOne, +) from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference -from pymongo.results import BulkWriteResult +from pymongo.results import BulkWriteResult, ClientBulkWriteResult from pymongo.server_api import ServerApi from pymongo.server_description import ServerDescription from pymongo.server_selectors import Selection, writable_server_selector @@ -288,11 +298,61 @@ def parse_bulk_write_result(result): } +def parse_client_bulk_write_individual(op_type, result): + if op_type == "insert": + return {"insertedId": result.inserted_id} + if op_type == "update": + if result.upserted_id: + return { + "matchedCount": result.matched_count, + "modifiedCount": result.modified_count, + "upsertedId": result.upserted_id, + } + else: + return { + "matchedCount": result.matched_count, + "modifiedCount": result.modified_count, + } + if op_type == "delete": + return { + "deletedCount": result.deleted_count, + } + + +def parse_client_bulk_write_result(result): + insert_results, update_results, delete_results = {}, {}, {} + if result.has_verbose_results: + for idx, res in result.insert_results.items(): + insert_results[str(idx)] = parse_client_bulk_write_individual("insert", res) + for idx, res in result.update_results.items(): + update_results[str(idx)] = parse_client_bulk_write_individual("update", res) + for idx, res in result.delete_results.items(): + delete_results[str(idx)] = parse_client_bulk_write_individual("delete", res) + + return { + "deletedCount": result.deleted_count, + "insertedCount": result.inserted_count, + "matchedCount": result.matched_count, + "modifiedCount": result.modified_count, + "upsertedCount": result.upserted_count, + "insertResults": insert_results, + "updateResults": update_results, + "deleteResults": delete_results, + } + + def parse_bulk_write_error_result(error): write_result = BulkWriteResult(error.details, True) return parse_bulk_write_result(write_result) +def parse_client_bulk_write_error_result(error): + write_result = error.partial_result + if not write_result: + return None + return parse_client_bulk_write_result(write_result) + + class NonLazyCursor: """A find cursor proxy that creates the remote cursor when initialized.""" @@ -475,6 +535,11 @@ class EntityMapUtil: if entity_type == "client": kwargs: dict = {} observe_events = spec.get("observeEvents", []) + + # The unified tests use topologyOpeningEvent, we use topologyOpenedEvent + for i in range(len(observe_events)): + if "topologyOpeningEvent" == observe_events[i]: + observe_events[i] = "topologyOpenedEvent" ignore_commands = spec.get("ignoreCommandMonitoringEvents", []) observe_sensitive_commands = spec.get("observeSensitiveCommands", False) ignore_commands = [cmd.lower() for cmd in ignore_commands] @@ -924,6 +989,10 @@ class MatchEvaluatorUtil: self.test.assertIsInstance(actual, ServerHeartbeatFailedEvent) elif name == "topologyDescriptionChangedEvent": self.test.assertIsInstance(actual, TopologyDescriptionChangedEvent) + elif name == "topologyOpeningEvent": + self.test.assertIsInstance(actual, TopologyOpenedEvent) + elif name == "topologyClosedEvent": + self.test.assertIsInstance(actual, TopologyClosedEvent) else: raise Exception(f"Unsupported event type {name}") @@ -936,6 +1005,8 @@ def coerce_result(opname, result): return {"acknowledged": False} if opname == "bulkWrite": return parse_bulk_write_result(result) + if opname == "clientBulkWrite": + return parse_client_bulk_write_result(result) if opname == "insertOne": return {"insertedId": result.inserted_id} if opname == "insertMany": @@ -964,7 +1035,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.20") + SCHEMA_VERSION = Version.from_string("1.21") RUN_ON_LOAD_BALANCER = True RUN_ON_SERVERLESS = True TEST_SPEC: Any @@ -1028,8 +1099,18 @@ class UnifiedSpecTestMixinV1(IntegrationTest): if "retryable-writes" in cls.TEST_SPEC["description"]: raise unittest.SkipTest("MMAPv1 does not support retryWrites=True") + # Speed up the tests by decreasing the heartbeat frequency. + cls.knobs = client_knobs( + heartbeat_frequency=0.1, + min_heartbeat_interval=0.1, + kill_cursor_frequency=0.1, + events_queue_frequency=0.1, + ) + cls.knobs.enable() + @classmethod def tearDownClass(cls): + cls.knobs.disable() for client in cls.mongos_clients: client.close() super().tearDownClass() @@ -1131,20 +1212,27 @@ class UnifiedSpecTestMixinV1(IntegrationTest): expect_result = spec.get("expectResult") error_response = spec.get("errorResponse") if error_response: - self.match_evaluator.match_result(error_response, exception.details) + if isinstance(exception, ClientBulkWriteException): + self.match_evaluator.match_result(error_response, exception.error.details) + else: + self.match_evaluator.match_result(error_response, exception.details) if is_error: # already satisfied because exception was raised pass if is_client_error: + if isinstance(exception, ClientBulkWriteException): + error = exception.error + else: + error = exception # Connection errors are considered client errors. - if isinstance(exception, ConnectionFailure): - self.assertNotIsInstance(exception, NotPrimaryError) - elif isinstance(exception, (InvalidOperation, ConfigurationError, EncryptionError)): + if isinstance(error, ConnectionFailure): + self.assertNotIsInstance(error, NotPrimaryError) + elif isinstance(error, (InvalidOperation, ConfigurationError, EncryptionError)): pass else: - self.assertNotIsInstance(exception, PyMongoError) + self.assertNotIsInstance(error, PyMongoError) if is_timeout_error: self.assertIsInstance(exception, PyMongoError) @@ -1155,21 +1243,31 @@ class UnifiedSpecTestMixinV1(IntegrationTest): if error_contains: if isinstance(exception, BulkWriteError): errmsg = str(exception.details).lower() + elif isinstance(exception, ClientBulkWriteException): + errmsg = str(exception.details).lower() else: errmsg = str(exception).lower() self.assertIn(error_contains.lower(), errmsg) if error_code: - self.assertEqual(error_code, exception.details.get("code")) + if isinstance(exception, ClientBulkWriteException): + self.assertEqual(error_code, exception.error.details.get("code")) + else: + self.assertEqual(error_code, exception.details.get("code")) if error_code_name: - self.assertEqual(error_code_name, exception.details.get("codeName")) + if isinstance(exception, ClientBulkWriteException): + self.assertEqual(error_code, exception.error.details.get("codeName")) + else: + self.assertEqual(error_code_name, exception.details.get("codeName")) if error_labels_contain: + if isinstance(exception, ClientBulkWriteException): + error = exception.error + else: + error = exception labels = [ - err_label - for err_label in error_labels_contain - if exception.has_error_label(err_label) + err_label for err_label in error_labels_contain if error.has_error_label(err_label) ] self.assertEqual(labels, error_labels_contain) @@ -1182,8 +1280,13 @@ class UnifiedSpecTestMixinV1(IntegrationTest): if isinstance(exception, BulkWriteError): result = parse_bulk_write_error_result(exception) self.match_evaluator.match_result(expect_result, result) + elif isinstance(exception, ClientBulkWriteException): + result = parse_client_bulk_write_error_result(exception) + self.match_evaluator.match_result(expect_result, result) else: - self.fail(f"expectResult can only be specified with {BulkWriteError} exceptions") + self.fail( + f"expectResult can only be specified with {BulkWriteError} or {ClientBulkWriteException} exceptions" + ) return exception @@ -1461,6 +1564,8 @@ class UnifiedSpecTestMixinV1(IntegrationTest): target_opname = camel_to_snake(opname) if target_opname == "iterate_once": target_opname = "try_next" + if target_opname == "client_bulk_write": + target_opname = "bulk_write" try: cmd = getattr(target, target_opname) except AttributeError: diff --git a/test/utils.py b/test/utils.py index 0c08dca95..fa198b1c6 100644 --- a/test/utils.py +++ b/test/utils.py @@ -1251,10 +1251,10 @@ def prepare_spec_arguments(spec, arguments, opname, entity_map, with_txn_callbac # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = getattr(ReturnDocument, arguments.pop(arg_name).upper()) - elif c2s == "requests": + elif "bulk_write" in opname and (c2s == "requests" or c2s == "models"): # Parse each request into a bulk write model. requests = [] - for request in arguments["requests"]: + for request in arguments[c2s]: if "name" in request: # CRUD v2 format bulk_model = camel_to_upper_camel(request["name"]) @@ -1266,7 +1266,7 @@ def prepare_spec_arguments(spec, arguments, opname, entity_map, with_txn_callbac bulk_class = getattr(operations, camel_to_upper_camel(bulk_model)) bulk_arguments = camel_to_snake_args(spec) requests.append(bulk_class(**dict(bulk_arguments))) - arguments["requests"] = requests + arguments[c2s] = requests elif arg_name == "session": arguments["session"] = entity_map[arguments["session"]] elif opname == "open_download_stream" and arg_name == "id": diff --git a/test/versioned-api/crud-api-version-1.json b/test/versioned-api/crud-api-version-1.json index a387d0587..fe668620f 100644 --- a/test/versioned-api/crud-api-version-1.json +++ b/test/versioned-api/crud-api-version-1.json @@ -50,7 +50,8 @@ }, "apiDeprecationErrors": true } - ] + ], + "namespace": "versioned-api-tests.test" }, "initialData": [ { @@ -426,6 +427,85 @@ } ] }, + { + "description": "client bulkWrite appends declared API version", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "versioned-api-tests.test", + "document": { + "_id": 6, + "x": 6 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 6 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 6, + "x": 6 + } + } + ], + "nsInfo": [ + { + "ns": "versioned-api-tests.test" + } + ], + "apiVersion": "1", + "apiStrict": { + "$$unsetOrMatches": false + }, + "apiDeprecationErrors": true + } + } + } + ] + } + ] + }, { "description": "countDocuments appends declared API version", "operations": [ diff --git a/tools/synchro.py b/tools/synchro.py index 94f3d7f8f..e0af50229 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -39,6 +39,7 @@ replacements = { "AsyncDatabaseChangeStream": "DatabaseChangeStream", "AsyncClusterChangeStream": "ClusterChangeStream", "_AsyncBulk": "_Bulk", + "_AsyncClientBulk": "_ClientBulk", "AsyncConnection": "Connection", "async_command": "command", "async_receive_message": "receive_message", @@ -151,6 +152,7 @@ converted_tests = [ "pymongo_mocks.py", "utils_spec_runner.py", "test_client.py", + "test_client_bulk_write.py", "test_collection.py", "test_cursor.py", "test_database.py", @@ -272,15 +274,26 @@ def translate_docstrings(lines: list[str]) -> list[str]: # This sequence of replacements fixes the grammar issues caused by translating async -> sync if "an Async" in lines[i]: lines[i] = lines[i].replace("an Async", "a Async") + if "an 'Async" in lines[i]: + lines[i] = lines[i].replace("an 'Async", "a 'Async") if "An Async" in lines[i]: lines[i] = lines[i].replace("An Async", "A Async") + if "An 'Async" in lines[i]: + lines[i] = lines[i].replace("An 'Async", "A 'Async") if "an asynchronous" in lines[i]: lines[i] = lines[i].replace("an asynchronous", "a") if "An asynchronous" in lines[i]: lines[i] = lines[i].replace("An asynchronous", "A") + # This ensures docstring links are for `pymongo.X` instead of `pymongo.synchronous.X` + if "pymongo.asynchronous" in lines[i] and "import" not in lines[i]: + lines[i] = lines[i].replace("pymongo.asynchronous", "pymongo") lines[i] = lines[i].replace(k, replacements[k]) if "Sync" in lines[i] and "Synchronous" not in lines[i] and replacements[k] in lines[i]: lines[i] = lines[i].replace("Sync", "") + if "async for" in lines[i] or "async with" in lines[i] or "async def" in lines[i]: + lines[i] = lines[i].replace("async ", "") + if "await " in lines[i] and "tailable" not in lines[i]: + lines[i] = lines[i].replace("await ", "") for i in range(len(lines)): for k in docstring_replacements: # type: ignore[assignment] if f":param {k[1]}: **Not supported by {k[0]}**." in lines[i]: