mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-4550 Add MongoClient.bulk_write API (#1745)

Browse Source
This commit is contained in:
Shruti Sridhar 2024-08-06 11:10:01 -07:00 committed by GitHub
parent da2465f2c7
commit d08fec6342
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 11221 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
pymongo/_client_bulk_shared.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2024-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
"""Constants, types, and classes shared across Client Bulk Write API implementations."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, NoReturn
from pymongo.errors import ClientBulkWriteException, OperationFailure
from pymongo.helpers_shared import _get_wce_doc
if TYPE_CHECKING:
from pymongo.typings import _DocumentOut
def _merge_command(
ops: list[tuple[str, Mapping[str, Any]]],
offset: int,
full_result: MutableMapping[str, Any],
result: Mapping[str, Any],
) -> None:
"""Merge result of a single bulk write batch into the full result."""
if result.get("error"):
full_result["error"] = result["error"]
full_result["nInserted"] += result.get("nInserted", 0)
full_result["nDeleted"] += result.get("nDeleted", 0)
full_result["nMatched"] += result.get("nMatched", 0)
full_result["nModified"] += result.get("nModified", 0)
full_result["nUpserted"] += result.get("nUpserted", 0)
write_errors = result.get("writeErrors")
if write_errors:
for doc in write_errors:
# Leave the server response intact for APM.
replacement = doc.copy()
original_index = doc["idx"] + offset
replacement["idx"] = original_index
# Add the failed operation to the error document.
replacement["op"] = ops[original_index][1]
full_result["writeErrors"].append(replacement)
wce = _get_wce_doc(result)
if wce:
full_result["writeConcernErrors"].append(wce)
def _throw_client_bulk_write_exception(
full_result: _DocumentOut, verbose_results: bool
) -> NoReturn:
"""Raise a ClientBulkWriteException from the full result."""
# retryWrites on MMAPv1 should raise an actionable error.
if full_result["writeErrors"]:
full_result["writeErrors"].sort(key=lambda error: error["idx"])
err = full_result["writeErrors"][0]
code = err["code"]
msg = err["errmsg"]
if code == 20 and msg.startswith("Transaction numbers"):
errmsg = (
"This MongoDB deployment does not support "
"retryable writes. Please add retryWrites=false "
"to your connection string."
)
raise OperationFailure(errmsg, code, full_result)
raise ClientBulkWriteException(full_result, verbose_results)

788
pymongo/asynchronous/client_bulk.py Normal file
View File

@ -0,0 +1,788 @@
# Copyright 2024-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The client-level bulk write operations interface.
.. versionadded:: 4.9
"""
from __future__ import annotations
import copy
import datetime
import logging
from collections.abc import MutableMapping
from itertools import islice
from typing import (
TYPE_CHECKING,
Any,
Mapping,
Optional,
Type,
Union,
)
from bson.objectid import ObjectId
from bson.raw_bson import RawBSONDocument
from pymongo import _csot, common
from pymongo.asynchronous.client_session import AsyncClientSession, _validate_session_write_concern
from pymongo.asynchronous.collection import AsyncCollection
from pymongo.asynchronous.command_cursor import AsyncCommandCursor
from pymongo.asynchronous.database import AsyncDatabase
from pymongo.asynchronous.helpers import _handle_reauth
if TYPE_CHECKING:
from pymongo.asynchronous.mongo_client import AsyncMongoClient
from pymongo.asynchronous.pool import AsyncConnection
from pymongo._client_bulk_shared import (
_merge_command,
_throw_client_bulk_write_exception,
)
from pymongo.common import (
validate_is_document_type,
validate_ok_for_replace,
validate_ok_for_update,
)
from pymongo.errors import (
ConfigurationError,
ConnectionFailure,
InvalidOperation,
NotPrimaryError,
OperationFailure,
WaitQueueTimeoutError,
)
from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES
from pymongo.logger import _COMMAND_LOGGER, _CommandStatusMessage, _debug_log
from pymongo.message import (
_ClientBulkWriteContext,
_convert_client_bulk_exception,
_convert_exception,
_convert_write_result,
_randint,
)
from pymongo.read_preferences import ReadPreference
from pymongo.results import (
ClientBulkWriteResult,
DeleteResult,
InsertOneResult,
UpdateResult,
)
from pymongo.typings import _DocumentOut, _Pipeline
from pymongo.write_concern import WriteConcern
_IS_SYNC = False
class _AsyncClientBulk:
"""The private guts of the client-level bulk write API."""
def __init__(
self,
client: AsyncMongoClient,
write_concern: WriteConcern,
ordered: bool = True,
bypass_document_validation: Optional[bool] = None,
comment: Optional[str] = None,
let: Optional[Any] = None,
verbose_results: bool = False,
) -> None:
"""Initialize a _AsyncClientBulk instance."""
self.client = client
self.write_concern = write_concern
self.let = let
if self.let is not None:
common.validate_is_document_type("let", self.let)
self.ordered = ordered
self.bypass_doc_val = bypass_document_validation
self.comment = comment
self.verbose_results = verbose_results
self.ops: list[tuple[str, Mapping[str, Any]]] = []
self.idx_offset: int = 0
self.total_ops: int = 0
self.executed = False
self.uses_upsert = False
self.uses_collation = False
self.uses_array_filters = False
self.uses_hint_update = False
self.uses_hint_delete = False
self.is_retryable = self.client.options.retry_writes
self.retrying = False
self.started_retryable_write = False
@property
def bulk_ctx_class(self) -> Type[_ClientBulkWriteContext]:
return _ClientBulkWriteContext
def add_insert(self, namespace: str, document: _DocumentOut) -> None:
"""Add an insert document to the list of ops."""
validate_is_document_type("document", document)
# Generate ObjectId client side.
if not (isinstance(document, RawBSONDocument) or "_id" in document):
document["_id"] = ObjectId()
cmd = {"insert": namespace, "document": document}
self.ops.append(("insert", cmd))
self.total_ops += 1
def add_update(
self,
namespace: str,
selector: Mapping[str, Any],
update: Union[Mapping[str, Any], _Pipeline],
multi: bool = False,
upsert: Optional[bool] = None,
collation: Optional[Mapping[str, Any]] = None,
array_filters: Optional[list[Mapping[str, Any]]] = None,
hint: Union[str, dict[str, Any], None] = None,
) -> None:
"""Create an update document and add it to the list of ops."""
validate_ok_for_update(update)
cmd = {
"update": namespace,
"filter": selector,
"updateMods": update,
"multi": multi,
}
if upsert is not None:
self.uses_upsert = True
cmd["upsert"] = upsert
if array_filters is not None:
self.uses_array_filters = True
cmd["arrayFilters"] = array_filters
if hint is not None:
self.uses_hint_update = True
cmd["hint"] = hint
if collation is not None:
self.uses_collation = True
cmd["collation"] = collation
if multi:
# A bulk_write containing an update_many is not retryable.
self.is_retryable = False
self.ops.append(("update", cmd))
self.total_ops += 1
def add_replace(
self,
namespace: str,
selector: Mapping[str, Any],
replacement: Mapping[str, Any],
upsert: Optional[bool] = None,
collation: Optional[Mapping[str, Any]] = None,
hint: Union[str, dict[str, Any], None] = None,
) -> None:
"""Create a replace document and add it to the list of ops."""
validate_ok_for_replace(replacement)
cmd = {
"update": namespace,
"filter": selector,
"updateMods": replacement,
"multi": False,
}
if upsert is not None:
self.uses_upsert = True
cmd["upsert"] = upsert
if hint is not None:
self.uses_hint_update = True
cmd["hint"] = hint
if collation is not None:
self.uses_collation = True
cmd["collation"] = collation
self.ops.append(("replace", cmd))
self.total_ops += 1
def add_delete(
self,
namespace: str,
selector: Mapping[str, Any],
multi: bool,
collation: Optional[Mapping[str, Any]] = None,
hint: Union[str, dict[str, Any], None] = None,
) -> None:
"""Create a delete document and add it to the list of ops."""
cmd = {"delete": namespace, "filter": selector, "multi": multi}
if hint is not None:
self.uses_hint_delete = True
cmd["hint"] = hint
if collation is not None:
self.uses_collation = True
cmd["collation"] = collation
if multi:
# A bulk_write containing an update_many is not retryable.
self.is_retryable = False
self.ops.append(("delete", cmd))
self.total_ops += 1
@_handle_reauth
async def write_command(
self,
bwc: _ClientBulkWriteContext,
cmd: MutableMapping[str, Any],
request_id: int,
msg: Union[bytes, dict[str, Any]],
op_docs: list[Mapping[str, Any]],
ns_docs: list[Mapping[str, Any]],
client: AsyncMongoClient,
) -> dict[str, Any]:
"""A proxy for AsyncConnection.write_command that handles event publishing."""
cmd["ops"] = op_docs
cmd["nsInfo"] = ns_docs
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.STARTED,
command=cmd,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
bwc._start(cmd, request_id, op_docs, ns_docs)
try:
reply = await bwc.conn.write_command(request_id, msg, bwc.codec) # type: ignore[misc, arg-type]
duration = datetime.datetime.now() - bwc.start_time
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.SUCCEEDED,
durationMS=duration,
reply=reply,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
bwc._succeed(request_id, reply, duration) # type: ignore[arg-type]
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, (NotPrimaryError, OperationFailure)):
failure: _DocumentOut = exc.details # type: ignore[assignment]
else:
failure = _convert_exception(exc)
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.FAILED,
durationMS=duration,
failure=failure,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
isServerSideError=isinstance(exc, OperationFailure),
)
if bwc.publish:
bwc._fail(request_id, failure, duration)
# Top-level error will be embedded in ClientBulkWriteException.
reply = {"error": exc}
finally:
bwc.start_time = datetime.datetime.now()
return reply # type: ignore[return-value]
async def unack_write(
self,
bwc: _ClientBulkWriteContext,
cmd: MutableMapping[str, Any],
request_id: int,
msg: bytes,
op_docs: list[Mapping[str, Any]],
ns_docs: list[Mapping[str, Any]],
client: AsyncMongoClient,
) -> Optional[Mapping[str, Any]]:
"""A proxy for AsyncConnection.unack_write that handles event publishing."""
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.STARTED,
command=cmd,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
cmd = bwc._start(cmd, request_id, op_docs, ns_docs)
try:
result = await bwc.conn.unack_write(msg, bwc.max_bson_size) # type: ignore[func-returns-value, misc, override]
duration = datetime.datetime.now() - bwc.start_time
if result is not None:
reply = _convert_write_result(bwc.name, cmd, result) # type: ignore[arg-type]
else:
# Comply with APM spec.
reply = {"ok": 1}
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.SUCCEEDED,
durationMS=duration,
reply=reply,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
bwc._succeed(request_id, reply, duration)
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, OperationFailure):
failure: _DocumentOut = _convert_write_result(bwc.name, cmd, exc.details) # type: ignore[arg-type]
elif isinstance(exc, NotPrimaryError):
failure = exc.details # type: ignore[assignment]
else:
failure = _convert_exception(exc)
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.FAILED,
durationMS=duration,
failure=failure,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
isServerSideError=isinstance(exc, OperationFailure),
)
if bwc.publish:
assert bwc.start_time is not None
bwc._fail(request_id, failure, duration)
# Top-level error will be embedded in ClientBulkWriteException.
reply = {"error": exc}
finally:
bwc.start_time = datetime.datetime.now()
return result # type: ignore[return-value]
async def _execute_batch_unack(
self,
bwc: _ClientBulkWriteContext,
cmd: dict[str, Any],
ops: list[tuple[str, Mapping[str, Any]]],
) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Executes a batch of bulkWrite server commands (unack)."""
request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops)
await self.unack_write(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type]
return to_send_ops, to_send_ns
async def _execute_batch(
self,
bwc: _ClientBulkWriteContext,
cmd: dict[str, Any],
ops: list[tuple[str, Mapping[str, Any]]],
) -> tuple[dict[str, Any], list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Executes a batch of bulkWrite server commands (ack)."""
request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops)
result = await self.write_command(
bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client
) # type: ignore[arg-type]
await self.client._process_response(result, bwc.session) # type: ignore[arg-type]
return result, to_send_ops, to_send_ns # type: ignore[return-value]
async def _process_results_cursor(
self,
full_result: MutableMapping[str, Any],
result: MutableMapping[str, Any],
conn: AsyncConnection,
session: Optional[AsyncClientSession],
) -> None:
"""Internal helper for processing the server reply command cursor."""
if result.get("cursor"):
coll = AsyncCollection(
database=AsyncDatabase(self.client, "admin"),
name="$cmd.bulkWrite",
)
cmd_cursor = AsyncCommandCursor(
coll,
result["cursor"],
conn.address,
session=session,
explicit_session=session is not None,
comment=self.comment,
)
await cmd_cursor._maybe_pin_connection(conn)
# Iterate the cursor to get individual write results.
try:
async for doc in cmd_cursor:
original_index = doc["idx"] + self.idx_offset
op_type, op = self.ops[original_index]
if not doc["ok"]:
result["writeErrors"].append(doc)
if self.ordered:
return
# Record individual write result.
if doc["ok"] and self.verbose_results:
if op_type == "insert":
inserted_id = op["document"]["_id"]
res = InsertOneResult(inserted_id, acknowledged=True) # type: ignore[assignment]
if op_type in ["update", "replace"]:
op_type = "update"
res = UpdateResult(doc, acknowledged=True, in_client_bulk=True) # type: ignore[assignment]
if op_type == "delete":
res = DeleteResult(doc, acknowledged=True) # type: ignore[assignment]
full_result[f"{op_type}Results"][original_index] = res
except Exception as exc:
# Attempt to close the cursor, then raise top-level error.
if cmd_cursor.alive:
await cmd_cursor.close()
result["error"] = _convert_client_bulk_exception(exc)
async def _execute_command(
self,
write_concern: WriteConcern,
session: Optional[AsyncClientSession],
conn: AsyncConnection,
op_id: int,
retryable: bool,
full_result: MutableMapping[str, Any],
final_write_concern: Optional[WriteConcern] = None,
) -> None:
"""Internal helper for executing batches of bulkWrite commands."""
db_name = "admin"
cmd_name = "bulkWrite"
listeners = self.client._event_listeners
# AsyncConnection.command validates the session, but we use
# AsyncConnection.write_command
conn.validate_session(self.client, session)
bwc = self.bulk_ctx_class(
db_name,
cmd_name,
conn,
op_id,
listeners, # type: ignore[arg-type]
session,
self.client.codec_options,
)
while self.idx_offset < self.total_ops:
# If this is the last possible batch, use the
# final write concern.
if self.total_ops - self.idx_offset <= bwc.max_write_batch_size:
write_concern = final_write_concern or write_concern
# Construct the server command, specifying the relevant options.
cmd = {"bulkWrite": 1}
cmd["errorsOnly"] = not self.verbose_results
cmd["ordered"] = self.ordered # type: ignore[assignment]
not_in_transaction = session and not session.in_transaction
if not_in_transaction or not session:
_csot.apply_write_concern(cmd, write_concern)
if self.bypass_doc_val is not None:
cmd["bypassDocumentValidation"] = self.bypass_doc_val
if self.comment:
cmd["comment"] = self.comment # type: ignore[assignment]
if self.let:
cmd["let"] = self.let
if session:
# Start a new retryable write unless one was already
# started for this command.
if retryable and not self.started_retryable_write:
session._start_retryable_write()
self.started_retryable_write = True
session._apply_to(cmd, retryable, ReadPreference.PRIMARY, conn)
conn.send_cluster_time(cmd, session, self.client)
conn.add_server_api(cmd)
# CSOT: apply timeout before encoding the command.
conn.apply_timeout(self.client, cmd)
ops = islice(self.ops, self.idx_offset, None)
# Run as many ops as possible in one server command.
if write_concern.acknowledged:
raw_result, to_send_ops, _ = await self._execute_batch(bwc, cmd, ops) # type: ignore[arg-type]
result = copy.deepcopy(raw_result)
# Top-level server/network error.
if result.get("error"):
error = result["error"]
retryable_top_level_error = (
isinstance(error.details, dict)
and error.details.get("code", 0) in _RETRYABLE_ERROR_CODES
)
retryable_network_error = isinstance(
error, ConnectionFailure
) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError))
# Synthesize the full bulk result without modifying the
# current one because this write operation may be retried.
if retryable and (retryable_top_level_error or retryable_network_error):
full = copy.deepcopy(full_result)
_merge_command(self.ops, self.idx_offset, full, result)
_throw_client_bulk_write_exception(full, self.verbose_results)
else:
_merge_command(self.ops, self.idx_offset, full_result, result)
_throw_client_bulk_write_exception(full_result, self.verbose_results)
result["error"] = None
result["writeErrors"] = []
if result.get("nErrors", 0) < len(to_send_ops):
full_result["anySuccessful"] = True
# Top-level command error.
if not result["ok"]:
result["error"] = raw_result
_merge_command(self.ops, self.idx_offset, full_result, result)
break
if retryable:
# Retryable writeConcernErrors halt the execution of this batch.
wce = result.get("writeConcernError", {})
if wce.get("code", 0) in _RETRYABLE_ERROR_CODES:
# Synthesize the full bulk result without modifying the
# current one because this write operation may be retried.
full = copy.deepcopy(full_result)
_merge_command(self.ops, self.idx_offset, full, result)
_throw_client_bulk_write_exception(full, self.verbose_results)
# Process the server reply as a command cursor.
await self._process_results_cursor(full_result, result, conn, session)
# Merge this batch's results with the full results.
_merge_command(self.ops, self.idx_offset, full_result, result)
# We're no longer in a retry once a command succeeds.
self.retrying = False
self.started_retryable_write = False
else:
to_send_ops, _ = await self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type]
self.idx_offset += len(to_send_ops)
# We halt execution if we hit a top-level error,
# or an individual error in an ordered bulk write.
if full_result["error"] or (self.ordered and full_result["writeErrors"]):
break
async def execute_command(
self,
session: Optional[AsyncClientSession],
operation: str,
) -> MutableMapping[str, Any]:
"""Execute commands with w=1 WriteConcern."""
full_result: MutableMapping[str, Any] = {
"anySuccessful": False,
"error": None,
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nDeleted": 0,
"insertResults": {},
"updateResults": {},
"deleteResults": {},
}
op_id = _randint()
async def retryable_bulk(
session: Optional[AsyncClientSession],
conn: AsyncConnection,
retryable: bool,
) -> None:
if conn.max_wire_version < 25:
raise InvalidOperation(
"MongoClient.bulk_write requires MongoDB server version 8.0+."
)
await self._execute_command(
self.write_concern,
session,
conn,
op_id,
retryable,
full_result,
)
await self.client._retryable_write(
self.is_retryable,
retryable_bulk,
session,
operation,
bulk=self,
operation_id=op_id,
)
if full_result["error"] or full_result["writeErrors"] or full_result["writeConcernErrors"]:
_throw_client_bulk_write_exception(full_result, self.verbose_results)
return full_result
async def execute_command_unack_unordered(
self,
conn: AsyncConnection,
) -> None:
"""Execute commands with OP_MSG and w=0 writeConcern, unordered."""
db_name = "admin"
cmd_name = "bulkWrite"
listeners = self.client._event_listeners
op_id = _randint()
bwc = self.bulk_ctx_class(
db_name,
cmd_name,
conn,
op_id,
listeners, # type: ignore[arg-type]
None,
self.client.codec_options,
)
while self.idx_offset < self.total_ops:
# Construct the server command, specifying the relevant options.
cmd = {"bulkWrite": 1}
cmd["errorsOnly"] = not self.verbose_results
cmd["ordered"] = self.ordered # type: ignore[assignment]
if self.bypass_doc_val is not None:
cmd["bypassDocumentValidation"] = self.bypass_doc_val
cmd["writeConcern"] = {"w": 0} # type: ignore[assignment]
if self.comment:
cmd["comment"] = self.comment # type: ignore[assignment]
if self.let:
cmd["let"] = self.let
conn.add_server_api(cmd)
ops = islice(self.ops, self.idx_offset, None)
# Run as many ops as possible in one server command.
to_send_ops, _ = await self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type]
self.idx_offset += len(to_send_ops)
async def execute_command_unack_ordered(
self,
conn: AsyncConnection,
) -> None:
"""Execute commands with OP_MSG and w=0 WriteConcern, ordered."""
full_result: MutableMapping[str, Any] = {
"anySuccessful": False,
"error": None,
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nDeleted": 0,
"insertResults": {},
"updateResults": {},
"deleteResults": {},
}
# Ordered bulk writes have to be acknowledged so that we stop
# processing at the first error, even when the application
# specified unacknowledged writeConcern.
initial_write_concern = WriteConcern()
op_id = _randint()
try:
await self._execute_command(
initial_write_concern,
None,
conn,
op_id,
False,
full_result,
self.write_concern,
)
except OperationFailure:
pass
async def execute_no_results(
self,
conn: AsyncConnection,
) -> None:
"""Execute all operations, returning no results (w=0)."""
if self.uses_collation:
raise ConfigurationError("Collation is unsupported for unacknowledged writes.")
if self.uses_array_filters:
raise ConfigurationError("arrayFilters is unsupported for unacknowledged writes.")
# Cannot have both unacknowledged writes and bypass document validation.
if self.bypass_doc_val is not None:
raise OperationFailure(
"Cannot set bypass_document_validation with unacknowledged write concern"
)
if self.ordered:
return await self.execute_command_unack_ordered(conn)
return await self.execute_command_unack_unordered(conn)
async def execute(
self,
session: Optional[AsyncClientSession],
operation: str,
) -> Any:
"""Execute operations."""
if not self.ops:
raise InvalidOperation("No operations to execute")
if self.executed:
raise InvalidOperation("Bulk operations can only be executed once.")
self.executed = True
session = _validate_session_write_concern(session, self.write_concern)
if not self.write_concern.acknowledged:
async with await self.client._conn_for_writes(session, operation) as connection:
if connection.max_wire_version < 25:
raise InvalidOperation(
"MongoClient.bulk_write requires MongoDB server version 8.0+."
)
await self.execute_no_results(connection)
return ClientBulkWriteResult(None, False, False) # type: ignore[arg-type]
result = await self.execute_command(session, operation)
return ClientBulkWriteResult(
result,
self.write_concern.acknowledged,
self.verbose_results,
)

175
pymongo/asynchronous/mongo_client.py
View File

@ -61,6 +61,7 @@ from bson.timestamp import Timestamp
from pymongo import _csot, common, helpers_shared, uri_parser
from pymongo.asynchronous import client_session, database, periodic_executor
from pymongo.asynchronous.change_stream import AsyncChangeStream, AsyncClusterChangeStream
from pymongo.asynchronous.client_bulk import _AsyncClientBulk
from pymongo.asynchronous.client_session import _EmptyServerSession
from pymongo.asynchronous.command_cursor import AsyncCommandCursor
from pymongo.asynchronous.settings import TopologySettings
@ -69,6 +70,7 @@ from pymongo.client_options import ClientOptions
from pymongo.errors import (
AutoReconnect,
BulkWriteError,
ClientBulkWriteException,
ConfigurationError,
ConnectionFailure,
InvalidOperation,
@ -83,8 +85,17 @@ from pymongo.lock import _HAS_REGISTER_AT_FORK, _ALock, _create_lock, _release_l
from pymongo.logger import _CLIENT_LOGGER, _log_or_warn
from pymongo.message import _CursorAddress, _GetMore, _Query
from pymongo.monitoring import ConnectionClosedReason
from pymongo.operations import _Op
from pymongo.operations import (
DeleteMany,
DeleteOne,
InsertOne,
ReplaceOne,
UpdateMany,
UpdateOne,
_Op,
)
from pymongo.read_preferences import ReadPreference, _ServerMode
from pymongo.results import ClientBulkWriteResult
from pymongo.server_selectors import writable_server_selector
from pymongo.server_type import SERVER_TYPE
from pymongo.topology_description import TOPOLOGY_TYPE, TopologyDescription
@ -130,6 +141,15 @@ _ReadCall = Callable[
_IS_SYNC = False
_WriteOp = Union[
InsertOne,
DeleteOne,
DeleteMany,
ReplaceOne,
UpdateOne,
UpdateMany,
]
class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]):
HOST = "localhost"
@ -1720,7 +1740,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]):
retryable: bool,
func: _WriteCall[T],
session: Optional[AsyncClientSession],
bulk: Optional[_AsyncBulk],
bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]],
operation: str,
operation_id: Optional[int] = None,
) -> T:
@ -1750,7 +1770,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]):
self,
func: _WriteCall[T] | _ReadCall[T],
session: Optional[AsyncClientSession],
bulk: Optional[_AsyncBulk],
bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]],
operation: str,
is_read: bool = False,
address: Optional[_Address] = None,
@ -1833,7 +1853,7 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]):
func: _WriteCall[T],
session: Optional[AsyncClientSession],
operation: str,
bulk: Optional[_AsyncBulk] = None,
bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]] = None,
operation_id: Optional[int] = None,
) -> T:
"""Execute an operation with consecutive retries if possible
@ -2204,10 +2224,134 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]):
session=session,
)
@_csot.apply
async def bulk_write(
self,
models: Sequence[_WriteOp[_DocumentType]],
session: Optional[AsyncClientSession] = None,
ordered: bool = True,
verbose_results: bool = False,
bypass_document_validation: Optional[bool] = None,
comment: Optional[Any] = None,
let: Optional[Mapping] = None,
write_concern: Optional[WriteConcern] = None,
) -> ClientBulkWriteResult:
"""Send a batch of write operations, potentially across multiple namespaces, to the server.
Requests are passed as a list of write operation instances (
:class:`~pymongo.operations.InsertOne`,
:class:`~pymongo.operations.UpdateOne`,
:class:`~pymongo.operations.UpdateMany`,
:class:`~pymongo.operations.ReplaceOne`,
:class:`~pymongo.operations.DeleteOne`, or
:class:`~pymongo.operations.DeleteMany`).
>>> async for doc in db.test.find({}):
... print(doc)
...
{'x': 1, '_id': ObjectId('54f62e60fba5226811f634ef')}
{'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')}
...
>>> async for doc in db.coll.find({}):
... print(doc)
...
{'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')}
...
>>> # DeleteMany, UpdateOne, and UpdateMany are also available.
>>> from pymongo import InsertOne, DeleteOne, ReplaceOne
>>> models = [InsertOne(namespace="db.test", document={'y': 1}),
... DeleteOne(namespace="db.test", filter={'x': 1}),
... InsertOne(namespace="db.coll", document={'y': 2}),
... ReplaceOne(namespace="db.test", filter={'w': 1}, replacement={'z': 1}, upsert=True)]
>>> result = await client.bulk_write(models=models)
>>> result.inserted_count
2
>>> result.deleted_count
1
>>> result.modified_count
0
>>> result.upserted_ids
{3: ObjectId('54f62ee28891e756a6e1abd5')}
>>> async for doc in db.test.find({}):
... print(doc)
...
{'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')}
{'y': 1, '_id': ObjectId('54f62ee2fba5226811f634f1')}
{'z': 1, '_id': ObjectId('54f62ee28891e756a6e1abd5')}
...
>>> async for doc in db.coll.find({}):
... print(doc)
...
{'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')}
{'y': 2, '_id': ObjectId('507f1f77bcf86cd799439012')}
:param models: A list of write operation instances.
:param session: (optional) An instance of
:class:`~pymongo.asynchronous.client_session.AsyncClientSession`.
:param ordered: If ``True`` (the default), requests will be
performed on the server serially, in the order provided. If an error
occurs all remaining operations are aborted. If ``False``, requests
will be still performed on the server serially, in the order provided,
but all operations will be attempted even if any errors occur.
:param verbose_results: If ``True``, detailed results for each
successful operation will be included in the returned
:class:`~pymongo.results.ClientBulkWriteResult`. Default is ``False``.
:param bypass_document_validation: (optional) If ``True``, allows the
write to opt-out of document level validation. Default is ``False``.
:param comment: (optional) A user-provided comment to attach to this
command.
:param let: (optional) Map of parameter names and values. Values must be
constant or closed expressions that do not reference document
fields. Parameters can then be accessed as variables in an
aggregate expression context (e.g. "$$var").
:param write_concern: (optional) The write concern to use for this bulk write.
:return: An instance of :class:`~pymongo.results.ClientBulkWriteResult`.
.. seealso:: :ref:`writes-and-ids`
.. note:: requires MongoDB server version 8.0+.
.. versionadded:: 4.9
"""
if self._options.auto_encryption_opts:
raise InvalidOperation(
"MongoClient.bulk_write does not currently support automatic encryption"
)
if session and session.in_transaction:
# Inherit the transaction write concern.
if write_concern:
raise InvalidOperation("Cannot set write concern after starting a transaction")
write_concern = session._transaction.opts.write_concern # type: ignore[union-attr]
else:
# Inherit the client's write concern if none is provided.
if not write_concern:
write_concern = self.write_concern
common.validate_list("models", models)
blk = _AsyncClientBulk(
self,
write_concern=write_concern, # type: ignore[arg-type]
ordered=ordered,
bypass_document_validation=bypass_document_validation,
comment=comment,
let=let,
verbose_results=verbose_results,
)
for model in models:
try:
model._add_to_client_bulk(blk)
except AttributeError:
raise TypeError(f"{model!r} is not a valid request") from None
return await blk.execute(session, _Op.BULK_WRITE)
def _retryable_error_doc(exc: PyMongoError) -> Optional[Mapping[str, Any]]:
"""Return the server response from PyMongo exception or None."""
if isinstance(exc, BulkWriteError):
if isinstance(exc, (BulkWriteError, ClientBulkWriteException)):
# Check the last writeConcernError to determine if this
# BulkWriteError is retryable.
wces = exc.details["writeConcernErrors"]
@ -2242,10 +2386,14 @@ def _add_retryable_write_error(exc: PyMongoError, max_wire_version: int, is_mong
# AsyncConnection errors are always retryable except NotPrimaryError and WaitQueueTimeoutError which is
# handled above.
if isinstance(exc, ConnectionFailure) and not isinstance(
exc, (NotPrimaryError, WaitQueueTimeoutError)
if isinstance(exc, ClientBulkWriteException):
exc_to_check = exc.error
else:
exc_to_check = exc
if isinstance(exc_to_check, ConnectionFailure) and not isinstance(
exc_to_check, (NotPrimaryError, WaitQueueTimeoutError)
):
exc._add_error_label("RetryableWriteError")
exc_to_check._add_error_label("RetryableWriteError")
class _MongoClientErrorHandler:
@ -2292,6 +2440,8 @@ class _MongoClientErrorHandler:
return
self.handled = True
if self.session:
if isinstance(exc_val, ClientBulkWriteException):
exc_val = exc_val.error
if isinstance(exc_val, ConnectionFailure):
if self.session.in_transaction:
exc_val._add_error_label("TransientTransactionError")
@ -2303,7 +2453,7 @@ class _MongoClientErrorHandler:
):
await self.session._unpin()
err_ctx = _ErrorContext(
exc_val,
exc_val, # type: ignore[arg-type]
self.max_wire_version,
self.sock_generation,
self.completed_handshake,
@ -2330,7 +2480,7 @@ class _ClientConnectionRetryable(Generic[T]):
self,
mongo_client: AsyncMongoClient,
func: _WriteCall[T] | _ReadCall[T],
bulk: Optional[_AsyncBulk],
bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]],
operation: str,
is_read: bool = False,
session: Optional[AsyncClientSession] = None,
@ -2407,7 +2557,10 @@ class _ClientConnectionRetryable(Generic[T]):
if not self._is_read:
if not self._retryable:
raise
retryable_write_error_exc = exc.has_error_label("RetryableWriteError")
if isinstance(exc, ClientBulkWriteException) and exc.error:
retryable_write_error_exc = exc.error.has_error_label("RetryableWriteError")
else:
retryable_write_error_exc = exc.has_error_label("RetryableWriteError")
if retryable_write_error_exc:
assert self._session
await self._session._unpin()

57
pymongo/errors.py
View File

@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Any, Iterable, Mapping, Optional, Sequence, Un
from bson.errors import InvalidDocument
if TYPE_CHECKING:
from pymongo.results import ClientBulkWriteResult
from pymongo.typings import _DocumentOut
@ -308,6 +309,62 @@ class BulkWriteError(OperationFailure):
return False
class ClientBulkWriteException(OperationFailure):
"""Exception class for client-level bulk write errors."""
details: _DocumentOut
verbose: bool
def __init__(self, results: _DocumentOut, verbose: bool) -> None:
super().__init__("batch op errors occurred", 65, results)
self.verbose = verbose
def __reduce__(self) -> tuple[Any, Any]:
return self.__class__, (self.details,)
@property
def error(self) -> Optional[Any]:
"""A top-level error that occurred when attempting to
communicate with the server or execute the bulk write.
This value may not be populated if the exception was
thrown due to errors occurring on individual writes.
"""
return self.details.get("error", None)
@property
def write_concern_errors(self) -> Optional[list[WriteConcernError]]:
"""Write concern errors that occurred during the bulk write.
This list may have multiple items if more than one
server command was required to execute the bulk write.
"""
return self.details.get("writeConcernErrors", [])
@property
def write_errors(self) -> Optional[Mapping[int, WriteError]]:
"""Errors that occurred during the execution of individual write operations.
This map will contain at most one entry if the bulk write was ordered.
"""
return self.details.get("writeErrors", {})
@property
def partial_result(self) -> Optional[ClientBulkWriteResult]:
"""The results of any successful operations that were
performed before the error was encountered.
"""
from pymongo.results import ClientBulkWriteResult
if self.details.get("anySuccessful"):
return ClientBulkWriteResult(
self.details, # type: ignore[arg-type]
acknowledged=True,
has_verbose_results=self.verbose,
)
return None
class InvalidOperation(PyMongoError):
"""Raised when a client attempts to perform an invalid operation."""

405
pymongo/message.py
View File

@ -21,6 +21,7 @@ MongoDB.
"""
from __future__ import annotations
import copy
import datetime
import random
import struct
@ -101,7 +102,12 @@ _OP_MAP = {
_UPDATE: b"\x04updates\x00\x00\x00\x00\x00",
_DELETE: b"\x04deletes\x00\x00\x00\x00\x00",
}
_FIELD_MAP = {"insert": "documents", "update": "updates", "delete": "deletes"}
_FIELD_MAP = {
"insert": "documents",
"update": "updates",
"delete": "deletes",
"bulkWrite": "bulkWrite",
}
_UNICODE_REPLACE_CODEC_OPTIONS: CodecOptions[Mapping[str, Any]] = CodecOptions(
unicode_decode_error_handler="replace"
@ -136,6 +142,17 @@ def _convert_exception(exception: Exception) -> dict[str, Any]:
return {"errmsg": str(exception), "errtype": exception.__class__.__name__}
def _convert_client_bulk_exception(exception: Exception) -> dict[str, Any]:
"""Convert an Exception into a failure document for publishing,
for use in client-level bulk write API.
"""
return {
"errmsg": str(exception),
"code": exception.code, # type: ignore[attr-defined]
"errtype": exception.__class__.__name__,
}
def _convert_write_result(
operation: str, command: Mapping[str, Any], result: Mapping[str, Any]
) -> dict[str, Any]:
@ -551,8 +568,8 @@ _OP_MSG_MAP = {
}
class _BulkWriteContext:
"""A wrapper around AsyncConnection for use with write splitting functions."""
class _BulkWriteContextBase:
"""Private base class for wrapping around AsyncConnection to use with write splitting functions."""
__slots__ = (
"db_name",
@ -576,7 +593,7 @@ class _BulkWriteContext:
conn: _AgnosticConnection,
operation_id: int,
listeners: _EventListeners,
session: _AgnosticClientSession,
session: Optional[_AgnosticClientSession],
op_type: int,
codec: CodecOptions,
):
@ -593,17 +610,6 @@ class _BulkWriteContext:
self.op_type = op_type
self.codec = codec
def batch_command(
self, cmd: MutableMapping[str, Any], docs: list[Mapping[str, Any]]
) -> tuple[int, Union[bytes, dict[str, Any]], list[Mapping[str, Any]]]:
namespace = self.db_name + ".$cmd"
request_id, msg, to_send = _do_batched_op_msg(
namespace, self.op_type, cmd, docs, self.codec, self
)
if not to_send:
raise InvalidOperation("cannot do an empty bulk write")
return request_id, msg, to_send
@property
def max_bson_size(self) -> int:
"""A proxy for SockInfo.max_bson_size."""
@ -627,22 +633,6 @@ class _BulkWriteContext:
"""The maximum size of a BSON command before batch splitting."""
return self.max_bson_size
def _start(
self, cmd: MutableMapping[str, Any], request_id: int, docs: list[Mapping[str, Any]]
) -> MutableMapping[str, Any]:
"""Publish a CommandStartedEvent."""
cmd[self.field] = docs
self.listeners.publish_command_start(
cmd,
self.db_name,
request_id,
self.conn.address,
self.conn.server_connection_id,
self.op_id,
self.conn.service_id,
)
return cmd
def _succeed(self, request_id: int, reply: _DocumentOut, duration: datetime.timedelta) -> None:
"""Publish a CommandSucceededEvent."""
self.listeners.publish_command_success(
@ -672,6 +662,61 @@ class _BulkWriteContext:
)
class _BulkWriteContext(_BulkWriteContextBase):
"""A wrapper around AsyncConnection/Connection for use with the collection-level bulk write API."""
__slots__ = ()
def __init__(
self,
database_name: str,
cmd_name: str,
conn: _AgnosticConnection,
operation_id: int,
listeners: _EventListeners,
session: Optional[_AgnosticClientSession],
op_type: int,
codec: CodecOptions,
):
super().__init__(
database_name,
cmd_name,
conn,
operation_id,
listeners,
session,
op_type,
codec,
)
def batch_command(
self, cmd: MutableMapping[str, Any], docs: list[Mapping[str, Any]]
) -> tuple[int, Union[bytes, dict[str, Any]], list[Mapping[str, Any]]]:
namespace = self.db_name + ".$cmd"
request_id, msg, to_send = _do_batched_op_msg(
namespace, self.op_type, cmd, docs, self.codec, self
)
if not to_send:
raise InvalidOperation("cannot do an empty bulk write")
return request_id, msg, to_send
def _start(
self, cmd: MutableMapping[str, Any], request_id: int, docs: list[Mapping[str, Any]]
) -> MutableMapping[str, Any]:
"""Publish a CommandStartedEvent."""
cmd[self.field] = docs
self.listeners.publish_command_start(
cmd,
self.db_name,
request_id,
self.conn.address,
self.conn.server_connection_id,
self.op_id,
self.conn.service_id,
)
return cmd
class _EncryptedBulkWriteContext(_BulkWriteContext):
__slots__ = ()
@ -878,6 +923,304 @@ def _do_batched_op_msg(
return _batched_op_msg(operation, command, docs, ack, opts, ctx)
class _ClientBulkWriteContext(_BulkWriteContextBase):
"""A wrapper around AsyncConnection/Connection for use with the client-level bulk write API."""
__slots__ = ()
def __init__(
self,
database_name: str,
cmd_name: str,
conn: _AgnosticConnection,
operation_id: int,
listeners: _EventListeners,
session: Optional[_AgnosticClientSession],
codec: CodecOptions,
):
super().__init__(
database_name,
cmd_name,
conn,
operation_id,
listeners,
session,
0,
codec,
)
def batch_command(
self, cmd: MutableMapping[str, Any], operations: list[tuple[str, Mapping[str, Any]]]
) -> tuple[int, Union[bytes, dict[str, Any]], list[Mapping[str, Any]], list[Mapping[str, Any]]]:
request_id, msg, to_send_ops, to_send_ns = _client_do_batched_op_msg(
cmd, operations, self.codec, self
)
if not to_send_ops:
raise InvalidOperation("cannot do an empty bulk write")
return request_id, msg, to_send_ops, to_send_ns
def _start(
self,
cmd: MutableMapping[str, Any],
request_id: int,
op_docs: list[Mapping[str, Any]],
ns_docs: list[Mapping[str, Any]],
) -> MutableMapping[str, Any]:
"""Publish a CommandStartedEvent."""
cmd["ops"] = op_docs
cmd["nsInfo"] = ns_docs
self.listeners.publish_command_start(
cmd,
self.db_name,
request_id,
self.conn.address,
self.conn.server_connection_id,
self.op_id,
self.conn.service_id,
)
return cmd
_OP_MSG_OVERHEAD = 1000
def _client_construct_op_msg(
command: Mapping[str, Any],
to_send_ops: list[Mapping[str, Any]],
to_send_ns: list[Mapping[str, Any]],
ack: bool,
opts: CodecOptions,
buf: _BytesIO,
) -> int:
# Write flags
flags = b"\x00\x00\x00\x00" if ack else b"\x02\x00\x00\x00"
buf.write(flags)
# Type 0 Section
buf.write(b"\x00")
buf.write(_dict_to_bson(command, False, opts))
# Type 1 Section for ops
buf.write(b"\x01")
size_location = buf.tell()
# Save space for size
buf.write(b"\x00\x00\x00\x00")
buf.write(b"ops\x00")
# Write all the ops documents
for op in to_send_ops:
buf.write(_dict_to_bson(op, False, opts))
resume_location = buf.tell()
# Write type 1 section size
length = buf.tell()
buf.seek(size_location)
buf.write(_pack_int(length - size_location))
buf.seek(resume_location)
# Type 1 Section for nsInfo
buf.write(b"\x01")
size_location = buf.tell()
# Save space for size
buf.write(b"\x00\x00\x00\x00")
buf.write(b"nsInfo\x00")
# Write all the nsInfo documents
for ns in to_send_ns:
buf.write(_dict_to_bson(ns, False, opts))
# Write type 1 section size
length = buf.tell()
buf.seek(size_location)
buf.write(_pack_int(length - size_location))
return length
def _client_batched_op_msg_impl(
command: Mapping[str, Any],
operations: list[tuple[str, Mapping[str, Any]]],
ack: bool,
opts: CodecOptions,
ctx: _ClientBulkWriteContext,
buf: _BytesIO,
) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]], int]:
"""Create a batched OP_MSG write for client-level bulk write."""
def _check_doc_size_limits(
op_type: str,
document: Mapping[str, Any],
limit: int,
) -> int:
doc_size = len(_dict_to_bson(document, False, opts))
if doc_size > limit:
_raise_document_too_large(op_type, doc_size, limit)
return doc_size
max_bson_size = ctx.max_bson_size
max_write_batch_size = ctx.max_write_batch_size
max_message_size = ctx.max_message_size
# Don't include bulkWrite-command-agnostic fields in document size calculations.
abridged_keys = ["bulkWrite", "errorsOnly", "ordered"]
if command.get("bypassDocumentValidation"):
abridged_keys.append("bypassDocumentValidation")
if command.get("comment"):
abridged_keys.append("comment")
if command.get("let"):
abridged_keys.append("let")
command_abridged = {key: command[key] for key in abridged_keys}
command_len_abridged = len(_dict_to_bson(command_abridged, False, opts))
# When OP_MSG is used unacknowledged we have to check command
# document size client-side or applications won't be notified.
if not ack:
_check_doc_size_limits("bulkWrite", command_abridged, max_bson_size + _COMMAND_OVERHEAD)
# Maximum combined size of the ops and nsInfo document sequences.
max_doc_sequences_bytes = max_message_size - (_OP_MSG_OVERHEAD + command_len_abridged)
ns_info = {}
to_send_ops: list[Mapping[str, Any]] = []
to_send_ns: list[Mapping[str, int]] = []
total_ops_length = 0
total_ns_length = 0
idx = 0
for real_op_type, op_doc in operations:
op_type = real_op_type
# Check insert/replace document size if unacknowledged.
if real_op_type == "insert":
if not ack:
_check_doc_size_limits(real_op_type, op_doc["document"], max_bson_size)
if real_op_type == "replace":
op_type = "update"
if not ack:
_check_doc_size_limits(real_op_type, op_doc["updateMods"], max_bson_size)
ns_doc_to_send = None
ns_length = 0
namespace = op_doc[op_type]
if namespace not in ns_info:
ns_doc_to_send = {"ns": namespace}
new_ns_index = len(to_send_ns)
ns_info[namespace] = new_ns_index
# First entry in the operation doc has the operation type as its
# key and the index of its namespace within ns_info as its value.
op_doc_to_send = copy.deepcopy(op_doc)
op_doc_to_send[op_type] = ns_info[namespace] # type: ignore[index]
# Encode current operation doc and, if newly added, namespace doc.
op_length = len(_dict_to_bson(op_doc_to_send, False, opts))
if ns_doc_to_send:
ns_length = len(_dict_to_bson(ns_doc_to_send, False, opts))
# Check operation document size if unacknowledged.
if not ack:
_check_doc_size_limits(op_type, op_doc_to_send, max_bson_size + _COMMAND_OVERHEAD)
new_message_size = total_ops_length + total_ns_length + op_length + ns_length
# We have enough data, return this batch.
if new_message_size > max_doc_sequences_bytes:
break
to_send_ops.append(op_doc_to_send)
total_ops_length += op_length
if ns_doc_to_send:
to_send_ns.append(ns_doc_to_send)
total_ns_length += ns_length
idx += 1
# We have enough documents, return this batch.
if idx == max_write_batch_size:
break
# Construct the entire OP_MSG.
length = _client_construct_op_msg(command, to_send_ops, to_send_ns, ack, opts, buf)
return to_send_ops, to_send_ns, length
def _client_encode_batched_op_msg(
command: Mapping[str, Any],
operations: list[tuple[str, Mapping[str, Any]]],
ack: bool,
opts: CodecOptions,
ctx: _ClientBulkWriteContext,
) -> tuple[bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Encode the next batched client-level bulkWrite
operation as OP_MSG.
"""
buf = _BytesIO()
to_send_ops, to_send_ns, _ = _client_batched_op_msg_impl(
command, operations, ack, opts, ctx, buf
)
return buf.getvalue(), to_send_ops, to_send_ns
def _client_batched_op_msg_compressed(
command: Mapping[str, Any],
operations: list[tuple[str, Mapping[str, Any]]],
ack: bool,
opts: CodecOptions,
ctx: _ClientBulkWriteContext,
) -> tuple[int, bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Create the next batched client-level bulkWrite operation
with OP_MSG, compressed.
"""
data, to_send_ops, to_send_ns = _client_encode_batched_op_msg(
command, operations, ack, opts, ctx
)
assert ctx.conn.compression_context is not None
request_id, msg = _compress(2013, data, ctx.conn.compression_context)
return request_id, msg, to_send_ops, to_send_ns
def _client_batched_op_msg(
command: Mapping[str, Any],
operations: list[tuple[str, Mapping[str, Any]]],
ack: bool,
opts: CodecOptions,
ctx: _ClientBulkWriteContext,
) -> tuple[int, bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""OP_MSG implementation entry point for client-level bulkWrite."""
buf = _BytesIO()
# Save space for message length and request id
buf.write(_ZERO_64)
# responseTo, opCode
buf.write(b"\x00\x00\x00\x00\xdd\x07\x00\x00")
to_send_ops, to_send_ns, length = _client_batched_op_msg_impl(
command, operations, ack, opts, ctx, buf
)
# Header - request id and message length
buf.seek(4)
request_id = _randint()
buf.write(_pack_int(request_id))
buf.seek(0)
buf.write(_pack_int(length))
return request_id, buf.getvalue(), to_send_ops, to_send_ns
def _client_do_batched_op_msg(
command: MutableMapping[str, Any],
operations: list[tuple[str, Mapping[str, Any]]],
opts: CodecOptions,
ctx: _ClientBulkWriteContext,
) -> tuple[int, bytes, list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Create the next batched client-level bulkWrite
operation using OP_MSG.
"""
command["$db"] = "admin"
if "writeConcern" in command:
ack = bool(command["writeConcern"].get("w", 1))
else:
ack = True
if ctx.conn.compression_context:
return _client_batched_op_msg_compressed(command, operations, ack, opts, ctx)
return _client_batched_op_msg(command, operations, ack, opts, ctx)
# End OP_MSG -----------------------------------------------------

317
pymongo/operations.py
View File

@ -34,12 +34,13 @@ from bson.raw_bson import RawBSONDocument
from pymongo import helpers_shared
from pymongo.collation import validate_collation_or_none
from pymongo.common import validate_is_mapping, validate_list
from pymongo.errors import InvalidOperation
from pymongo.helpers_shared import _gen_index_name, _index_document, _index_list
from pymongo.typings import _CollationIn, _DocumentType, _Pipeline
from pymongo.write_concern import validate_boolean
if TYPE_CHECKING:
from pymongo.typings import _AgnosticBulk
from pymongo.typings import _AgnosticBulk, _AgnosticClientBulk
# Hint supports index name, "myIndex", a list of either strings or index pairs: [('x', 1), ('y', -1), 'z''], or a dictionary
@ -52,6 +53,7 @@ _IndexKeyHint = Union[str, _IndexList]
class _Op(str, enum.Enum):
ABORT = "abortTransaction"
AGGREGATE = "aggregate"
BULK_WRITE = "bulkWrite"
COMMIT = "commitTransaction"
COUNT = "count"
CREATE = "create"
@ -83,48 +85,130 @@ class _Op(str, enum.Enum):
class InsertOne(Generic[_DocumentType]):
"""Represents an insert_one operation."""
__slots__ = ("_doc",)
__slots__ = (
"_doc",
"_namespace",
)
def __init__(self, document: _DocumentType) -> None:
def __init__(self, document: _DocumentType, namespace: Optional[str] = None) -> None:
"""Create an InsertOne instance.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`,
:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
:param document: The document to insert. If the document is missing an
_id field one will be added.
:param namespace: (optional) The namespace in which to insert a document.
.. versionchanged:: 4.9
Added the `namespace` option to support `MongoClient.bulk_write`.
"""
self._doc = document
self._namespace = namespace
def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None:
"""Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`."""
bulkobj.add_insert(self._doc) # type: ignore[arg-type]
def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None:
"""Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`."""
if not self._namespace:
raise InvalidOperation(
"MongoClient.bulk_write requires a namespace to be provided for each write operation"
)
bulkobj.add_insert(
self._namespace,
self._doc, # type: ignore[arg-type]
)
def __repr__(self) -> str:
return f"InsertOne({self._doc!r})"
if self._namespace:
return f"{self.__class__.__name__}({self._doc!r}, {self._namespace!r})"
return f"{self.__class__.__name__}({self._doc!r})"
def __eq__(self, other: Any) -> bool:
if type(other) == type(self):
return other._doc == self._doc
return other._doc == self._doc and other._namespace == self._namespace
return NotImplemented
def __ne__(self, other: Any) -> bool:
return not self == other
class DeleteOne:
"""Represents a delete_one operation."""
class _DeleteOp:
"""Private base class for delete operations."""
__slots__ = ("_filter", "_collation", "_hint")
__slots__ = (
"_filter",
"_collation",
"_hint",
"_namespace",
)
def __init__(
self,
filter: Mapping[str, Any],
collation: Optional[_CollationIn] = None,
hint: Optional[_IndexKeyHint] = None,
namespace: Optional[str] = None,
) -> None:
if filter is not None:
validate_is_mapping("filter", filter)
if hint is not None and not isinstance(hint, str):
self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint)
else:
self._hint = hint
self._filter = filter
self._collation = collation
self._namespace = namespace
def __eq__(self, other: Any) -> bool:
if type(other) == type(self):
return (
other._filter,
other._collation,
other._hint,
other._namespace,
) == (
self._filter,
self._collation,
self._hint,
self._namespace,
)
return NotImplemented
def __ne__(self, other: Any) -> bool:
return not self == other
def __repr__(self) -> str:
if self._namespace:
return "{}({!r}, {!r}, {!r}, {!r})".format(
self.__class__.__name__,
self._filter,
self._collation,
self._hint,
self._namespace,
)
return f"{self.__class__.__name__}({self._filter!r}, {self._collation!r}, {self._hint!r})"
class DeleteOne(_DeleteOp):
"""Represents a delete_one operation."""
__slots__ = ()
def __init__(
self,
filter: Mapping[str, Any],
collation: Optional[_CollationIn] = None,
hint: Optional[_IndexKeyHint] = None,
namespace: Optional[str] = None,
) -> None:
"""Create a DeleteOne instance.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`,
:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
:param filter: A query that matches the document to delete.
:param collation: An instance of
@ -135,20 +219,16 @@ class DeleteOne:
:meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g.
``[('field', ASCENDING)]``). This option is only supported on
MongoDB 4.4 and above.
:param namespace: (optional) The namespace in which to delete a document.
.. versionchanged:: 4.9
Added the `namespace` option to support `MongoClient.bulk_write`.
.. versionchanged:: 3.11
Added the ``hint`` option.
.. versionchanged:: 3.5
Added the `collation` option.
"""
if filter is not None:
validate_is_mapping("filter", filter)
if hint is not None and not isinstance(hint, str):
self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint)
else:
self._hint = hint
self._filter = filter
self._collation = collation
super().__init__(filter, collation, hint, namespace)
def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None:
"""Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`."""
@ -159,36 +239,37 @@ class DeleteOne:
hint=self._hint,
)
def __repr__(self) -> str:
return f"DeleteOne({self._filter!r}, {self._collation!r}, {self._hint!r})"
def __eq__(self, other: Any) -> bool:
if type(other) == type(self):
return (other._filter, other._collation, other._hint) == (
self._filter,
self._collation,
self._hint,
def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None:
"""Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`."""
if not self._namespace:
raise InvalidOperation(
"MongoClient.bulk_write requires a namespace to be provided for each write operation"
)
return NotImplemented
def __ne__(self, other: Any) -> bool:
return not self == other
bulkobj.add_delete(
self._namespace,
self._filter,
multi=False,
collation=validate_collation_or_none(self._collation),
hint=self._hint,
)
class DeleteMany:
class DeleteMany(_DeleteOp):
"""Represents a delete_many operation."""
__slots__ = ("_filter", "_collation", "_hint")
__slots__ = ()
def __init__(
self,
filter: Mapping[str, Any],
collation: Optional[_CollationIn] = None,
hint: Optional[_IndexKeyHint] = None,
namespace: Optional[str] = None,
) -> None:
"""Create a DeleteMany instance.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`,
:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
:param filter: A query that matches the documents to delete.
:param collation: An instance of
@ -199,20 +280,16 @@ class DeleteMany:
:meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g.
``[('field', ASCENDING)]``). This option is only supported on
MongoDB 4.4 and above.
:param namespace: (optional) The namespace in which to delete documents.
.. versionchanged:: 4.9
Added the `namespace` option to support `MongoClient.bulk_write`.
.. versionchanged:: 3.11
Added the ``hint`` option.
.. versionchanged:: 3.5
Added the `collation` option.
"""
if filter is not None:
validate_is_mapping("filter", filter)
if hint is not None and not isinstance(hint, str):
self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint)
else:
self._hint = hint
self._filter = filter
self._collation = collation
super().__init__(filter, collation, hint, namespace)
def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None:
"""Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`."""
@ -223,26 +300,32 @@ class DeleteMany:
hint=self._hint,
)
def __repr__(self) -> str:
return f"DeleteMany({self._filter!r}, {self._collation!r}, {self._hint!r})"
def __eq__(self, other: Any) -> bool:
if type(other) == type(self):
return (other._filter, other._collation, other._hint) == (
self._filter,
self._collation,
self._hint,
def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None:
"""Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`."""
if not self._namespace:
raise InvalidOperation(
"MongoClient.bulk_write requires a namespace to be provided for each write operation"
)
return NotImplemented
def __ne__(self, other: Any) -> bool:
return not self == other
bulkobj.add_delete(
self._namespace,
self._filter,
multi=True,
collation=validate_collation_or_none(self._collation),
hint=self._hint,
)
class ReplaceOne(Generic[_DocumentType]):
"""Represents a replace_one operation."""
__slots__ = ("_filter", "_doc", "_upsert", "_collation", "_hint")
__slots__ = (
"_filter",
"_doc",
"_upsert",
"_collation",
"_hint",
"_namespace",
)
def __init__(
self,
@ -251,10 +334,12 @@ class ReplaceOne(Generic[_DocumentType]):
upsert: bool = False,
collation: Optional[_CollationIn] = None,
hint: Optional[_IndexKeyHint] = None,
namespace: Optional[str] = None,
) -> None:
"""Create a ReplaceOne instance.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`,
:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
:param filter: A query that matches the document to replace.
:param replacement: The new document.
@ -268,7 +353,10 @@ class ReplaceOne(Generic[_DocumentType]):
:meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g.
``[('field', ASCENDING)]``). This option is only supported on
MongoDB 4.2 and above.
:param namespace: (optional) The namespace in which to replace a document.
.. versionchanged:: 4.9
Added the `namespace` option to support `MongoClient.bulk_write`.
.. versionchanged:: 3.11
Added the ``hint`` option.
.. versionchanged:: 3.5
@ -282,10 +370,12 @@ class ReplaceOne(Generic[_DocumentType]):
self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint)
else:
self._hint = hint
self._filter = filter
self._doc = replacement
self._upsert = upsert
self._collation = collation
self._namespace = namespace
def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None:
"""Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`."""
@ -297,6 +387,21 @@ class ReplaceOne(Generic[_DocumentType]):
hint=self._hint,
)
def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None:
"""Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`."""
if not self._namespace:
raise InvalidOperation(
"MongoClient.bulk_write requires a namespace to be provided for each write operation"
)
bulkobj.add_replace(
self._namespace,
self._filter,
self._doc,
self._upsert,
collation=validate_collation_or_none(self._collation),
hint=self._hint,
)
def __eq__(self, other: Any) -> bool:
if type(other) == type(self):
return (
@ -305,12 +410,14 @@ class ReplaceOne(Generic[_DocumentType]):
other._upsert,
other._collation,
other._hint,
other._namespace,
) == (
self._filter,
self._doc,
self._upsert,
self._collation,
other._hint,
self._namespace,
)
return NotImplemented
@ -318,6 +425,16 @@ class ReplaceOne(Generic[_DocumentType]):
return not self == other
def __repr__(self) -> str:
if self._namespace:
return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
self.__class__.__name__,
self._filter,
self._doc,
self._upsert,
self._collation,
self._hint,
self._namespace,
)
return "{}({!r}, {!r}, {!r}, {!r}, {!r})".format(
self.__class__.__name__,
self._filter,
@ -331,16 +448,25 @@ class ReplaceOne(Generic[_DocumentType]):
class _UpdateOp:
"""Private base class for update operations."""
__slots__ = ("_filter", "_doc", "_upsert", "_collation", "_array_filters", "_hint")
__slots__ = (
"_filter",
"_doc",
"_upsert",
"_collation",
"_array_filters",
"_hint",
"_namespace",
)
def __init__(
self,
filter: Mapping[str, Any],
doc: Union[Mapping[str, Any], _Pipeline],
upsert: bool,
upsert: Optional[bool],
collation: Optional[_CollationIn],
array_filters: Optional[list[Mapping[str, Any]]],
hint: Optional[_IndexKeyHint],
namespace: Optional[str],
):
if filter is not None:
validate_is_mapping("filter", filter)
@ -358,6 +484,7 @@ class _UpdateOp:
self._upsert = upsert
self._collation = collation
self._array_filters = array_filters
self._namespace = namespace
def __eq__(self, other: object) -> bool:
if isinstance(other, type(self)):
@ -368,6 +495,7 @@ class _UpdateOp:
other._collation,
other._array_filters,
other._hint,
other._namespace,
) == (
self._filter,
self._doc,
@ -375,10 +503,25 @@ class _UpdateOp:
self._collation,
self._array_filters,
self._hint,
self._namespace,
)
return NotImplemented
def __ne__(self, other: Any) -> bool:
return not self == other
def __repr__(self) -> str:
if self._namespace:
return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
self.__class__.__name__,
self._filter,
self._doc,
self._upsert,
self._collation,
self._array_filters,
self._hint,
self._namespace,
)
return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
self.__class__.__name__,
self._filter,
@ -399,14 +542,16 @@ class UpdateOne(_UpdateOp):
self,
filter: Mapping[str, Any],
update: Union[Mapping[str, Any], _Pipeline],
upsert: bool = False,
upsert: Optional[bool] = None,
collation: Optional[_CollationIn] = None,
array_filters: Optional[list[Mapping[str, Any]]] = None,
hint: Optional[_IndexKeyHint] = None,
namespace: Optional[str] = None,
) -> None:
"""Represents an update_one operation.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`,
:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
:param filter: A query that matches the document to update.
:param update: The modifications to apply.
@ -422,7 +567,10 @@ class UpdateOne(_UpdateOp):
:meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g.
``[('field', ASCENDING)]``). This option is only supported on
MongoDB 4.2 and above.
:param namespace: (optional) The namespace in which to update a document.
.. versionchanged:: 4.9
Added the `namespace` option to support `MongoClient.bulk_write`.
.. versionchanged:: 3.11
Added the `hint` option.
.. versionchanged:: 3.9
@ -432,11 +580,28 @@ class UpdateOne(_UpdateOp):
.. versionchanged:: 3.5
Added the `collation` option.
"""
super().__init__(filter, update, upsert, collation, array_filters, hint)
super().__init__(filter, update, upsert, collation, array_filters, hint, namespace)
def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None:
"""Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`."""
bulkobj.add_update(
self._filter,
self._doc,
False,
bool(self._upsert),
collation=validate_collation_or_none(self._collation),
array_filters=self._array_filters,
hint=self._hint,
)
def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None:
"""Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`."""
if not self._namespace:
raise InvalidOperation(
"MongoClient.bulk_write requires a namespace to be provided for each write operation"
)
bulkobj.add_update(
self._namespace,
self._filter,
self._doc,
False,
@ -456,14 +621,16 @@ class UpdateMany(_UpdateOp):
self,
filter: Mapping[str, Any],
update: Union[Mapping[str, Any], _Pipeline],
upsert: bool = False,
upsert: Optional[bool] = None,
collation: Optional[_CollationIn] = None,
array_filters: Optional[list[Mapping[str, Any]]] = None,
hint: Optional[_IndexKeyHint] = None,
namespace: Optional[str] = None,
) -> None:
"""Create an UpdateMany instance.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write` and :meth:`~pymongo.collection.Collection.bulk_write`.
For use with :meth:`~pymongo.asynchronous.collection.AsyncCollection.bulk_write`, :meth:`~pymongo.collection.Collection.bulk_write`,
:meth:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.bulk_write` and :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
:param filter: A query that matches the documents to update.
:param update: The modifications to apply.
@ -479,7 +646,10 @@ class UpdateMany(_UpdateOp):
:meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g.
``[('field', ASCENDING)]``). This option is only supported on
MongoDB 4.2 and above.
:param namespace: (optional) The namespace in which to update documents.
.. versionchanged:: 4.9
Added the `namespace` option to support `MongoClient.bulk_write`.
.. versionchanged:: 3.11
Added the `hint` option.
.. versionchanged:: 3.9
@ -489,11 +659,28 @@ class UpdateMany(_UpdateOp):
.. versionchanged:: 3.5
Added the `collation` option.
"""
super().__init__(filter, update, upsert, collation, array_filters, hint)
super().__init__(filter, update, upsert, collation, array_filters, hint, namespace)
def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None:
"""Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`."""
bulkobj.add_update(
self._filter,
self._doc,
True,
bool(self._upsert),
collation=validate_collation_or_none(self._collation),
array_filters=self._array_filters,
hint=self._hint,
)
def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None:
"""Add this operation to the _AsyncClientBulk/_ClientBulk instance `bulkobj`."""
if not self._namespace:
raise InvalidOperation(
"MongoClient.bulk_write requires a namespace to be provided for each write operation"
)
bulkobj.add_update(
self._namespace,
self._filter,
self._doc,
True,

159
pymongo/results.py
View File

@ -18,7 +18,7 @@
"""
from __future__ import annotations
from typing import Any, Mapping, Optional, cast
from typing import Any, Mapping, MutableMapping, Optional, cast
from pymongo.errors import InvalidOperation
@ -65,7 +65,9 @@ class _WriteResult:
class InsertOneResult(_WriteResult):
"""The return type for :meth:`~pymongo.collection.Collection.insert_one`."""
"""The return type for :meth:`~pymongo.collection.Collection.insert_one`
and as part of :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
"""
__slots__ = ("__inserted_id",)
@ -113,13 +115,23 @@ class InsertManyResult(_WriteResult):
class UpdateResult(_WriteResult):
"""The return type for :meth:`~pymongo.collection.Collection.update_one`,
:meth:`~pymongo.collection.Collection.update_many`, and
:meth:`~pymongo.collection.Collection.replace_one`.
:meth:`~pymongo.collection.Collection.replace_one`, and as part of
:meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
"""
__slots__ = ("__raw_result",)
__slots__ = (
"__raw_result",
"__in_client_bulk",
)
def __init__(self, raw_result: Optional[Mapping[str, Any]], acknowledged: bool):
def __init__(
self,
raw_result: Optional[Mapping[str, Any]],
acknowledged: bool,
in_client_bulk: bool = False,
):
self.__raw_result = raw_result
self.__in_client_bulk = in_client_bulk
super().__init__(acknowledged)
def __repr__(self) -> str:
@ -134,9 +146,9 @@ class UpdateResult(_WriteResult):
def matched_count(self) -> int:
"""The number of documents matched for this update."""
self._raise_if_unacknowledged("matched_count")
if self.upserted_id is not None:
return 0
assert self.__raw_result is not None
if not self.__in_client_bulk and self.upserted_id is not None:
return 0
return self.__raw_result.get("n", 0)
@property
@ -153,12 +165,21 @@ class UpdateResult(_WriteResult):
"""
self._raise_if_unacknowledged("upserted_id")
assert self.__raw_result is not None
return self.__raw_result.get("upserted")
if self.__in_client_bulk and self.__raw_result.get("upserted"):
return self.__raw_result["upserted"]["_id"]
return self.__raw_result.get("upserted", None)
@property
def did_upsert(self) -> bool:
"""Whether or not an upsert took place."""
assert self.__raw_result is not None
return len(self.__raw_result.get("upserted", {})) > 0
class DeleteResult(_WriteResult):
"""The return type for :meth:`~pymongo.collection.Collection.delete_one`
and :meth:`~pymongo.collection.Collection.delete_many`
and as part of :meth:`~pymongo.mongo_client.MongoClient.bulk_write`.
"""
__slots__ = ("__raw_result",)
@ -182,19 +203,12 @@ class DeleteResult(_WriteResult):
return self.__raw_result.get("n", 0)
class BulkWriteResult(_WriteResult):
"""An object wrapper for bulk API write results."""
class _BulkWriteResultBase(_WriteResult):
"""Private base class for bulk write API results."""
__slots__ = ("__bulk_api_result",)
def __init__(self, bulk_api_result: dict[str, Any], acknowledged: bool) -> None:
"""Create a BulkWriteResult instance.
:param bulk_api_result: A result dict from the bulk API
:param acknowledged: Was this write result acknowledged? If ``False``
then all properties of this object will raise
:exc:`~pymongo.errors.InvalidOperation`.
"""
self.__bulk_api_result = bulk_api_result
super().__init__(acknowledged)
@ -203,7 +217,7 @@ class BulkWriteResult(_WriteResult):
@property
def bulk_api_result(self) -> dict[str, Any]:
"""The raw bulk API result."""
"""The raw bulk write API result."""
return self.__bulk_api_result
@property
@ -228,7 +242,10 @@ class BulkWriteResult(_WriteResult):
def deleted_count(self) -> int:
"""The number of documents deleted."""
self._raise_if_unacknowledged("deleted_count")
return cast(int, self.__bulk_api_result.get("nRemoved"))
if "nRemoved" in self.__bulk_api_result:
return cast(int, self.__bulk_api_result.get("nRemoved"))
else:
return cast(int, self.__bulk_api_result.get("nDeleted"))
@property
def upserted_count(self) -> int:
@ -236,10 +253,112 @@ class BulkWriteResult(_WriteResult):
self._raise_if_unacknowledged("upserted_count")
return cast(int, self.__bulk_api_result.get("nUpserted"))
class BulkWriteResult(_BulkWriteResultBase):
"""An object wrapper for collection-level bulk write API results."""
__slots__ = ()
def __init__(self, bulk_api_result: dict[str, Any], acknowledged: bool) -> None:
"""Create a BulkWriteResult instance.
:param bulk_api_result: A result dict from the collection-level bulk write API
:param acknowledged: Was this write result acknowledged? If ``False``
then all properties of this object will raise
:exc:`~pymongo.errors.InvalidOperation`.
"""
super().__init__(bulk_api_result, acknowledged)
def __repr__(self) -> str:
return (
f"{self.__class__.__name__}({self.bulk_api_result!r}, acknowledged={self.acknowledged})"
)
@property
def upserted_ids(self) -> Optional[dict[int, Any]]:
"""A map of operation index to the _id of the upserted document."""
self._raise_if_unacknowledged("upserted_ids")
if self.__bulk_api_result:
if self.bulk_api_result:
return {upsert["index"]: upsert["_id"] for upsert in self.bulk_api_result["upserted"]}
return None
class ClientBulkWriteResult(_BulkWriteResultBase):
"""An object wrapper for client-level bulk write API results."""
__slots__ = ("__has_verbose_results",)
def __init__(
self,
bulk_api_result: MutableMapping[str, Any],
acknowledged: bool,
has_verbose_results: bool,
) -> None:
"""Create a ClientBulkWriteResult instance.
:param bulk_api_result: A result dict from the client-level bulk write API
:param acknowledged: Was this write result acknowledged? If ``False``
then all properties of this object will raise
:exc:`~pymongo.errors.InvalidOperation`.
:param has_verbose_results: Should the returned result be verbose?
If ``False``, then the ``insert_results``, ``update_results``, and
``delete_results`` properties of this object will raise
:exc:`~pymongo.errors.InvalidOperation`.
"""
self.__has_verbose_results = has_verbose_results
super().__init__(
bulk_api_result, # type: ignore[arg-type]
acknowledged,
)
def __repr__(self) -> str:
return "{}({!r}, acknowledged={}, verbose={})".format(
self.__class__.__name__,
self.bulk_api_result,
self.acknowledged,
self.has_verbose_results,
)
def _raise_if_not_verbose(self, property_name: str) -> None:
"""Raise an exception on property access if verbose results are off."""
if not self.__has_verbose_results:
raise InvalidOperation(
f"A value for {property_name} is not available when "
"the results are not set to be verbose. Check the "
"verbose_results attribute to avoid this error."
)
@property
def has_verbose_results(self) -> bool:
"""Whether the returned results should be verbose."""
return self.__has_verbose_results
@property
def insert_results(self) -> Mapping[int, InsertOneResult]:
"""A map of successful insertion operations to their results."""
self._raise_if_unacknowledged("insert_results")
self._raise_if_not_verbose("insert_results")
return cast(
Mapping[int, InsertOneResult],
self.bulk_api_result.get("insertResults"),
)
@property
def update_results(self) -> Mapping[int, UpdateResult]:
"""A map of successful update operations to their results."""
self._raise_if_unacknowledged("update_results")
self._raise_if_not_verbose("update_results")
return cast(
Mapping[int, UpdateResult],
self.bulk_api_result.get("updateResults"),
)
@property
def delete_results(self) -> Mapping[int, DeleteResult]:
"""A map of successful delete operations to their results."""
self._raise_if_unacknowledged("delete_results")
self._raise_if_not_verbose("delete_results")
return cast(
Mapping[int, DeleteResult],
self.bulk_api_result.get("deleteResults"),
)

786
pymongo/synchronous/client_bulk.py Normal file
View File

@ -0,0 +1,786 @@
# Copyright 2024-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The client-level bulk write operations interface.
.. versionadded:: 4.9
"""
from __future__ import annotations
import copy
import datetime
import logging
from collections.abc import MutableMapping
from itertools import islice
from typing import (
TYPE_CHECKING,
Any,
Mapping,
Optional,
Type,
Union,
)
from bson.objectid import ObjectId
from bson.raw_bson import RawBSONDocument
from pymongo import _csot, common
from pymongo.synchronous.client_session import ClientSession, _validate_session_write_concern
from pymongo.synchronous.collection import Collection
from pymongo.synchronous.command_cursor import CommandCursor
from pymongo.synchronous.database import Database
from pymongo.synchronous.helpers import _handle_reauth
if TYPE_CHECKING:
from pymongo.synchronous.mongo_client import MongoClient
from pymongo.synchronous.pool import Connection
from pymongo._client_bulk_shared import (
_merge_command,
_throw_client_bulk_write_exception,
)
from pymongo.common import (
validate_is_document_type,
validate_ok_for_replace,
validate_ok_for_update,
)
from pymongo.errors import (
ConfigurationError,
ConnectionFailure,
InvalidOperation,
NotPrimaryError,
OperationFailure,
WaitQueueTimeoutError,
)
from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES
from pymongo.logger import _COMMAND_LOGGER, _CommandStatusMessage, _debug_log
from pymongo.message import (
_ClientBulkWriteContext,
_convert_client_bulk_exception,
_convert_exception,
_convert_write_result,
_randint,
)
from pymongo.read_preferences import ReadPreference
from pymongo.results import (
ClientBulkWriteResult,
DeleteResult,
InsertOneResult,
UpdateResult,
)
from pymongo.typings import _DocumentOut, _Pipeline
from pymongo.write_concern import WriteConcern
_IS_SYNC = True
class _ClientBulk:
"""The private guts of the client-level bulk write API."""
def __init__(
self,
client: MongoClient,
write_concern: WriteConcern,
ordered: bool = True,
bypass_document_validation: Optional[bool] = None,
comment: Optional[str] = None,
let: Optional[Any] = None,
verbose_results: bool = False,
) -> None:
"""Initialize a _ClientBulk instance."""
self.client = client
self.write_concern = write_concern
self.let = let
if self.let is not None:
common.validate_is_document_type("let", self.let)
self.ordered = ordered
self.bypass_doc_val = bypass_document_validation
self.comment = comment
self.verbose_results = verbose_results
self.ops: list[tuple[str, Mapping[str, Any]]] = []
self.idx_offset: int = 0
self.total_ops: int = 0
self.executed = False
self.uses_upsert = False
self.uses_collation = False
self.uses_array_filters = False
self.uses_hint_update = False
self.uses_hint_delete = False
self.is_retryable = self.client.options.retry_writes
self.retrying = False
self.started_retryable_write = False
@property
def bulk_ctx_class(self) -> Type[_ClientBulkWriteContext]:
return _ClientBulkWriteContext
def add_insert(self, namespace: str, document: _DocumentOut) -> None:
"""Add an insert document to the list of ops."""
validate_is_document_type("document", document)
# Generate ObjectId client side.
if not (isinstance(document, RawBSONDocument) or "_id" in document):
document["_id"] = ObjectId()
cmd = {"insert": namespace, "document": document}
self.ops.append(("insert", cmd))
self.total_ops += 1
def add_update(
self,
namespace: str,
selector: Mapping[str, Any],
update: Union[Mapping[str, Any], _Pipeline],
multi: bool = False,
upsert: Optional[bool] = None,
collation: Optional[Mapping[str, Any]] = None,
array_filters: Optional[list[Mapping[str, Any]]] = None,
hint: Union[str, dict[str, Any], None] = None,
) -> None:
"""Create an update document and add it to the list of ops."""
validate_ok_for_update(update)
cmd = {
"update": namespace,
"filter": selector,
"updateMods": update,
"multi": multi,
}
if upsert is not None:
self.uses_upsert = True
cmd["upsert"] = upsert
if array_filters is not None:
self.uses_array_filters = True
cmd["arrayFilters"] = array_filters
if hint is not None:
self.uses_hint_update = True
cmd["hint"] = hint
if collation is not None:
self.uses_collation = True
cmd["collation"] = collation
if multi:
# A bulk_write containing an update_many is not retryable.
self.is_retryable = False
self.ops.append(("update", cmd))
self.total_ops += 1
def add_replace(
self,
namespace: str,
selector: Mapping[str, Any],
replacement: Mapping[str, Any],
upsert: Optional[bool] = None,
collation: Optional[Mapping[str, Any]] = None,
hint: Union[str, dict[str, Any], None] = None,
) -> None:
"""Create a replace document and add it to the list of ops."""
validate_ok_for_replace(replacement)
cmd = {
"update": namespace,
"filter": selector,
"updateMods": replacement,
"multi": False,
}
if upsert is not None:
self.uses_upsert = True
cmd["upsert"] = upsert
if hint is not None:
self.uses_hint_update = True
cmd["hint"] = hint
if collation is not None:
self.uses_collation = True
cmd["collation"] = collation
self.ops.append(("replace", cmd))
self.total_ops += 1
def add_delete(
self,
namespace: str,
selector: Mapping[str, Any],
multi: bool,
collation: Optional[Mapping[str, Any]] = None,
hint: Union[str, dict[str, Any], None] = None,
) -> None:
"""Create a delete document and add it to the list of ops."""
cmd = {"delete": namespace, "filter": selector, "multi": multi}
if hint is not None:
self.uses_hint_delete = True
cmd["hint"] = hint
if collation is not None:
self.uses_collation = True
cmd["collation"] = collation
if multi:
# A bulk_write containing an update_many is not retryable.
self.is_retryable = False
self.ops.append(("delete", cmd))
self.total_ops += 1
@_handle_reauth
def write_command(
self,
bwc: _ClientBulkWriteContext,
cmd: MutableMapping[str, Any],
request_id: int,
msg: Union[bytes, dict[str, Any]],
op_docs: list[Mapping[str, Any]],
ns_docs: list[Mapping[str, Any]],
client: MongoClient,
) -> dict[str, Any]:
"""A proxy for Connection.write_command that handles event publishing."""
cmd["ops"] = op_docs
cmd["nsInfo"] = ns_docs
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.STARTED,
command=cmd,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
bwc._start(cmd, request_id, op_docs, ns_docs)
try:
reply = bwc.conn.write_command(request_id, msg, bwc.codec) # type: ignore[misc, arg-type]
duration = datetime.datetime.now() - bwc.start_time
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.SUCCEEDED,
durationMS=duration,
reply=reply,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
bwc._succeed(request_id, reply, duration) # type: ignore[arg-type]
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, (NotPrimaryError, OperationFailure)):
failure: _DocumentOut = exc.details # type: ignore[assignment]
else:
failure = _convert_exception(exc)
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.FAILED,
durationMS=duration,
failure=failure,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
isServerSideError=isinstance(exc, OperationFailure),
)
if bwc.publish:
bwc._fail(request_id, failure, duration)
# Top-level error will be embedded in ClientBulkWriteException.
reply = {"error": exc}
finally:
bwc.start_time = datetime.datetime.now()
return reply # type: ignore[return-value]
def unack_write(
self,
bwc: _ClientBulkWriteContext,
cmd: MutableMapping[str, Any],
request_id: int,
msg: bytes,
op_docs: list[Mapping[str, Any]],
ns_docs: list[Mapping[str, Any]],
client: MongoClient,
) -> Optional[Mapping[str, Any]]:
"""A proxy for Connection.unack_write that handles event publishing."""
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.STARTED,
command=cmd,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
cmd = bwc._start(cmd, request_id, op_docs, ns_docs)
try:
result = bwc.conn.unack_write(msg, bwc.max_bson_size) # type: ignore[func-returns-value, misc, override]
duration = datetime.datetime.now() - bwc.start_time
if result is not None:
reply = _convert_write_result(bwc.name, cmd, result) # type: ignore[arg-type]
else:
# Comply with APM spec.
reply = {"ok": 1}
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.SUCCEEDED,
durationMS=duration,
reply=reply,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
)
if bwc.publish:
bwc._succeed(request_id, reply, duration)
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, OperationFailure):
failure: _DocumentOut = _convert_write_result(bwc.name, cmd, exc.details) # type: ignore[arg-type]
elif isinstance(exc, NotPrimaryError):
failure = exc.details # type: ignore[assignment]
else:
failure = _convert_exception(exc)
if _COMMAND_LOGGER.isEnabledFor(logging.DEBUG):
_debug_log(
_COMMAND_LOGGER,
clientId=client._topology_settings._topology_id,
message=_CommandStatusMessage.FAILED,
durationMS=duration,
failure=failure,
commandName=next(iter(cmd)),
databaseName=bwc.db_name,
requestId=request_id,
operationId=request_id,
driverConnectionId=bwc.conn.id,
serverConnectionId=bwc.conn.server_connection_id,
serverHost=bwc.conn.address[0],
serverPort=bwc.conn.address[1],
serviceId=bwc.conn.service_id,
isServerSideError=isinstance(exc, OperationFailure),
)
if bwc.publish:
assert bwc.start_time is not None
bwc._fail(request_id, failure, duration)
# Top-level error will be embedded in ClientBulkWriteException.
reply = {"error": exc}
finally:
bwc.start_time = datetime.datetime.now()
return result # type: ignore[return-value]
def _execute_batch_unack(
self,
bwc: _ClientBulkWriteContext,
cmd: dict[str, Any],
ops: list[tuple[str, Mapping[str, Any]]],
) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Executes a batch of bulkWrite server commands (unack)."""
request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops)
self.unack_write(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type]
return to_send_ops, to_send_ns
def _execute_batch(
self,
bwc: _ClientBulkWriteContext,
cmd: dict[str, Any],
ops: list[tuple[str, Mapping[str, Any]]],
) -> tuple[dict[str, Any], list[Mapping[str, Any]], list[Mapping[str, Any]]]:
"""Executes a batch of bulkWrite server commands (ack)."""
request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops)
result = self.write_command(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type]
self.client._process_response(result, bwc.session) # type: ignore[arg-type]
return result, to_send_ops, to_send_ns # type: ignore[return-value]
def _process_results_cursor(
self,
full_result: MutableMapping[str, Any],
result: MutableMapping[str, Any],
conn: Connection,
session: Optional[ClientSession],
) -> None:
"""Internal helper for processing the server reply command cursor."""
if result.get("cursor"):
coll = Collection(
database=Database(self.client, "admin"),
name="$cmd.bulkWrite",
)
cmd_cursor = CommandCursor(
coll,
result["cursor"],
conn.address,
session=session,
explicit_session=session is not None,
comment=self.comment,
)
cmd_cursor._maybe_pin_connection(conn)
# Iterate the cursor to get individual write results.
try:
for doc in cmd_cursor:
original_index = doc["idx"] + self.idx_offset
op_type, op = self.ops[original_index]
if not doc["ok"]:
result["writeErrors"].append(doc)
if self.ordered:
return
# Record individual write result.
if doc["ok"] and self.verbose_results:
if op_type == "insert":
inserted_id = op["document"]["_id"]
res = InsertOneResult(inserted_id, acknowledged=True) # type: ignore[assignment]
if op_type in ["update", "replace"]:
op_type = "update"
res = UpdateResult(doc, acknowledged=True, in_client_bulk=True) # type: ignore[assignment]
if op_type == "delete":
res = DeleteResult(doc, acknowledged=True) # type: ignore[assignment]
full_result[f"{op_type}Results"][original_index] = res
except Exception as exc:
# Attempt to close the cursor, then raise top-level error.
if cmd_cursor.alive:
cmd_cursor.close()
result["error"] = _convert_client_bulk_exception(exc)
def _execute_command(
self,
write_concern: WriteConcern,
session: Optional[ClientSession],
conn: Connection,
op_id: int,
retryable: bool,
full_result: MutableMapping[str, Any],
final_write_concern: Optional[WriteConcern] = None,
) -> None:
"""Internal helper for executing batches of bulkWrite commands."""
db_name = "admin"
cmd_name = "bulkWrite"
listeners = self.client._event_listeners
# Connection.command validates the session, but we use
# Connection.write_command
conn.validate_session(self.client, session)
bwc = self.bulk_ctx_class(
db_name,
cmd_name,
conn,
op_id,
listeners, # type: ignore[arg-type]
session,
self.client.codec_options,
)
while self.idx_offset < self.total_ops:
# If this is the last possible batch, use the
# final write concern.
if self.total_ops - self.idx_offset <= bwc.max_write_batch_size:
write_concern = final_write_concern or write_concern
# Construct the server command, specifying the relevant options.
cmd = {"bulkWrite": 1}
cmd["errorsOnly"] = not self.verbose_results
cmd["ordered"] = self.ordered # type: ignore[assignment]
not_in_transaction = session and not session.in_transaction
if not_in_transaction or not session:
_csot.apply_write_concern(cmd, write_concern)
if self.bypass_doc_val is not None:
cmd["bypassDocumentValidation"] = self.bypass_doc_val
if self.comment:
cmd["comment"] = self.comment # type: ignore[assignment]
if self.let:
cmd["let"] = self.let
if session:
# Start a new retryable write unless one was already
# started for this command.
if retryable and not self.started_retryable_write:
session._start_retryable_write()
self.started_retryable_write = True
session._apply_to(cmd, retryable, ReadPreference.PRIMARY, conn)
conn.send_cluster_time(cmd, session, self.client)
conn.add_server_api(cmd)
# CSOT: apply timeout before encoding the command.
conn.apply_timeout(self.client, cmd)
ops = islice(self.ops, self.idx_offset, None)
# Run as many ops as possible in one server command.
if write_concern.acknowledged:
raw_result, to_send_ops, _ = self._execute_batch(bwc, cmd, ops) # type: ignore[arg-type]
result = copy.deepcopy(raw_result)
# Top-level server/network error.
if result.get("error"):
error = result["error"]
retryable_top_level_error = (
isinstance(error.details, dict)
and error.details.get("code", 0) in _RETRYABLE_ERROR_CODES
)
retryable_network_error = isinstance(
error, ConnectionFailure
) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError))
# Synthesize the full bulk result without modifying the
# current one because this write operation may be retried.
if retryable and (retryable_top_level_error or retryable_network_error):
full = copy.deepcopy(full_result)
_merge_command(self.ops, self.idx_offset, full, result)
_throw_client_bulk_write_exception(full, self.verbose_results)
else:
_merge_command(self.ops, self.idx_offset, full_result, result)
_throw_client_bulk_write_exception(full_result, self.verbose_results)
result["error"] = None
result["writeErrors"] = []
if result.get("nErrors", 0) < len(to_send_ops):
full_result["anySuccessful"] = True
# Top-level command error.
if not result["ok"]:
result["error"] = raw_result
_merge_command(self.ops, self.idx_offset, full_result, result)
break
if retryable:
# Retryable writeConcernErrors halt the execution of this batch.
wce = result.get("writeConcernError", {})
if wce.get("code", 0) in _RETRYABLE_ERROR_CODES:
# Synthesize the full bulk result without modifying the
# current one because this write operation may be retried.
full = copy.deepcopy(full_result)
_merge_command(self.ops, self.idx_offset, full, result)
_throw_client_bulk_write_exception(full, self.verbose_results)
# Process the server reply as a command cursor.
self._process_results_cursor(full_result, result, conn, session)
# Merge this batch's results with the full results.
_merge_command(self.ops, self.idx_offset, full_result, result)
# We're no longer in a retry once a command succeeds.
self.retrying = False
self.started_retryable_write = False
else:
to_send_ops, _ = self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type]
self.idx_offset += len(to_send_ops)
# We halt execution if we hit a top-level error,
# or an individual error in an ordered bulk write.
if full_result["error"] or (self.ordered and full_result["writeErrors"]):
break
def execute_command(
self,
session: Optional[ClientSession],
operation: str,
) -> MutableMapping[str, Any]:
"""Execute commands with w=1 WriteConcern."""
full_result: MutableMapping[str, Any] = {
"anySuccessful": False,
"error": None,
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nDeleted": 0,
"insertResults": {},
"updateResults": {},
"deleteResults": {},
}
op_id = _randint()
def retryable_bulk(
session: Optional[ClientSession],
conn: Connection,
retryable: bool,
) -> None:
if conn.max_wire_version < 25:
raise InvalidOperation(
"MongoClient.bulk_write requires MongoDB server version 8.0+."
)
self._execute_command(
self.write_concern,
session,
conn,
op_id,
retryable,
full_result,
)
self.client._retryable_write(
self.is_retryable,
retryable_bulk,
session,
operation,
bulk=self,
operation_id=op_id,
)
if full_result["error"] or full_result["writeErrors"] or full_result["writeConcernErrors"]:
_throw_client_bulk_write_exception(full_result, self.verbose_results)
return full_result
def execute_command_unack_unordered(
self,
conn: Connection,
) -> None:
"""Execute commands with OP_MSG and w=0 writeConcern, unordered."""
db_name = "admin"
cmd_name = "bulkWrite"
listeners = self.client._event_listeners
op_id = _randint()
bwc = self.bulk_ctx_class(
db_name,
cmd_name,
conn,
op_id,
listeners, # type: ignore[arg-type]
None,
self.client.codec_options,
)
while self.idx_offset < self.total_ops:
# Construct the server command, specifying the relevant options.
cmd = {"bulkWrite": 1}
cmd["errorsOnly"] = not self.verbose_results
cmd["ordered"] = self.ordered # type: ignore[assignment]
if self.bypass_doc_val is not None:
cmd["bypassDocumentValidation"] = self.bypass_doc_val
cmd["writeConcern"] = {"w": 0} # type: ignore[assignment]
if self.comment:
cmd["comment"] = self.comment # type: ignore[assignment]
if self.let:
cmd["let"] = self.let
conn.add_server_api(cmd)
ops = islice(self.ops, self.idx_offset, None)
# Run as many ops as possible in one server command.
to_send_ops, _ = self._execute_batch_unack(bwc, cmd, ops) # type: ignore[arg-type]
self.idx_offset += len(to_send_ops)
def execute_command_unack_ordered(
self,
conn: Connection,
) -> None:
"""Execute commands with OP_MSG and w=0 WriteConcern, ordered."""
full_result: MutableMapping[str, Any] = {
"anySuccessful": False,
"error": None,
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nDeleted": 0,
"insertResults": {},
"updateResults": {},
"deleteResults": {},
}
# Ordered bulk writes have to be acknowledged so that we stop
# processing at the first error, even when the application
# specified unacknowledged writeConcern.
initial_write_concern = WriteConcern()
op_id = _randint()
try:
self._execute_command(
initial_write_concern,
None,
conn,
op_id,
False,
full_result,
self.write_concern,
)
except OperationFailure:
pass
def execute_no_results(
self,
conn: Connection,
) -> None:
"""Execute all operations, returning no results (w=0)."""
if self.uses_collation:
raise ConfigurationError("Collation is unsupported for unacknowledged writes.")
if self.uses_array_filters:
raise ConfigurationError("arrayFilters is unsupported for unacknowledged writes.")
# Cannot have both unacknowledged writes and bypass document validation.
if self.bypass_doc_val is not None:
raise OperationFailure(
"Cannot set bypass_document_validation with unacknowledged write concern"
)
if self.ordered:
return self.execute_command_unack_ordered(conn)
return self.execute_command_unack_unordered(conn)
def execute(
self,
session: Optional[ClientSession],
operation: str,
) -> Any:
"""Execute operations."""
if not self.ops:
raise InvalidOperation("No operations to execute")
if self.executed:
raise InvalidOperation("Bulk operations can only be executed once.")
self.executed = True
session = _validate_session_write_concern(session, self.write_concern)
if not self.write_concern.acknowledged:
with self.client._conn_for_writes(session, operation) as connection:
if connection.max_wire_version < 25:
raise InvalidOperation(
"MongoClient.bulk_write requires MongoDB server version 8.0+."
)
self.execute_no_results(connection)
return ClientBulkWriteResult(None, False, False) # type: ignore[arg-type]
result = self.execute_command(session, operation)
return ClientBulkWriteResult(
result,
self.write_concern.acknowledged,
self.verbose_results,
)

175
pymongo/synchronous/mongo_client.py
View File

@ -62,6 +62,7 @@ from pymongo.client_options import ClientOptions
from pymongo.errors import (
AutoReconnect,
BulkWriteError,
ClientBulkWriteException,
ConfigurationError,
ConnectionFailure,
InvalidOperation,
@ -76,12 +77,22 @@ from pymongo.lock import _HAS_REGISTER_AT_FORK, _create_lock, _release_locks
from pymongo.logger import _CLIENT_LOGGER, _log_or_warn
from pymongo.message import _CursorAddress, _GetMore, _Query
from pymongo.monitoring import ConnectionClosedReason
from pymongo.operations import _Op
from pymongo.operations import (
DeleteMany,
DeleteOne,
InsertOne,
ReplaceOne,
UpdateMany,
UpdateOne,
_Op,
)
from pymongo.read_preferences import ReadPreference, _ServerMode
from pymongo.results import ClientBulkWriteResult
from pymongo.server_selectors import writable_server_selector
from pymongo.server_type import SERVER_TYPE
from pymongo.synchronous import client_session, database, periodic_executor
from pymongo.synchronous.change_stream import ChangeStream, ClusterChangeStream
from pymongo.synchronous.client_bulk import _ClientBulk
from pymongo.synchronous.client_session import _EmptyServerSession
from pymongo.synchronous.command_cursor import CommandCursor
from pymongo.synchronous.settings import TopologySettings
@ -127,6 +138,15 @@ _ReadCall = Callable[
_IS_SYNC = True
_WriteOp = Union[
InsertOne,
DeleteOne,
DeleteMany,
ReplaceOne,
UpdateOne,
UpdateMany,
]
class MongoClient(common.BaseObject, Generic[_DocumentType]):
HOST = "localhost"
@ -1715,7 +1735,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]):
retryable: bool,
func: _WriteCall[T],
session: Optional[ClientSession],
bulk: Optional[_Bulk],
bulk: Optional[Union[_Bulk, _ClientBulk]],
operation: str,
operation_id: Optional[int] = None,
) -> T:
@ -1745,7 +1765,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]):
self,
func: _WriteCall[T] | _ReadCall[T],
session: Optional[ClientSession],
bulk: Optional[_Bulk],
bulk: Optional[Union[_Bulk, _ClientBulk]],
operation: str,
is_read: bool = False,
address: Optional[_Address] = None,
@ -1828,7 +1848,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]):
func: _WriteCall[T],
session: Optional[ClientSession],
operation: str,
bulk: Optional[_Bulk] = None,
bulk: Optional[Union[_Bulk, _ClientBulk]] = None,
operation_id: Optional[int] = None,
) -> T:
"""Execute an operation with consecutive retries if possible
@ -2193,10 +2213,134 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]):
session=session,
)
@_csot.apply
def bulk_write(
self,
models: Sequence[_WriteOp[_DocumentType]],
session: Optional[ClientSession] = None,
ordered: bool = True,
verbose_results: bool = False,
bypass_document_validation: Optional[bool] = None,
comment: Optional[Any] = None,
let: Optional[Mapping] = None,
write_concern: Optional[WriteConcern] = None,
) -> ClientBulkWriteResult:
"""Send a batch of write operations, potentially across multiple namespaces, to the server.
Requests are passed as a list of write operation instances (
:class:`~pymongo.operations.InsertOne`,
:class:`~pymongo.operations.UpdateOne`,
:class:`~pymongo.operations.UpdateMany`,
:class:`~pymongo.operations.ReplaceOne`,
:class:`~pymongo.operations.DeleteOne`, or
:class:`~pymongo.operations.DeleteMany`).
>>> for doc in db.test.find({}):
... print(doc)
...
{'x': 1, '_id': ObjectId('54f62e60fba5226811f634ef')}
{'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')}
...
>>> for doc in db.coll.find({}):
... print(doc)
...
{'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')}
...
>>> # DeleteMany, UpdateOne, and UpdateMany are also available.
>>> from pymongo import InsertOne, DeleteOne, ReplaceOne
>>> models = [InsertOne(namespace="db.test", document={'y': 1}),
... DeleteOne(namespace="db.test", filter={'x': 1}),
... InsertOne(namespace="db.coll", document={'y': 2}),
... ReplaceOne(namespace="db.test", filter={'w': 1}, replacement={'z': 1}, upsert=True)]
>>> result = client.bulk_write(models=models)
>>> result.inserted_count
2
>>> result.deleted_count
1
>>> result.modified_count
0
>>> result.upserted_ids
{3: ObjectId('54f62ee28891e756a6e1abd5')}
>>> for doc in db.test.find({}):
... print(doc)
...
{'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')}
{'y': 1, '_id': ObjectId('54f62ee2fba5226811f634f1')}
{'z': 1, '_id': ObjectId('54f62ee28891e756a6e1abd5')}
...
>>> for doc in db.coll.find({}):
... print(doc)
...
{'x': 2, '_id': ObjectId('507f1f77bcf86cd799439011')}
{'y': 2, '_id': ObjectId('507f1f77bcf86cd799439012')}
:param models: A list of write operation instances.
:param session: (optional) An instance of
:class:`~pymongo.client_session.ClientSession`.
:param ordered: If ``True`` (the default), requests will be
performed on the server serially, in the order provided. If an error
occurs all remaining operations are aborted. If ``False``, requests
will be still performed on the server serially, in the order provided,
but all operations will be attempted even if any errors occur.
:param verbose_results: If ``True``, detailed results for each
successful operation will be included in the returned
:class:`~pymongo.results.ClientBulkWriteResult`. Default is ``False``.
:param bypass_document_validation: (optional) If ``True``, allows the
write to opt-out of document level validation. Default is ``False``.
:param comment: (optional) A user-provided comment to attach to this
command.
:param let: (optional) Map of parameter names and values. Values must be
constant or closed expressions that do not reference document
fields. Parameters can then be accessed as variables in an
aggregate expression context (e.g. "$$var").
:param write_concern: (optional) The write concern to use for this bulk write.
:return: An instance of :class:`~pymongo.results.ClientBulkWriteResult`.
.. seealso:: :ref:`writes-and-ids`
.. note:: requires MongoDB server version 8.0+.
.. versionadded:: 4.9
"""
if self._options.auto_encryption_opts:
raise InvalidOperation(
"MongoClient.bulk_write does not currently support automatic encryption"
)
if session and session.in_transaction:
# Inherit the transaction write concern.
if write_concern:
raise InvalidOperation("Cannot set write concern after starting a transaction")
write_concern = session._transaction.opts.write_concern # type: ignore[union-attr]
else:
# Inherit the client's write concern if none is provided.
if not write_concern:
write_concern = self.write_concern
common.validate_list("models", models)
blk = _ClientBulk(
self,
write_concern=write_concern, # type: ignore[arg-type]
ordered=ordered,
bypass_document_validation=bypass_document_validation,
comment=comment,
let=let,
verbose_results=verbose_results,
)
for model in models:
try:
model._add_to_client_bulk(blk)
except AttributeError:
raise TypeError(f"{model!r} is not a valid request") from None
return blk.execute(session, _Op.BULK_WRITE)
def _retryable_error_doc(exc: PyMongoError) -> Optional[Mapping[str, Any]]:
"""Return the server response from PyMongo exception or None."""
if isinstance(exc, BulkWriteError):
if isinstance(exc, (BulkWriteError, ClientBulkWriteException)):
# Check the last writeConcernError to determine if this
# BulkWriteError is retryable.
wces = exc.details["writeConcernErrors"]
@ -2231,10 +2375,14 @@ def _add_retryable_write_error(exc: PyMongoError, max_wire_version: int, is_mong
# Connection errors are always retryable except NotPrimaryError and WaitQueueTimeoutError which is
# handled above.
if isinstance(exc, ConnectionFailure) and not isinstance(
exc, (NotPrimaryError, WaitQueueTimeoutError)
if isinstance(exc, ClientBulkWriteException):
exc_to_check = exc.error
else:
exc_to_check = exc
if isinstance(exc_to_check, ConnectionFailure) and not isinstance(
exc_to_check, (NotPrimaryError, WaitQueueTimeoutError)
):
exc._add_error_label("RetryableWriteError")
exc_to_check._add_error_label("RetryableWriteError")
class _MongoClientErrorHandler:
@ -2279,6 +2427,8 @@ class _MongoClientErrorHandler:
return
self.handled = True
if self.session:
if isinstance(exc_val, ClientBulkWriteException):
exc_val = exc_val.error
if isinstance(exc_val, ConnectionFailure):
if self.session.in_transaction:
exc_val._add_error_label("TransientTransactionError")
@ -2290,7 +2440,7 @@ class _MongoClientErrorHandler:
):
self.session._unpin()
err_ctx = _ErrorContext(
exc_val,
exc_val, # type: ignore[arg-type]
self.max_wire_version,
self.sock_generation,
self.completed_handshake,
@ -2317,7 +2467,7 @@ class _ClientConnectionRetryable(Generic[T]):
self,
mongo_client: MongoClient,
func: _WriteCall[T] | _ReadCall[T],
bulk: Optional[_Bulk],
bulk: Optional[Union[_Bulk, _ClientBulk]],
operation: str,
is_read: bool = False,
session: Optional[ClientSession] = None,
@ -2394,7 +2544,10 @@ class _ClientConnectionRetryable(Generic[T]):
if not self._is_read:
if not self._retryable:
raise
retryable_write_error_exc = exc.has_error_label("RetryableWriteError")
if isinstance(exc, ClientBulkWriteException) and exc.error:
retryable_write_error_exc = exc.error.has_error_label("RetryableWriteError")
else:
retryable_write_error_exc = exc.has_error_label("RetryableWriteError")
if retryable_write_error_exc:
assert self._session
self._session._unpin()

3
pymongo/typings.py
View File

@ -30,11 +30,13 @@ from bson.typings import _DocumentOut, _DocumentType, _DocumentTypeArg
if TYPE_CHECKING:
from pymongo.asynchronous.bulk import _AsyncBulk
from pymongo.asynchronous.client_bulk import _AsyncClientBulk
from pymongo.asynchronous.client_session import AsyncClientSession
from pymongo.asynchronous.mongo_client import AsyncMongoClient
from pymongo.asynchronous.pool import AsyncConnection
from pymongo.collation import Collation
from pymongo.synchronous.bulk import _Bulk
from pymongo.synchronous.client_bulk import _ClientBulk
from pymongo.synchronous.client_session import ClientSession
from pymongo.synchronous.mongo_client import MongoClient
from pymongo.synchronous.pool import Connection
@ -53,6 +55,7 @@ _AgnosticMongoClient = Union["AsyncMongoClient", "MongoClient"]
_AgnosticConnection = Union["AsyncConnection", "Connection"]
_AgnosticClientSession = Union["AsyncClientSession", "ClientSession"]
_AgnosticBulk = Union["_AsyncBulk", "_Bulk"]
_AgnosticClientBulk = Union["_AsyncClientBulk", "_ClientBulk"]
def strip_optional(elem: Optional[_T]) -> _T:

571
test/asynchronous/test_client_bulk_write.py Normal file
View File

@ -0,0 +1,571 @@
# Copyright 2024-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test the client bulk write API."""
from __future__ import annotations
import sys
sys.path[0:0] = [""]
from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest
from test.utils import (
OvertCommandListener,
async_rs_or_single_client,
)
from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts
from pymongo.errors import (
ClientBulkWriteException,
DocumentTooLarge,
InvalidOperation,
NetworkTimeout,
)
from pymongo.monitoring import *
from pymongo.operations import *
from pymongo.write_concern import WriteConcern
_IS_SYNC = False
class TestClientBulkWrite(AsyncIntegrationTest):
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_returns_error_if_no_namespace_provided(self):
client = await async_rs_or_single_client()
self.addAsyncCleanup(client.aclose)
models = [InsertOne(document={"a": "b"})]
with self.assertRaises(InvalidOperation) as context:
await client.bulk_write(models=models)
self.assertIn(
"MongoClient.bulk_write requires a namespace to be provided for each write operation",
context.exception._message,
)
# https://github.com/mongodb/specifications/tree/master/source/crud/tests
class TestClientBulkWriteCRUD(AsyncIntegrationTest):
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_batch_splits_if_num_operations_too_large(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"]
models = []
for _ in range(max_write_batch_size + 1):
models.append(InsertOne(namespace="db.coll", document={"a": "b"}))
self.addAsyncCleanup(client.db["coll"].drop)
result = await client.bulk_write(models=models)
self.assertEqual(result.inserted_count, max_write_batch_size + 1)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
first_event, second_event = bulk_write_events
self.assertEqual(len(first_event.command["ops"]), max_write_batch_size)
self.assertEqual(len(second_event.command["ops"]), 1)
self.assertEqual(first_event.operation_id, second_event.operation_id)
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_batch_splits_if_ops_payload_too_large(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"]
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
models = []
num_models = int(max_message_size_bytes / max_bson_object_size + 1)
b_repeated = "b" * (max_bson_object_size - 500)
for _ in range(num_models):
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
self.addAsyncCleanup(client.db["coll"].drop)
result = await client.bulk_write(models=models)
self.assertEqual(result.inserted_count, num_models)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
first_event, second_event = bulk_write_events
self.assertEqual(len(first_event.command["ops"]), num_models - 1)
self.assertEqual(len(second_event.command["ops"]), 1)
self.assertEqual(first_event.operation_id, second_event.operation_id)
@async_client_context.require_version_min(8, 0, 0, -24)
@async_client_context.require_failCommand_fail_point
async def test_collects_write_concern_errors_across_batches(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(
event_listeners=[listener],
retryWrites=False,
)
self.addAsyncCleanup(client.aclose)
max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"]
fail_command = {
"configureFailPoint": "failCommand",
"mode": {"times": 2},
"data": {
"failCommands": ["bulkWrite"],
"writeConcernError": {"code": 91, "errmsg": "Replication is being shut down"},
},
}
async with self.fail_point(fail_command):
models = []
for _ in range(max_write_batch_size + 1):
models.append(
InsertOne(
namespace="db.coll",
document={"a": "b"},
)
)
self.addAsyncCleanup(client.db["coll"].drop)
with self.assertRaises(ClientBulkWriteException) as context:
await client.bulk_write(models=models)
self.assertEqual(len(context.exception.write_concern_errors), 2) # type: ignore[arg-type]
self.assertIsNotNone(context.exception.partial_result)
self.assertEqual(
context.exception.partial_result.inserted_count, max_write_batch_size + 1
)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_collects_write_errors_across_batches_unordered(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
collection = client.db["coll"]
self.addAsyncCleanup(collection.drop)
await collection.drop()
await collection.insert_one(document={"_id": 1})
max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"]
models = []
for _ in range(max_write_batch_size + 1):
models.append(
InsertOne(
namespace="db.coll",
document={"_id": 1},
)
)
with self.assertRaises(ClientBulkWriteException) as context:
await client.bulk_write(models=models, ordered=False)
self.assertEqual(len(context.exception.write_errors), max_write_batch_size + 1) # type: ignore[arg-type]
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_collects_write_errors_across_batches_ordered(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
collection = client.db["coll"]
self.addAsyncCleanup(collection.drop)
await collection.drop()
await collection.insert_one(document={"_id": 1})
max_write_batch_size = (await async_client_context.hello)["maxWriteBatchSize"]
models = []
for _ in range(max_write_batch_size + 1):
models.append(
InsertOne(
namespace="db.coll",
document={"_id": 1},
)
)
with self.assertRaises(ClientBulkWriteException) as context:
await client.bulk_write(models=models, ordered=True)
self.assertEqual(len(context.exception.write_errors), 1) # type: ignore[arg-type]
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 1)
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_handles_cursor_requiring_getMore(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
collection = client.db["coll"]
self.addAsyncCleanup(collection.drop)
await collection.drop()
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
models = []
a_repeated = "a" * (max_bson_object_size // 2)
b_repeated = "b" * (max_bson_object_size // 2)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": a_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": b_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
result = await client.bulk_write(models=models, verbose_results=True)
self.assertEqual(result.upserted_count, 2)
self.assertEqual(len(result.update_results), 2)
get_more_event = False
for event in listener.started_events:
if event.command_name == "getMore":
get_more_event = True
self.assertTrue(get_more_event)
@async_client_context.require_version_min(8, 0, 0, -24)
@async_client_context.require_no_standalone
async def test_handles_cursor_requiring_getMore_within_transaction(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
collection = client.db["coll"]
self.addAsyncCleanup(collection.drop)
await collection.drop()
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
async with client.start_session() as session:
await session.start_transaction()
models = []
a_repeated = "a" * (max_bson_object_size // 2)
b_repeated = "b" * (max_bson_object_size // 2)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": a_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": b_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
result = await client.bulk_write(models=models, session=session, verbose_results=True)
self.assertEqual(result.upserted_count, 2)
self.assertEqual(len(result.update_results), 2)
get_more_event = False
for event in listener.started_events:
if event.command_name == "getMore":
get_more_event = True
self.assertTrue(get_more_event)
@async_client_context.require_version_min(8, 0, 0, -24)
@async_client_context.require_failCommand_fail_point
async def test_handles_getMore_error(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
collection = client.db["coll"]
self.addAsyncCleanup(collection.drop)
await collection.drop()
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
fail_command = {
"configureFailPoint": "failCommand",
"mode": {"times": 1},
"data": {"failCommands": ["getMore"], "errorCode": 8},
}
async with self.fail_point(fail_command):
models = []
a_repeated = "a" * (max_bson_object_size // 2)
b_repeated = "b" * (max_bson_object_size // 2)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": a_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": b_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
with self.assertRaises(ClientBulkWriteException) as context:
await client.bulk_write(models=models, verbose_results=True)
self.assertIsNotNone(context.exception.error)
self.assertEqual(context.exception.error["code"], 8)
self.assertIsNotNone(context.exception.partial_result)
self.assertEqual(context.exception.partial_result.upserted_count, 2)
self.assertEqual(len(context.exception.partial_result.update_results), 1)
get_more_event = False
kill_cursors_event = False
for event in listener.started_events:
if event.command_name == "getMore":
get_more_event = True
if event.command_name == "killCursors":
kill_cursors_event = True
self.assertTrue(get_more_event)
self.assertTrue(kill_cursors_event)
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_returns_error_if_unacknowledged_too_large_insert(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
b_repeated = "b" * max_bson_object_size
# Insert document.
models_insert = [InsertOne(namespace="db.coll", document={"a": b_repeated})]
with self.assertRaises(DocumentTooLarge):
await client.bulk_write(models=models_insert, write_concern=WriteConcern(w=0))
# Replace document.
models_replace = [ReplaceOne(namespace="db.coll", filter={}, replacement={"a": b_repeated})]
with self.assertRaises(DocumentTooLarge):
await client.bulk_write(models=models_replace, write_concern=WriteConcern(w=0))
async def _setup_namespace_test_models(self):
max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"]
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
ops_bytes = max_message_size_bytes - 1122
num_models = ops_bytes // max_bson_object_size
remainder_bytes = ops_bytes % max_bson_object_size
models = []
b_repeated = "b" * (max_bson_object_size - 57)
for _ in range(num_models):
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
if remainder_bytes >= 217:
num_models += 1
b_repeated = "b" * (remainder_bytes - 57)
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
return num_models, models
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_no_batch_splits_if_new_namespace_is_not_too_large(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
num_models, models = await self._setup_namespace_test_models()
models.append(
InsertOne(
namespace="db.coll",
document={"a": "b"},
)
)
self.addAsyncCleanup(client.db["coll"].drop)
# No batch splitting required.
result = await client.bulk_write(models=models)
self.assertEqual(result.inserted_count, num_models + 1)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 1)
event = bulk_write_events[0]
self.assertEqual(len(event.command["ops"]), num_models + 1)
self.assertEqual(len(event.command["nsInfo"]), 1)
self.assertEqual(event.command["nsInfo"][0]["ns"], "db.coll")
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_batch_splits_if_new_namespace_is_too_large(self):
listener = OvertCommandListener()
client = await async_rs_or_single_client(event_listeners=[listener])
self.addAsyncCleanup(client.aclose)
num_models, models = await self._setup_namespace_test_models()
c_repeated = "c" * 200
namespace = f"db.{c_repeated}"
models.append(
InsertOne(
namespace=namespace,
document={"a": "b"},
)
)
self.addAsyncCleanup(client.db["coll"].drop)
self.addAsyncCleanup(client.db[c_repeated].drop)
# Batch splitting required.
result = await client.bulk_write(models=models)
self.assertEqual(result.inserted_count, num_models + 1)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
first_event, second_event = bulk_write_events
self.assertEqual(len(first_event.command["ops"]), num_models)
self.assertEqual(len(first_event.command["nsInfo"]), 1)
self.assertEqual(first_event.command["nsInfo"][0]["ns"], "db.coll")
self.assertEqual(len(second_event.command["ops"]), 1)
self.assertEqual(len(second_event.command["nsInfo"]), 1)
self.assertEqual(second_event.command["nsInfo"][0]["ns"], namespace)
@async_client_context.require_version_min(8, 0, 0, -24)
async def test_returns_error_if_no_writes_can_be_added_to_ops(self):
client = await async_rs_or_single_client()
self.addAsyncCleanup(client.aclose)
max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"]
# Document too large.
b_repeated = "b" * max_message_size_bytes
models = [InsertOne(namespace="db.coll", document={"a": b_repeated})]
with self.assertRaises(InvalidOperation) as context:
await client.bulk_write(models=models)
self.assertIn("cannot do an empty bulk write", context.exception._message)
# Namespace too large.
c_repeated = "c" * max_message_size_bytes
namespace = f"db.{c_repeated}"
models = [InsertOne(namespace=namespace, document={"a": "b"})]
with self.assertRaises(InvalidOperation) as context:
await client.bulk_write(models=models)
self.assertIn("cannot do an empty bulk write", context.exception._message)
@async_client_context.require_version_min(8, 0, 0, -24)
@unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed")
async def test_returns_error_if_auto_encryption_configured(self):
opts = AutoEncryptionOpts(
key_vault_namespace="db.coll",
kms_providers={"aws": {"accessKeyId": "foo", "secretAccessKey": "bar"}},
)
client = await async_rs_or_single_client(auto_encryption_opts=opts)
self.addAsyncCleanup(client.aclose)
models = [InsertOne(namespace="db.coll", document={"a": "b"})]
with self.assertRaises(InvalidOperation) as context:
await client.bulk_write(models=models)
self.assertIn(
"bulk_write does not currently support automatic encryption", context.exception._message
)
# https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/tests/README.md#11-multi-batch-bulkwrites
class TestClientBulkWriteTimeout(AsyncIntegrationTest):
@async_client_context.require_version_min(8, 0, 0, -24)
@async_client_context.require_failCommand_fail_point
async def test_timeout_in_multi_batch_bulk_write(self):
internal_client = await async_rs_or_single_client(timeoutMS=None)
self.addAsyncCleanup(internal_client.aclose)
collection = internal_client.db["coll"]
self.addAsyncCleanup(collection.drop)
await collection.drop()
max_bson_object_size = (await async_client_context.hello)["maxBsonObjectSize"]
max_message_size_bytes = (await async_client_context.hello)["maxMessageSizeBytes"]
fail_command = {
"configureFailPoint": "failCommand",
"mode": {"times": 2},
"data": {"failCommands": ["bulkWrite"], "blockConnection": True, "blockTimeMS": 1010},
}
async with self.fail_point(fail_command):
models = []
num_models = int(max_message_size_bytes / max_bson_object_size + 1)
b_repeated = "b" * (max_bson_object_size - 500)
for _ in range(num_models):
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
listener = OvertCommandListener()
client = await async_rs_or_single_client(
event_listeners=[listener],
readConcernLevel="majority",
readPreference="primary",
timeoutMS=2000,
w="majority",
)
self.addAsyncCleanup(client.aclose)
with self.assertRaises(ClientBulkWriteException) as context:
await client.bulk_write(models=models)
self.assertIsInstance(context.exception.error, NetworkTimeout)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)

218
test/command_monitoring/unacknowledged-client-bulkWrite.json Normal file
View File

@ -0,0 +1,218 @@
{
"description": "unacknowledged-client-bulkWrite",
"schemaVersion": "1.7",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client",
"useMultipleMongoses": false,
"observeEvents": [
"commandStartedEvent",
"commandSucceededEvent",
"commandFailedEvent"
],
"uriOptions": {
"w": 0
}
}
},
{
"database": {
"id": "database",
"client": "client",
"databaseName": "command-monitoring-tests"
}
},
{
"collection": {
"id": "collection",
"database": "database",
"collectionName": "test"
}
}
],
"initialData": [
{
"collectionName": "test",
"databaseName": "command-monitoring-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"_yamlAnchors": {
"namespace": "command-monitoring-tests.test"
},
"tests": [
{
"description": "A successful mixed client bulkWrite",
"operations": [
{
"object": "client",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "command-monitoring-tests.test",
"document": {
"_id": 4,
"x": 44
}
}
},
{
"updateOne": {
"namespace": "command-monitoring-tests.test",
"filter": {
"_id": 3
},
"update": {
"$set": {
"x": 333
}
}
}
}
]
},
"expectResult": {
"insertedCount": {
"$$unsetOrMatches": 0
},
"upsertedCount": {
"$$unsetOrMatches": 0
},
"matchedCount": {
"$$unsetOrMatches": 0
},
"modifiedCount": {
"$$unsetOrMatches": 0
},
"deletedCount": {
"$$unsetOrMatches": 0
},
"insertResults": {
"$$unsetOrMatches": {}
},
"updateResults": {
"$$unsetOrMatches": {}
},
"deleteResults": {
"$$unsetOrMatches": {}
}
}
},
{
"object": "collection",
"name": "find",
"arguments": {
"filter": {}
},
"expectResult": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 333
},
{
"_id": 4,
"x": 44
}
]
}
],
"expectEvents": [
{
"client": "client",
"ignoreExtraEvents": true,
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
},
{
"update": 0,
"filter": {
"_id": 3
},
"updateMods": {
"$set": {
"x": 333
}
},
"multi": false
}
],
"nsInfo": [
{
"ns": "command-monitoring-tests.test"
}
]
}
}
},
{
"commandSucceededEvent": {
"commandName": "bulkWrite",
"reply": {
"ok": 1,
"nInserted": {
"$$exists": false
},
"nMatched": {
"$$exists": false
},
"nModified": {
"$$exists": false
},
"nUpserted": {
"$$exists": false
},
"nDeleted": {
"$$exists": false
}
}
}
}
]
}
]
}
]
}

267
test/crud/unified/client-bulkWrite-delete-options.json Normal file
View File

@ -0,0 +1,267 @@
{
"description": "client bulkWrite delete options",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0",
"collation": {
"locale": "simple"
},
"hint": "_id_"
},
"tests": [
{
"description": "client bulk write delete with collation",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"collation": {
"locale": "simple"
}
}
},
{
"deleteMany": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": {
"$gt": 1
}
},
"collation": {
"locale": "simple"
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 3,
"insertResults": {},
"updateResults": {},
"deleteResults": {
"0": {
"deletedCount": 1
},
"1": {
"deletedCount": 2
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"delete": 0,
"filter": {
"_id": 1
},
"collation": {
"locale": "simple"
},
"multi": false
},
{
"delete": 0,
"filter": {
"_id": {
"$gt": 1
}
},
"collation": {
"locale": "simple"
},
"multi": true
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": []
}
]
},
{
"description": "client bulk write delete with hint",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"hint": "_id_"
}
},
{
"deleteMany": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": {
"$gt": 1
}
},
"hint": "_id_"
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 3,
"insertResults": {},
"updateResults": {},
"deleteResults": {
"0": {
"deletedCount": 1
},
"1": {
"deletedCount": 2
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"delete": 0,
"filter": {
"_id": 1
},
"hint": "_id_",
"multi": false
},
{
"delete": 0,
"filter": {
"_id": {
"$gt": 1
}
},
"hint": "_id_",
"multi": true
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": []
}
]
}
]
}

68
test/crud/unified/client-bulkWrite-errorResponse.json Normal file
View File

@ -0,0 +1,68 @@
{
"description": "client bulkWrite errorResponse",
"schemaVersion": "1.12",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"useMultipleMongoses": false
}
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite operations support errorResponse assertions",
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 8
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 1
}
}
}
]
},
"expectError": {
"errorCode": 8,
"errorResponse": {
"code": 8
}
}
}
]
}
]
}

454
test/crud/unified/client-bulkWrite-errors.json Normal file
View File

@ -0,0 +1,454 @@
{
"description": "client bulkWrite errors",
"schemaVersion": "1.21",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
],
"uriOptions": {
"retryWrites": false
},
"useMultipleMongoses": false
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0",
"writeConcernErrorCode": 91,
"writeConcernErrorMessage": "Replication is being shut down",
"undefinedVarCode": 17276
},
"tests": [
{
"description": "an individual operation fails during an ordered bulkWrite",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id2"
]
}
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 3
}
}
}
],
"verboseResults": true
},
"expectError": {
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 1,
"insertResults": {},
"updateResults": {},
"deleteResults": {
"0": {
"deletedCount": 1
}
}
},
"writeErrors": {
"1": {
"code": 17276
}
}
}
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
},
{
"description": "an individual operation fails during an unordered bulkWrite",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id2"
]
}
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 3
}
}
}
],
"verboseResults": true,
"ordered": false
},
"expectError": {
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 2,
"insertResults": {},
"updateResults": {},
"deleteResults": {
"0": {
"deletedCount": 1
},
"2": {
"deletedCount": 1
}
}
},
"writeErrors": {
"1": {
"code": 17276
}
}
}
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 2,
"x": 22
}
]
}
]
},
{
"description": "detailed results are omitted from error when verboseResults is false",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id2"
]
}
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 3
}
}
}
],
"verboseResults": false
},
"expectError": {
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 1,
"insertResults": {
"$$unsetOrMatches": {}
},
"updateResults": {
"$$unsetOrMatches": {}
},
"deleteResults": {
"$$unsetOrMatches": {}
}
},
"writeErrors": {
"1": {
"code": 17276
}
}
}
}
]
},
{
"description": "a top-level failure occurs during a bulkWrite",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 8
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"x": 1
}
}
}
],
"verboseResults": true
},
"expectError": {
"errorCode": 8
}
}
]
},
{
"description": "a bulk write with only errors does not report a partial result",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id2"
]
}
}
}
}
],
"verboseResults": true
},
"expectError": {
"expectResult": {
"$$unsetOrMatches": {}
},
"writeErrors": {
"0": {
"code": 17276
}
}
}
}
]
},
{
"description": "a write concern error occurs during a bulkWrite",
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"writeConcernError": {
"code": 91,
"errmsg": "Replication is being shut down"
}
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 10
}
}
}
],
"verboseResults": true
},
"expectError": {
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 10
}
},
"updateResults": {},
"deleteResults": {}
},
"writeConcernErrors": [
{
"code": 91,
"message": "Replication is being shut down"
}
]
}
}
]
},
{
"description": "an empty list of write models is a client-side error",
"operations": [
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"models": [],
"verboseResults": true
},
"expectError": {
"isClientError": true
}
}
]
}
]
}

314
test/crud/unified/client-bulkWrite-mixed-namespaces.json Normal file
View File

@ -0,0 +1,314 @@
{
"description": "client bulkWrite with mixed namespaces",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "db0"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
},
{
"collection": {
"id": "collection1",
"database": "database0",
"collectionName": "coll1"
}
},
{
"database": {
"id": "database1",
"client": "client0",
"databaseName": "db1"
}
},
{
"collection": {
"id": "collection2",
"database": "database1",
"collectionName": "coll2"
}
}
],
"initialData": [
{
"databaseName": "db0",
"collectionName": "coll0",
"documents": []
},
{
"databaseName": "db0",
"collectionName": "coll1",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
}
]
},
{
"databaseName": "db1",
"collectionName": "coll2",
"documents": [
{
"_id": 3,
"x": 33
},
{
"_id": 4,
"x": 44
}
]
}
],
"_yamlAnchors": {
"db0Coll0Namespace": "db0.coll0",
"db0Coll1Namespace": "db0.coll1",
"db1Coll2Namespace": "db1.coll2"
},
"tests": [
{
"description": "client bulkWrite with mixed namespaces",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "db0.coll0",
"document": {
"_id": 1
}
}
},
{
"insertOne": {
"namespace": "db0.coll0",
"document": {
"_id": 2
}
}
},
{
"updateOne": {
"namespace": "db0.coll1",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"deleteOne": {
"namespace": "db1.coll2",
"filter": {
"_id": 3
}
}
},
{
"deleteOne": {
"namespace": "db0.coll1",
"filter": {
"_id": 2
}
}
},
{
"replaceOne": {
"namespace": "db1.coll2",
"filter": {
"_id": 4
},
"replacement": {
"x": 45
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 2,
"upsertedCount": 0,
"matchedCount": 2,
"modifiedCount": 2,
"deletedCount": 2,
"insertResults": {
"0": {
"insertedId": 1
},
"1": {
"insertedId": 2
}
},
"updateResults": {
"2": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"5": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {
"3": {
"deletedCount": 1
},
"4": {
"deletedCount": 1
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"bulkWrite": 1,
"ops": [
{
"insert": 0,
"document": {
"_id": 1
}
},
{
"insert": 0,
"document": {
"_id": 2
}
},
{
"update": 1,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"delete": 2,
"filter": {
"_id": 3
},
"multi": false
},
{
"delete": 1,
"filter": {
"_id": 2
},
"multi": false
},
{
"update": 2,
"filter": {
"_id": 4
},
"updateMods": {
"x": 45
},
"multi": false
}
],
"nsInfo": [
{
"ns": "db0.coll0"
},
{
"ns": "db0.coll1"
},
{
"ns": "db1.coll2"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "db0",
"collectionName": "coll0",
"documents": [
{
"_id": 1
},
{
"_id": 2
}
]
},
{
"databaseName": "db0",
"collectionName": "coll1",
"documents": [
{
"_id": 1,
"x": 12
}
]
},
{
"databaseName": "db1",
"collectionName": "coll2",
"documents": [
{
"_id": 4,
"x": 45
}
]
}
]
}
]
}

715
test/crud/unified/client-bulkWrite-options.json Normal file
View File

@ -0,0 +1,715 @@
{
"description": "client bulkWrite top-level options",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"client": {
"id": "writeConcernClient",
"uriOptions": {
"w": 1
},
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0",
"comment": {
"bulk": "write"
},
"let": {
"id1": 1,
"id2": 2
},
"writeConcern": {
"w": "majority"
}
},
"tests": [
{
"description": "client bulkWrite comment",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 3,
"x": 33
}
}
}
],
"comment": {
"bulk": "write"
},
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 3
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"comment": {
"bulk": "write"
},
"ops": [
{
"insert": 0,
"document": {
"_id": 3,
"x": 33
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
},
{
"description": "client bulkWrite bypassDocumentValidation",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 3,
"x": 33
}
}
}
],
"bypassDocumentValidation": true,
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 3
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"bypassDocumentValidation": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 3,
"x": 33
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
},
{
"description": "client bulkWrite let",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id1"
]
}
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id2"
]
}
}
}
}
],
"let": {
"id1": 1,
"id2": 2
},
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 1,
"modifiedCount": 1,
"deletedCount": 1,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {
"1": {
"deletedCount": 1
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"let": {
"id1": 1,
"id2": 2
},
"ops": [
{
"update": 0,
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id1"
]
}
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"delete": 0,
"filter": {
"$expr": {
"$eq": [
"$_id",
"$$id2"
]
}
},
"multi": false
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"x": 12
}
]
}
]
},
{
"description": "client bulkWrite bypassDocumentValidation: false is sent",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 3,
"x": 33
}
}
}
],
"bypassDocumentValidation": false,
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 3
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"bypassDocumentValidation": false,
"ops": [
{
"insert": 0,
"document": {
"_id": 3,
"x": 33
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
},
{
"description": "client bulkWrite writeConcern",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 3,
"x": 33
}
}
}
],
"writeConcern": {
"w": "majority"
},
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 3
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"writeConcern": {
"w": "majority"
},
"ops": [
{
"insert": 0,
"document": {
"_id": 3,
"x": 33
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
]
},
{
"description": "client bulkWrite inherits writeConcern from client",
"operations": [
{
"object": "writeConcernClient",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 3,
"x": 33
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 3
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "writeConcernClient",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"writeConcern": {
"w": 1
},
"ops": [
{
"insert": 0,
"document": {
"_id": 3,
"x": 33
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
]
},
{
"description": "client bulkWrite writeConcern option overrides client writeConcern",
"operations": [
{
"object": "writeConcernClient",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 3,
"x": 33
}
}
}
],
"writeConcern": {
"w": "majority"
},
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 3
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "writeConcernClient",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"writeConcern": {
"w": "majority"
},
"ops": [
{
"insert": 0,
"document": {
"_id": 3,
"x": 33
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
]
}
]
}

290
test/crud/unified/client-bulkWrite-ordered.json Normal file
View File

@ -0,0 +1,290 @@
{
"description": "client bulkWrite with ordered option",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": []
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite with ordered: false",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 1,
"x": 11
}
}
}
],
"verboseResults": true,
"ordered": false
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 1
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": false,
"ops": [
{
"insert": 0,
"document": {
"_id": 1,
"x": 11
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
]
},
{
"description": "client bulkWrite with ordered: true",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 1,
"x": 11
}
}
}
],
"verboseResults": true,
"ordered": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 1
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 1,
"x": 11
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
]
},
{
"description": "client bulkWrite defaults to ordered: true",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 1,
"x": 11
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 1
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 1,
"x": 11
}
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
]
}
]
}

832
test/crud/unified/client-bulkWrite-results.json Normal file
View File

@ -0,0 +1,832 @@
{
"description": "client bulkWrite results",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
},
{
"_id": 5,
"x": 55
},
{
"_id": 6,
"x": 66
},
{
"_id": 7,
"x": 77
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite with verboseResults: true returns detailed results",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 8,
"x": 88
}
}
},
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$inc": {
"x": 2
}
}
}
},
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 4
},
"replacement": {
"x": 44
},
"upsert": true
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 5
}
}
},
{
"deleteMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 1,
"matchedCount": 3,
"modifiedCount": 3,
"deletedCount": 3,
"insertResults": {
"0": {
"insertedId": 8
}
},
"updateResults": {
"1": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"2": {
"matchedCount": 2,
"modifiedCount": 2,
"upsertedId": {
"$$exists": false
}
},
"3": {
"matchedCount": 1,
"modifiedCount": 0,
"upsertedId": 4
}
},
"deleteResults": {
"4": {
"deletedCount": 1
},
"5": {
"deletedCount": 2
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 8,
"x": 88
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$inc": {
"x": 2
}
},
"multi": true
},
{
"update": 0,
"filter": {
"_id": 4
},
"updateMods": {
"x": 44
},
"upsert": true,
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 5
},
"multi": false
},
{
"delete": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
},
"multi": true
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 12
},
{
"_id": 2,
"x": 24
},
{
"_id": 3,
"x": 35
},
{
"_id": 4,
"x": 44
},
{
"_id": 8,
"x": 88
}
]
}
]
},
{
"description": "client bulkWrite with verboseResults: false omits detailed results",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 8,
"x": 88
}
}
},
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$inc": {
"x": 2
}
}
}
},
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 4
},
"replacement": {
"x": 44
},
"upsert": true
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 5
}
}
},
{
"deleteMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
}
}
}
],
"verboseResults": false
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 1,
"matchedCount": 3,
"modifiedCount": 3,
"deletedCount": 3,
"insertResults": {
"$$unsetOrMatches": {}
},
"updateResults": {
"$$unsetOrMatches": {}
},
"deleteResults": {
"$$unsetOrMatches": {}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 8,
"x": 88
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$inc": {
"x": 2
}
},
"multi": true
},
{
"update": 0,
"filter": {
"_id": 4
},
"updateMods": {
"x": 44
},
"upsert": true,
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 5
},
"multi": false
},
{
"delete": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
},
"multi": true
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 12
},
{
"_id": 2,
"x": 24
},
{
"_id": 3,
"x": 35
},
{
"_id": 4,
"x": 44
},
{
"_id": 8,
"x": 88
}
]
}
]
},
{
"description": "client bulkWrite defaults to verboseResults: false",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "crud-tests.coll0",
"document": {
"_id": 8,
"x": 88
}
}
},
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$inc": {
"x": 2
}
}
}
},
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 4
},
"replacement": {
"x": 44
},
"upsert": true
}
},
{
"deleteOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 5
}
}
},
{
"deleteMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
}
}
}
]
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 1,
"matchedCount": 3,
"modifiedCount": 3,
"deletedCount": 3,
"insertResults": {
"$$unsetOrMatches": {}
},
"updateResults": {
"$$unsetOrMatches": {}
},
"deleteResults": {
"$$unsetOrMatches": {}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 8,
"x": 88
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$inc": {
"x": 2
}
},
"multi": true
},
{
"update": 0,
"filter": {
"_id": 4
},
"updateMods": {
"x": 44
},
"upsert": true,
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 5
},
"multi": false
},
{
"delete": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
},
"multi": true
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 12
},
{
"_id": 2,
"x": 24
},
{
"_id": 3,
"x": 35
},
{
"_id": 4,
"x": 44
},
{
"_id": 8,
"x": 88
}
]
}
]
}
]
}

948
test/crud/unified/client-bulkWrite-update-options.json Normal file
View File

@ -0,0 +1,948 @@
{
"description": "client bulkWrite update options",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"array": [
1,
2,
3
]
},
{
"_id": 2,
"array": [
1,
2,
3
]
},
{
"_id": 3,
"array": [
1,
2,
3
]
},
{
"_id": 4,
"array": [
1,
2,
3
]
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0",
"collation": {
"locale": "simple"
},
"hint": "_id_"
},
"tests": [
{
"description": "client bulkWrite update with arrayFilters",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$set": {
"array.$[i]": 4
}
},
"arrayFilters": [
{
"i": {
"$gte": 2
}
}
]
}
},
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$set": {
"array.$[i]": 5
}
},
"arrayFilters": [
{
"i": {
"$gte": 2
}
}
]
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 3,
"modifiedCount": 3,
"deletedCount": 0,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"1": {
"matchedCount": 2,
"modifiedCount": 2,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$set": {
"array.$[i]": 4
}
},
"arrayFilters": [
{
"i": {
"$gte": 2
}
}
],
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$set": {
"array.$[i]": 5
}
},
"arrayFilters": [
{
"i": {
"$gte": 2
}
}
],
"multi": true
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"array": [
1,
4,
4
]
},
{
"_id": 2,
"array": [
1,
5,
5
]
},
{
"_id": 3,
"array": [
1,
5,
5
]
},
{
"_id": 4,
"array": [
1,
2,
3
]
}
]
}
]
},
{
"description": "client bulkWrite update with collation",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$set": {
"array": [
1,
2,
4
]
}
},
"collation": {
"locale": "simple"
}
}
},
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$set": {
"array": [
1,
2,
5
]
}
},
"collation": {
"locale": "simple"
}
}
},
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 4
},
"replacement": {
"array": [
1,
2,
6
]
},
"collation": {
"locale": "simple"
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 4,
"modifiedCount": 4,
"deletedCount": 0,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"1": {
"matchedCount": 2,
"modifiedCount": 2,
"upsertedId": {
"$$exists": false
}
},
"2": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$set": {
"array": [
1,
2,
4
]
}
},
"collation": {
"locale": "simple"
},
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$set": {
"array": [
1,
2,
5
]
}
},
"collation": {
"locale": "simple"
},
"multi": true
},
{
"update": 0,
"filter": {
"_id": 4
},
"updateMods": {
"array": [
1,
2,
6
]
},
"collation": {
"locale": "simple"
},
"multi": false
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"array": [
1,
2,
4
]
},
{
"_id": 2,
"array": [
1,
2,
5
]
},
{
"_id": 3,
"array": [
1,
2,
5
]
},
{
"_id": 4,
"array": [
1,
2,
6
]
}
]
}
]
},
{
"description": "client bulkWrite update with hint",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$set": {
"array": [
1,
2,
4
]
}
},
"hint": "_id_"
}
},
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$set": {
"array": [
1,
2,
5
]
}
},
"hint": "_id_"
}
},
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 4
},
"replacement": {
"array": [
1,
2,
6
]
},
"hint": "_id_"
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 4,
"modifiedCount": 4,
"deletedCount": 0,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"1": {
"matchedCount": 2,
"modifiedCount": 2,
"upsertedId": {
"$$exists": false
}
},
"2": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$set": {
"array": [
1,
2,
4
]
}
},
"hint": "_id_",
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$set": {
"array": [
1,
2,
5
]
}
},
"hint": "_id_",
"multi": true
},
{
"update": 0,
"filter": {
"_id": 4
},
"updateMods": {
"array": [
1,
2,
6
]
},
"hint": "_id_",
"multi": false
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"array": [
1,
2,
4
]
},
{
"_id": 2,
"array": [
1,
2,
5
]
},
{
"_id": 3,
"array": [
1,
2,
5
]
},
{
"_id": 4,
"array": [
1,
2,
6
]
}
]
}
]
},
{
"description": "client bulkWrite update with upsert",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 5
},
"update": {
"$set": {
"array": [
1,
2,
4
]
}
},
"upsert": true
}
},
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 6
},
"replacement": {
"array": [
1,
2,
6
]
},
"upsert": true
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 2,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 1,
"modifiedCount": 0,
"upsertedId": 5
},
"1": {
"matchedCount": 1,
"modifiedCount": 0,
"upsertedId": 6
}
},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 5
},
"updateMods": {
"$set": {
"array": [
1,
2,
4
]
}
},
"upsert": true,
"multi": false
},
{
"update": 0,
"filter": {
"_id": 6
},
"updateMods": {
"array": [
1,
2,
6
]
},
"upsert": true,
"multi": false
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"array": [
1,
2,
3
]
},
{
"_id": 2,
"array": [
1,
2,
3
]
},
{
"_id": 3,
"array": [
1,
2,
3
]
},
{
"_id": 4,
"array": [
1,
2,
3
]
},
{
"_id": 5,
"array": [
1,
2,
4
]
},
{
"_id": 6,
"array": [
1,
2,
6
]
}
]
}
]
}
]
}

257
test/crud/unified/client-bulkWrite-update-pipeline.json Normal file
View File

@ -0,0 +1,257 @@
{
"description": "client bulkWrite update pipeline",
"schemaVersion": "1.1",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 1
},
{
"_id": 2,
"x": 2
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite updateOne with pipeline",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": [
{
"$addFields": {
"foo": 1
}
}
]
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 1,
"modifiedCount": 1,
"deletedCount": 0,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": [
{
"$addFields": {
"foo": 1
}
}
],
"multi": false
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"x": 1,
"foo": 1
},
{
"_id": 2,
"x": 2
}
]
}
]
},
{
"description": "client bulkWrite updateMany with pipeline",
"operations": [
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {},
"update": [
{
"$addFields": {
"foo": 1
}
}
]
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 0,
"upsertedCount": 0,
"matchedCount": 2,
"modifiedCount": 2,
"deletedCount": 0,
"insertResults": {},
"updateResults": {
"0": {
"matchedCount": 2,
"modifiedCount": 2,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {},
"updateMods": [
{
"$addFields": {
"foo": 1
}
}
],
"multi": true
}
],
"nsInfo": [
{
"ns": "crud-tests.coll0"
}
]
}
}
}
]
}
],
"outcome": [
{
"databaseName": "crud-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"x": 1,
"foo": 1
},
{
"_id": 2,
"x": 2,
"foo": 1
}
]
}
]
}
]
}

216
test/crud/unified/client-bulkWrite-update-validation.json Normal file
View File

@ -0,0 +1,216 @@
{
"description": "client-bulkWrite-update-validation",
"schemaVersion": "1.1",
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "crud-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"_yamlAnchors": {
"namespace": "crud-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite replaceOne prohibits atomic modifiers",
"operations": [
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"models": [
{
"replaceOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"replacement": {
"$set": {
"x": 22
}
}
}
}
]
},
"expectError": {
"isClientError": true
}
}
],
"expectEvents": [
{
"client": "client0",
"events": []
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
},
{
"description": "client bulkWrite updateOne requires atomic modifiers",
"operations": [
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"models": [
{
"updateOne": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"x": 22
}
}
}
]
},
"expectError": {
"isClientError": true
}
}
],
"expectEvents": [
{
"client": "client0",
"events": []
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
},
{
"description": "client bulkWrite updateMany requires atomic modifiers",
"operations": [
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"models": [
{
"updateMany": {
"namespace": "crud-tests.coll0",
"filter": {
"_id": {
"$gt": 1
}
},
"update": {
"x": 44
}
}
}
]
},
"expectError": {
"isClientError": true
}
}
],
"expectEvents": [
{
"client": "client0",
"events": []
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "crud-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
}
]
}

350
test/retryable_writes/unified/client-bulkWrite-clientErrors.json Normal file
View File

@ -0,0 +1,350 @@
{
"description": "client bulkWrite retryable writes with client errors",
"schemaVersion": "1.21",
"runOnRequirements": [
{
"minServerVersion": "8.0",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
]
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
],
"useMultipleMongoses": false
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "retryable-writes-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "retryable-writes-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"_yamlAnchors": {
"namespace": "retryable-writes-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite with one network error succeeds after retry",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"closeConnection": true
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-tests.coll0",
"document": {
"_id": 4,
"x": 44
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 4
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
},
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "retryable-writes-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
},
{
"_id": 4,
"x": 44
}
]
}
]
},
{
"description": "client bulkWrite with two network errors fails after retry",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"bulkWrite"
],
"closeConnection": true
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-tests.coll0",
"document": {
"_id": 4,
"x": 44
}
}
}
],
"verboseResults": true
},
"expectError": {
"isClientError": true,
"errorLabelsContain": [
"RetryableWriteError"
]
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
},
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "retryable-writes-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
]
}
]
}

872
test/retryable_writes/unified/client-bulkWrite-serverErrors.json Normal file
View File

@ -0,0 +1,872 @@
{
"description": "client bulkWrite retryable writes",
"schemaVersion": "1.21",
"runOnRequirements": [
{
"minServerVersion": "8.0",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
]
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
],
"useMultipleMongoses": false
}
},
{
"client": {
"id": "clientRetryWritesFalse",
"uriOptions": {
"retryWrites": false
},
"observeEvents": [
"commandStartedEvent"
],
"useMultipleMongoses": false
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "retryable-writes-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "retryable-writes-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"_yamlAnchors": {
"namespace": "retryable-writes-tests.coll0"
},
"tests": [
{
"description": "client bulkWrite with no multi: true operations succeeds after retryable top-level error",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 189,
"errorLabels": [
"RetryableWriteError"
]
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-tests.coll0",
"document": {
"_id": 4,
"x": 44
}
}
},
{
"updateOne": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"replaceOne": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 2
},
"replacement": {
"x": 222
}
}
},
{
"deleteOne": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 3
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 2,
"modifiedCount": 2,
"deletedCount": 1,
"insertResults": {
"0": {
"insertedId": 4
}
},
"updateResults": {
"1": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"2": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {
"3": {
"deletedCount": 1
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"_id": 2
},
"updateMods": {
"x": 222
},
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 3
},
"multi": false
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
},
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"_id": 2
},
"updateMods": {
"x": 222
},
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 3
},
"multi": false
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "retryable-writes-tests",
"documents": [
{
"_id": 1,
"x": 12
},
{
"_id": 2,
"x": 222
},
{
"_id": 4,
"x": 44
}
]
}
]
},
{
"description": "client bulkWrite with multi: true operations fails after retryable top-level error",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 189,
"errorLabels": [
"RetryableWriteError"
]
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateMany": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"deleteMany": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 3
}
}
}
]
},
"expectError": {
"errorCode": 189,
"errorLabelsContain": [
"RetryableWriteError"
]
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": true
},
{
"delete": 0,
"filter": {
"_id": 3
},
"multi": true
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
]
}
}
}
]
}
]
},
{
"description": "client bulkWrite with no multi: true operations succeeds after retryable writeConcernError",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorLabels": [
"RetryableWriteError"
],
"writeConcernError": {
"code": 91,
"errmsg": "Replication is being shut down"
}
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-tests.coll0",
"document": {
"_id": 4,
"x": 44
}
}
},
{
"updateOne": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"replaceOne": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 2
},
"replacement": {
"x": 222
}
}
},
{
"deleteOne": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 3
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 2,
"modifiedCount": 2,
"deletedCount": 1,
"insertResults": {
"0": {
"insertedId": 4
}
},
"updateResults": {
"1": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"2": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
}
},
"deleteResults": {
"3": {
"deletedCount": 1
}
}
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"_id": 2
},
"updateMods": {
"x": 222
},
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 3
},
"multi": false
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
},
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"_id": 2
},
"updateMods": {
"x": 222
},
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 3
},
"multi": false
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
],
"lsid": {
"$$exists": true
},
"txnNumber": {
"$$exists": true
}
}
}
}
]
}
]
},
{
"description": "client bulkWrite with multi: true operations fails after retryable writeConcernError",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorLabels": [
"RetryableWriteError"
],
"writeConcernError": {
"code": 91,
"errmsg": "Replication is being shut down"
}
}
}
}
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"updateMany": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"deleteMany": {
"namespace": "retryable-writes-tests.coll0",
"filter": {
"_id": 3
}
}
}
]
},
"expectError": {
"writeConcernErrors": [
{
"code": 91,
"message": "Replication is being shut down"
}
]
}
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": true
},
{
"delete": 0,
"filter": {
"_id": 3
},
"multi": true
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
]
}
}
}
]
}
]
},
{
"description": "client bulkWrite with retryWrites: false does not retry",
"operations": [
{
"object": "testRunner",
"name": "failPoint",
"arguments": {
"client": "clientRetryWritesFalse",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 189,
"errorLabels": [
"RetryableWriteError"
]
}
}
}
},
{
"object": "clientRetryWritesFalse",
"name": "clientBulkWrite",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-tests.coll0",
"document": {
"_id": 4,
"x": 44
}
}
}
]
},
"expectError": {
"errorCode": 189,
"errorLabelsContain": [
"RetryableWriteError"
]
}
}
],
"expectEvents": [
{
"client": "clientRetryWritesFalse",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 4,
"x": 44
}
}
],
"nsInfo": [
{
"ns": "retryable-writes-tests.coll0"
}
]
}
}
}
]
}
]
}
]
}

216
test/retryable_writes/unified/handshakeError.json
View File

@ -53,6 +53,222 @@
}
],
"tests": [
{
"description": "client.clientBulkWrite succeeds after retryable handshake network error",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"ping",
"saslContinue"
],
"closeConnection": true
}
}
}
},
{
"name": "runCommand",
"object": "database",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
}
},
"expectError": {
"isError": true
}
},
{
"name": "clientBulkWrite",
"object": "client",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-handshake-tests.coll",
"document": {
"_id": 8,
"x": 88
}
}
}
]
}
}
],
"expectEvents": [
{
"client": "client",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client",
"events": [
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-writes-handshake-tests"
}
},
{
"commandFailedEvent": {
"commandName": "ping"
}
},
{
"commandStartedEvent": {
"commandName": "bulkWrite"
}
},
{
"commandSucceededEvent": {
"commandName": "bulkWrite"
}
}
]
}
]
},
{
"description": "client.clientBulkWrite succeeds after retryable handshake server error (ShutdownInProgress)",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"ping",
"saslContinue"
],
"closeConnection": true
}
}
}
},
{
"name": "runCommand",
"object": "database",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
}
},
"expectError": {
"isError": true
}
},
{
"name": "clientBulkWrite",
"object": "client",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "retryable-writes-handshake-tests.coll",
"document": {
"_id": 8,
"x": 88
}
}
}
]
}
}
],
"expectEvents": [
{
"client": "client",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client",
"events": [
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-writes-handshake-tests"
}
},
{
"commandFailedEvent": {
"commandName": "ping"
}
},
{
"commandStartedEvent": {
"commandName": "bulkWrite"
}
},
{
"commandSucceededEvent": {
"commandName": "bulkWrite"
}
}
]
}
]
},
{
"description": "collection.insertOne succeeds after retryable handshake network error",
"operations": [

187
test/server_selection_logging/operation-id.json
View File

@ -47,6 +47,9 @@
}
}
],
"_yamlAnchors": {
"namespace": "logging-tests.server-selection"
},
"tests": [
{
"description": "Successful bulkWrite operation: log messages have operationIds",
@ -224,6 +227,190 @@
]
}
]
},
{
"description": "Successful client bulkWrite operation: log messages have operationIds",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"operations": [
{
"name": "waitForEvent",
"object": "testRunner",
"arguments": {
"client": "client",
"event": {
"topologyDescriptionChangedEvent": {}
},
"count": 2
}
},
{
"name": "clientBulkWrite",
"object": "client",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "logging-tests.server-selection",
"document": {
"x": 1
}
}
}
]
}
}
],
"expectLogMessages": [
{
"client": "client",
"messages": [
{
"level": "debug",
"component": "serverSelection",
"data": {
"message": "Server selection started",
"operationId": {
"$$type": [
"int",
"long"
]
},
"operation": "bulkWrite"
}
},
{
"level": "debug",
"component": "serverSelection",
"data": {
"message": "Server selection succeeded",
"operationId": {
"$$type": [
"int",
"long"
]
},
"operation": "bulkWrite"
}
}
]
}
]
},
{
"description": "Failed client bulkWrite operation: log messages have operationIds",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "failPointClient",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": "alwaysOn",
"data": {
"failCommands": [
"hello",
"ismaster"
],
"appName": "loggingClient",
"closeConnection": true
}
}
}
},
{
"name": "waitForEvent",
"object": "testRunner",
"arguments": {
"client": "client",
"event": {
"serverDescriptionChangedEvent": {
"newDescription": {
"type": "Unknown"
}
}
},
"count": 1
}
},
{
"name": "clientBulkWrite",
"object": "client",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "logging-tests.server-selection",
"document": {
"x": 1
}
}
}
]
},
"expectError": {
"isClientError": true
}
}
],
"expectLogMessages": [
{
"client": "client",
"messages": [
{
"level": "debug",
"component": "serverSelection",
"data": {
"message": "Server selection started",
"operationId": {
"$$type": [
"int",
"long"
]
},
"operation": "bulkWrite"
}
},
{
"level": "debug",
"component": "serverSelection",
"data": {
"message": "Waiting for suitable server to become available",
"operationId": {
"$$type": [
"int",
"long"
]
},
"operation": "bulkWrite"
}
},
{
"level": "debug",
"component": "serverSelection",
"data": {
"message": "Server selection failed",
"operationId": {
"$$type": [
"int",
"long"
]
},
"operation": "bulkWrite"
}
}
]
}
]
}
]
}

571
test/test_client_bulk_write.py Normal file
View File

@ -0,0 +1,571 @@
# Copyright 2024-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test the client bulk write API."""
from __future__ import annotations
import sys
sys.path[0:0] = [""]
from test import IntegrationTest, client_context, unittest
from test.utils import (
OvertCommandListener,
rs_or_single_client,
)
from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts
from pymongo.errors import (
ClientBulkWriteException,
DocumentTooLarge,
InvalidOperation,
NetworkTimeout,
)
from pymongo.monitoring import *
from pymongo.operations import *
from pymongo.write_concern import WriteConcern
_IS_SYNC = True
class TestClientBulkWrite(IntegrationTest):
@client_context.require_version_min(8, 0, 0, -24)
def test_returns_error_if_no_namespace_provided(self):
client = rs_or_single_client()
self.addCleanup(client.close)
models = [InsertOne(document={"a": "b"})]
with self.assertRaises(InvalidOperation) as context:
client.bulk_write(models=models)
self.assertIn(
"MongoClient.bulk_write requires a namespace to be provided for each write operation",
context.exception._message,
)
# https://github.com/mongodb/specifications/tree/master/source/crud/tests
class TestClientBulkWriteCRUD(IntegrationTest):
@client_context.require_version_min(8, 0, 0, -24)
def test_batch_splits_if_num_operations_too_large(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
max_write_batch_size = (client_context.hello)["maxWriteBatchSize"]
models = []
for _ in range(max_write_batch_size + 1):
models.append(InsertOne(namespace="db.coll", document={"a": "b"}))
self.addCleanup(client.db["coll"].drop)
result = client.bulk_write(models=models)
self.assertEqual(result.inserted_count, max_write_batch_size + 1)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
first_event, second_event = bulk_write_events
self.assertEqual(len(first_event.command["ops"]), max_write_batch_size)
self.assertEqual(len(second_event.command["ops"]), 1)
self.assertEqual(first_event.operation_id, second_event.operation_id)
@client_context.require_version_min(8, 0, 0, -24)
def test_batch_splits_if_ops_payload_too_large(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"]
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
models = []
num_models = int(max_message_size_bytes / max_bson_object_size + 1)
b_repeated = "b" * (max_bson_object_size - 500)
for _ in range(num_models):
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
self.addCleanup(client.db["coll"].drop)
result = client.bulk_write(models=models)
self.assertEqual(result.inserted_count, num_models)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
first_event, second_event = bulk_write_events
self.assertEqual(len(first_event.command["ops"]), num_models - 1)
self.assertEqual(len(second_event.command["ops"]), 1)
self.assertEqual(first_event.operation_id, second_event.operation_id)
@client_context.require_version_min(8, 0, 0, -24)
@client_context.require_failCommand_fail_point
def test_collects_write_concern_errors_across_batches(self):
listener = OvertCommandListener()
client = rs_or_single_client(
event_listeners=[listener],
retryWrites=False,
)
self.addCleanup(client.close)
max_write_batch_size = (client_context.hello)["maxWriteBatchSize"]
fail_command = {
"configureFailPoint": "failCommand",
"mode": {"times": 2},
"data": {
"failCommands": ["bulkWrite"],
"writeConcernError": {"code": 91, "errmsg": "Replication is being shut down"},
},
}
with self.fail_point(fail_command):
models = []
for _ in range(max_write_batch_size + 1):
models.append(
InsertOne(
namespace="db.coll",
document={"a": "b"},
)
)
self.addCleanup(client.db["coll"].drop)
with self.assertRaises(ClientBulkWriteException) as context:
client.bulk_write(models=models)
self.assertEqual(len(context.exception.write_concern_errors), 2) # type: ignore[arg-type]
self.assertIsNotNone(context.exception.partial_result)
self.assertEqual(
context.exception.partial_result.inserted_count, max_write_batch_size + 1
)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
@client_context.require_version_min(8, 0, 0, -24)
def test_collects_write_errors_across_batches_unordered(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
collection = client.db["coll"]
self.addCleanup(collection.drop)
collection.drop()
collection.insert_one(document={"_id": 1})
max_write_batch_size = (client_context.hello)["maxWriteBatchSize"]
models = []
for _ in range(max_write_batch_size + 1):
models.append(
InsertOne(
namespace="db.coll",
document={"_id": 1},
)
)
with self.assertRaises(ClientBulkWriteException) as context:
client.bulk_write(models=models, ordered=False)
self.assertEqual(len(context.exception.write_errors), max_write_batch_size + 1) # type: ignore[arg-type]
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
@client_context.require_version_min(8, 0, 0, -24)
def test_collects_write_errors_across_batches_ordered(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
collection = client.db["coll"]
self.addCleanup(collection.drop)
collection.drop()
collection.insert_one(document={"_id": 1})
max_write_batch_size = (client_context.hello)["maxWriteBatchSize"]
models = []
for _ in range(max_write_batch_size + 1):
models.append(
InsertOne(
namespace="db.coll",
document={"_id": 1},
)
)
with self.assertRaises(ClientBulkWriteException) as context:
client.bulk_write(models=models, ordered=True)
self.assertEqual(len(context.exception.write_errors), 1) # type: ignore[arg-type]
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 1)
@client_context.require_version_min(8, 0, 0, -24)
def test_handles_cursor_requiring_getMore(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
collection = client.db["coll"]
self.addCleanup(collection.drop)
collection.drop()
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
models = []
a_repeated = "a" * (max_bson_object_size // 2)
b_repeated = "b" * (max_bson_object_size // 2)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": a_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": b_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
result = client.bulk_write(models=models, verbose_results=True)
self.assertEqual(result.upserted_count, 2)
self.assertEqual(len(result.update_results), 2)
get_more_event = False
for event in listener.started_events:
if event.command_name == "getMore":
get_more_event = True
self.assertTrue(get_more_event)
@client_context.require_version_min(8, 0, 0, -24)
@client_context.require_no_standalone
def test_handles_cursor_requiring_getMore_within_transaction(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
collection = client.db["coll"]
self.addCleanup(collection.drop)
collection.drop()
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
with client.start_session() as session:
session.start_transaction()
models = []
a_repeated = "a" * (max_bson_object_size // 2)
b_repeated = "b" * (max_bson_object_size // 2)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": a_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": b_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
result = client.bulk_write(models=models, session=session, verbose_results=True)
self.assertEqual(result.upserted_count, 2)
self.assertEqual(len(result.update_results), 2)
get_more_event = False
for event in listener.started_events:
if event.command_name == "getMore":
get_more_event = True
self.assertTrue(get_more_event)
@client_context.require_version_min(8, 0, 0, -24)
@client_context.require_failCommand_fail_point
def test_handles_getMore_error(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
collection = client.db["coll"]
self.addCleanup(collection.drop)
collection.drop()
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
fail_command = {
"configureFailPoint": "failCommand",
"mode": {"times": 1},
"data": {"failCommands": ["getMore"], "errorCode": 8},
}
with self.fail_point(fail_command):
models = []
a_repeated = "a" * (max_bson_object_size // 2)
b_repeated = "b" * (max_bson_object_size // 2)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": a_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
models.append(
UpdateOne(
namespace="db.coll",
filter={"_id": b_repeated},
update={"$set": {"x": 1}},
upsert=True,
)
)
with self.assertRaises(ClientBulkWriteException) as context:
client.bulk_write(models=models, verbose_results=True)
self.assertIsNotNone(context.exception.error)
self.assertEqual(context.exception.error["code"], 8)
self.assertIsNotNone(context.exception.partial_result)
self.assertEqual(context.exception.partial_result.upserted_count, 2)
self.assertEqual(len(context.exception.partial_result.update_results), 1)
get_more_event = False
kill_cursors_event = False
for event in listener.started_events:
if event.command_name == "getMore":
get_more_event = True
if event.command_name == "killCursors":
kill_cursors_event = True
self.assertTrue(get_more_event)
self.assertTrue(kill_cursors_event)
@client_context.require_version_min(8, 0, 0, -24)
def test_returns_error_if_unacknowledged_too_large_insert(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
b_repeated = "b" * max_bson_object_size
# Insert document.
models_insert = [InsertOne(namespace="db.coll", document={"a": b_repeated})]
with self.assertRaises(DocumentTooLarge):
client.bulk_write(models=models_insert, write_concern=WriteConcern(w=0))
# Replace document.
models_replace = [ReplaceOne(namespace="db.coll", filter={}, replacement={"a": b_repeated})]
with self.assertRaises(DocumentTooLarge):
client.bulk_write(models=models_replace, write_concern=WriteConcern(w=0))
def _setup_namespace_test_models(self):
max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"]
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
ops_bytes = max_message_size_bytes - 1122
num_models = ops_bytes // max_bson_object_size
remainder_bytes = ops_bytes % max_bson_object_size
models = []
b_repeated = "b" * (max_bson_object_size - 57)
for _ in range(num_models):
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
if remainder_bytes >= 217:
num_models += 1
b_repeated = "b" * (remainder_bytes - 57)
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
return num_models, models
@client_context.require_version_min(8, 0, 0, -24)
def test_no_batch_splits_if_new_namespace_is_not_too_large(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
num_models, models = self._setup_namespace_test_models()
models.append(
InsertOne(
namespace="db.coll",
document={"a": "b"},
)
)
self.addCleanup(client.db["coll"].drop)
# No batch splitting required.
result = client.bulk_write(models=models)
self.assertEqual(result.inserted_count, num_models + 1)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 1)
event = bulk_write_events[0]
self.assertEqual(len(event.command["ops"]), num_models + 1)
self.assertEqual(len(event.command["nsInfo"]), 1)
self.assertEqual(event.command["nsInfo"][0]["ns"], "db.coll")
@client_context.require_version_min(8, 0, 0, -24)
def test_batch_splits_if_new_namespace_is_too_large(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
num_models, models = self._setup_namespace_test_models()
c_repeated = "c" * 200
namespace = f"db.{c_repeated}"
models.append(
InsertOne(
namespace=namespace,
document={"a": "b"},
)
)
self.addCleanup(client.db["coll"].drop)
self.addCleanup(client.db[c_repeated].drop)
# Batch splitting required.
result = client.bulk_write(models=models)
self.assertEqual(result.inserted_count, num_models + 1)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)
first_event, second_event = bulk_write_events
self.assertEqual(len(first_event.command["ops"]), num_models)
self.assertEqual(len(first_event.command["nsInfo"]), 1)
self.assertEqual(first_event.command["nsInfo"][0]["ns"], "db.coll")
self.assertEqual(len(second_event.command["ops"]), 1)
self.assertEqual(len(second_event.command["nsInfo"]), 1)
self.assertEqual(second_event.command["nsInfo"][0]["ns"], namespace)
@client_context.require_version_min(8, 0, 0, -24)
def test_returns_error_if_no_writes_can_be_added_to_ops(self):
client = rs_or_single_client()
self.addCleanup(client.close)
max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"]
# Document too large.
b_repeated = "b" * max_message_size_bytes
models = [InsertOne(namespace="db.coll", document={"a": b_repeated})]
with self.assertRaises(InvalidOperation) as context:
client.bulk_write(models=models)
self.assertIn("cannot do an empty bulk write", context.exception._message)
# Namespace too large.
c_repeated = "c" * max_message_size_bytes
namespace = f"db.{c_repeated}"
models = [InsertOne(namespace=namespace, document={"a": "b"})]
with self.assertRaises(InvalidOperation) as context:
client.bulk_write(models=models)
self.assertIn("cannot do an empty bulk write", context.exception._message)
@client_context.require_version_min(8, 0, 0, -24)
@unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed")
def test_returns_error_if_auto_encryption_configured(self):
opts = AutoEncryptionOpts(
key_vault_namespace="db.coll",
kms_providers={"aws": {"accessKeyId": "foo", "secretAccessKey": "bar"}},
)
client = rs_or_single_client(auto_encryption_opts=opts)
self.addCleanup(client.close)
models = [InsertOne(namespace="db.coll", document={"a": "b"})]
with self.assertRaises(InvalidOperation) as context:
client.bulk_write(models=models)
self.assertIn(
"bulk_write does not currently support automatic encryption", context.exception._message
)
# https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/tests/README.md#11-multi-batch-bulkwrites
class TestClientBulkWriteTimeout(IntegrationTest):
@client_context.require_version_min(8, 0, 0, -24)
@client_context.require_failCommand_fail_point
def test_timeout_in_multi_batch_bulk_write(self):
internal_client = rs_or_single_client(timeoutMS=None)
self.addCleanup(internal_client.close)
collection = internal_client.db["coll"]
self.addCleanup(collection.drop)
collection.drop()
max_bson_object_size = (client_context.hello)["maxBsonObjectSize"]
max_message_size_bytes = (client_context.hello)["maxMessageSizeBytes"]
fail_command = {
"configureFailPoint": "failCommand",
"mode": {"times": 2},
"data": {"failCommands": ["bulkWrite"], "blockConnection": True, "blockTimeMS": 1010},
}
with self.fail_point(fail_command):
models = []
num_models = int(max_message_size_bytes / max_bson_object_size + 1)
b_repeated = "b" * (max_bson_object_size - 500)
for _ in range(num_models):
models.append(
InsertOne(
namespace="db.coll",
document={"a": b_repeated},
)
)
listener = OvertCommandListener()
client = rs_or_single_client(
event_listeners=[listener],
readConcernLevel="majority",
readPreference="primary",
timeoutMS=2000,
w="majority",
)
self.addCleanup(client.close)
with self.assertRaises(ClientBulkWriteException) as context:
client.bulk_write(models=models)
self.assertIsInstance(context.exception.error, NetworkTimeout)
bulk_write_events = []
for event in listener.started_events:
if event.command_name == "bulkWrite":
bulk_write_events.append(event)
self.assertEqual(len(bulk_write_events), 2)

592
test/transactions/unified/client-bulkWrite.json Normal file
View File

@ -0,0 +1,592 @@
{
"description": "client bulkWrite transactions",
"schemaVersion": "1.3",
"runOnRequirements": [
{
"minServerVersion": "8.0",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
]
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "transaction-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
},
{
"session": {
"id": "session0",
"client": "client0"
}
},
{
"client": {
"id": "client_with_wmajority",
"uriOptions": {
"w": "majority"
},
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"session": {
"id": "session_with_wmajority",
"client": "client_with_wmajority"
}
}
],
"_yamlAnchors": {
"namespace": "transaction-tests.coll0"
},
"initialData": [
{
"databaseName": "transaction-tests",
"collectionName": "coll0",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
},
{
"_id": 5,
"x": 55
},
{
"_id": 6,
"x": 66
},
{
"_id": 7,
"x": 77
}
]
}
],
"tests": [
{
"description": "client bulkWrite in a transaction",
"operations": [
{
"object": "session0",
"name": "startTransaction"
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"session": "session0",
"models": [
{
"insertOne": {
"namespace": "transaction-tests.coll0",
"document": {
"_id": 8,
"x": 88
}
}
},
{
"updateOne": {
"namespace": "transaction-tests.coll0",
"filter": {
"_id": 1
},
"update": {
"$inc": {
"x": 1
}
}
}
},
{
"updateMany": {
"namespace": "transaction-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"update": {
"$inc": {
"x": 2
}
}
}
},
{
"replaceOne": {
"namespace": "transaction-tests.coll0",
"filter": {
"_id": 4
},
"replacement": {
"x": 44
},
"upsert": true
}
},
{
"deleteOne": {
"namespace": "transaction-tests.coll0",
"filter": {
"_id": 5
}
}
},
{
"deleteMany": {
"namespace": "transaction-tests.coll0",
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 1,
"matchedCount": 3,
"modifiedCount": 3,
"deletedCount": 3,
"insertResults": {
"0": {
"insertedId": 8
}
},
"updateResults": {
"1": {
"matchedCount": 1,
"modifiedCount": 1,
"upsertedId": {
"$$exists": false
}
},
"2": {
"matchedCount": 2,
"modifiedCount": 2,
"upsertedId": {
"$$exists": false
}
},
"3": {
"matchedCount": 1,
"modifiedCount": 0,
"upsertedId": 4
}
},
"deleteResults": {
"4": {
"deletedCount": 1
},
"5": {
"deletedCount": 2
}
}
}
},
{
"object": "session0",
"name": "commitTransaction"
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"lsid": {
"$$sessionLsid": "session0"
},
"txnNumber": 1,
"startTransaction": true,
"autocommit": false,
"writeConcern": {
"$$exists": false
},
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 8,
"x": 88
}
},
{
"update": 0,
"filter": {
"_id": 1
},
"updateMods": {
"$inc": {
"x": 1
}
},
"multi": false
},
{
"update": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 1
}
},
{
"_id": {
"$lte": 3
}
}
]
},
"updateMods": {
"$inc": {
"x": 2
}
},
"multi": true
},
{
"update": 0,
"filter": {
"_id": 4
},
"updateMods": {
"x": 44
},
"upsert": true,
"multi": false
},
{
"delete": 0,
"filter": {
"_id": 5
},
"multi": false
},
{
"delete": 0,
"filter": {
"$and": [
{
"_id": {
"$gt": 5
}
},
{
"_id": {
"$lte": 7
}
}
]
},
"multi": true
}
],
"nsInfo": [
{
"ns": "transaction-tests.coll0"
}
]
}
}
},
{
"commandStartedEvent": {
"commandName": "commitTransaction",
"databaseName": "admin",
"command": {
"commitTransaction": 1,
"lsid": {
"$$sessionLsid": "session0"
},
"txnNumber": 1,
"startTransaction": {
"$$exists": false
},
"autocommit": false,
"writeConcern": {
"$$exists": false
}
}
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "transaction-tests",
"documents": [
{
"_id": 1,
"x": 12
},
{
"_id": 2,
"x": 24
},
{
"_id": 3,
"x": 35
},
{
"_id": 4,
"x": 44
},
{
"_id": 8,
"x": 88
}
]
}
]
},
{
"description": "client writeConcern ignored for client bulkWrite in transaction",
"operations": [
{
"object": "session_with_wmajority",
"name": "startTransaction",
"arguments": {
"writeConcern": {
"w": 1
}
}
},
{
"object": "client_with_wmajority",
"name": "clientBulkWrite",
"arguments": {
"session": "session_with_wmajority",
"models": [
{
"insertOne": {
"namespace": "transaction-tests.coll0",
"document": {
"_id": 8,
"x": 88
}
}
}
]
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"$$unsetOrMatches": {}
},
"updateResults": {
"$$unsetOrMatches": {}
},
"deleteResults": {
"$$unsetOrMatches": {}
}
}
},
{
"object": "session_with_wmajority",
"name": "commitTransaction"
}
],
"expectEvents": [
{
"client": "client_with_wmajority",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"lsid": {
"$$sessionLsid": "session_with_wmajority"
},
"txnNumber": 1,
"startTransaction": true,
"autocommit": false,
"writeConcern": {
"$$exists": false
},
"bulkWrite": 1,
"errorsOnly": true,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 8,
"x": 88
}
}
],
"nsInfo": [
{
"ns": "transaction-tests.coll0"
}
]
}
}
},
{
"commandStartedEvent": {
"command": {
"commitTransaction": 1,
"lsid": {
"$$sessionLsid": "session_with_wmajority"
},
"txnNumber": {
"$numberLong": "1"
},
"startTransaction": {
"$$exists": false
},
"autocommit": false,
"writeConcern": {
"w": 1
}
},
"commandName": "commitTransaction",
"databaseName": "admin"
}
}
]
}
],
"outcome": [
{
"collectionName": "coll0",
"databaseName": "transaction-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
},
{
"_id": 5,
"x": 55
},
{
"_id": 6,
"x": 66
},
{
"_id": 7,
"x": 77
},
{
"_id": 8,
"x": 88
}
]
}
]
},
{
"description": "client bulkWrite with writeConcern in a transaction causes a transaction error",
"operations": [
{
"object": "session0",
"name": "startTransaction"
},
{
"object": "client0",
"name": "clientBulkWrite",
"arguments": {
"session": "session0",
"writeConcern": {
"w": 1
},
"models": [
{
"insertOne": {
"namespace": "transaction-tests.coll0",
"document": {
"_id": 8,
"x": 88
}
}
}
]
},
"expectError": {
"isClientError": true,
"errorContains": "Cannot set write concern after starting a transaction"
}
}
]
}
]
}

294
test/transactions/unified/mongos-pin-auto.json
View File

@ -2004,6 +2004,104 @@
}
]
},
{
"description": "remain pinned after non-transient Interrupted error on clientBulkWrite bulkWrite",
"operations": [
{
"object": "session0",
"name": "startTransaction"
},
{
"object": "collection0",
"name": "insertOne",
"arguments": {
"session": "session0",
"document": {
"_id": 3
}
},
"expectResult": {
"$$unsetOrMatches": {
"insertedId": {
"$$unsetOrMatches": 3
}
}
}
},
{
"name": "targetedFailPoint",
"object": "testRunner",
"arguments": {
"session": "session0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 11601
}
}
}
},
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"session": "session0",
"models": [
{
"insertOne": {
"namespace": "database0.collection0",
"document": {
"_id": 8,
"x": 88
}
}
}
]
},
"expectError": {
"errorLabelsOmit": [
"TransientTransactionError"
]
}
},
{
"object": "testRunner",
"name": "assertSessionPinned",
"arguments": {
"session": "session0"
}
},
{
"object": "session0",
"name": "abortTransaction"
}
],
"outcome": [
{
"collectionName": "test",
"databaseName": "transaction-tests",
"documents": [
{
"_id": 1
},
{
"_id": 2
}
]
}
],
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
]
},
{
"description": "unpin after transient connection error on insertOne insert",
"operations": [
@ -5175,6 +5273,202 @@
]
}
]
},
{
"description": "unpin after transient connection error on clientBulkWrite bulkWrite",
"operations": [
{
"object": "session0",
"name": "startTransaction"
},
{
"object": "collection0",
"name": "insertOne",
"arguments": {
"session": "session0",
"document": {
"_id": 3
}
},
"expectResult": {
"$$unsetOrMatches": {
"insertedId": {
"$$unsetOrMatches": 3
}
}
}
},
{
"name": "targetedFailPoint",
"object": "testRunner",
"arguments": {
"session": "session0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"closeConnection": true
}
}
}
},
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"session": "session0",
"models": [
{
"insertOne": {
"namespace": "database0.collection0",
"document": {
"_id": 8,
"x": 88
}
}
}
]
},
"expectError": {
"errorLabelsContain": [
"TransientTransactionError"
]
}
},
{
"object": "testRunner",
"name": "assertSessionUnpinned",
"arguments": {
"session": "session0"
}
},
{
"object": "session0",
"name": "abortTransaction"
}
],
"outcome": [
{
"collectionName": "test",
"databaseName": "transaction-tests",
"documents": [
{
"_id": 1
},
{
"_id": 2
}
]
}
],
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
]
},
{
"description": "unpin after transient ShutdownInProgress error on clientBulkWrite bulkWrite",
"operations": [
{
"object": "session0",
"name": "startTransaction"
},
{
"object": "collection0",
"name": "insertOne",
"arguments": {
"session": "session0",
"document": {
"_id": 3
}
},
"expectResult": {
"$$unsetOrMatches": {
"insertedId": {
"$$unsetOrMatches": 3
}
}
}
},
{
"name": "targetedFailPoint",
"object": "testRunner",
"arguments": {
"session": "session0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 1
},
"data": {
"failCommands": [
"bulkWrite"
],
"errorCode": 91
}
}
}
},
{
"name": "clientBulkWrite",
"object": "client0",
"arguments": {
"session": "session0",
"models": [
{
"insertOne": {
"namespace": "database0.collection0",
"document": {
"_id": 8,
"x": 88
}
}
}
]
},
"expectError": {
"errorLabelsContain": [
"TransientTransactionError"
]
}
},
{
"object": "testRunner",
"name": "assertSessionUnpinned",
"arguments": {
"session": "session0"
}
},
{
"object": "session0",
"name": "abortTransaction"
}
],
"outcome": [
{
"collectionName": "test",
"databaseName": "transaction-tests",
"documents": [
{
"_id": 1
},
{
"_id": 2
}
]
}
],
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
]
}
]
}

113
test/unified_format.py
View File

@ -74,6 +74,7 @@ from pymongo import ASCENDING, CursorType, MongoClient, _csot
from pymongo.encryption_options import _HAVE_PYMONGOCRYPT
from pymongo.errors import (
BulkWriteError,
ClientBulkWriteException,
ConfigurationError,
ConnectionFailure,
EncryptionError,
@ -118,10 +119,18 @@ from pymongo.monitoring import (
_ServerEvent,
_ServerHeartbeatEvent,
)
from pymongo.operations import SearchIndexModel
from pymongo.operations import (
DeleteMany,
DeleteOne,
InsertOne,
ReplaceOne,
SearchIndexModel,
UpdateMany,
UpdateOne,
)
from pymongo.read_concern import ReadConcern
from pymongo.read_preferences import ReadPreference
from pymongo.results import BulkWriteResult
from pymongo.results import BulkWriteResult, ClientBulkWriteResult
from pymongo.server_api import ServerApi
from pymongo.server_description import ServerDescription
from pymongo.server_selectors import Selection, writable_server_selector
@ -289,11 +298,61 @@ def parse_bulk_write_result(result):
}
def parse_client_bulk_write_individual(op_type, result):
if op_type == "insert":
return {"insertedId": result.inserted_id}
if op_type == "update":
if result.upserted_id:
return {
"matchedCount": result.matched_count,
"modifiedCount": result.modified_count,
"upsertedId": result.upserted_id,
}
else:
return {
"matchedCount": result.matched_count,
"modifiedCount": result.modified_count,
}
if op_type == "delete":
return {
"deletedCount": result.deleted_count,
}
def parse_client_bulk_write_result(result):
insert_results, update_results, delete_results = {}, {}, {}
if result.has_verbose_results:
for idx, res in result.insert_results.items():
insert_results[str(idx)] = parse_client_bulk_write_individual("insert", res)
for idx, res in result.update_results.items():
update_results[str(idx)] = parse_client_bulk_write_individual("update", res)
for idx, res in result.delete_results.items():
delete_results[str(idx)] = parse_client_bulk_write_individual("delete", res)
return {
"deletedCount": result.deleted_count,
"insertedCount": result.inserted_count,
"matchedCount": result.matched_count,
"modifiedCount": result.modified_count,
"upsertedCount": result.upserted_count,
"insertResults": insert_results,
"updateResults": update_results,
"deleteResults": delete_results,
}
def parse_bulk_write_error_result(error):
write_result = BulkWriteResult(error.details, True)
return parse_bulk_write_result(write_result)
def parse_client_bulk_write_error_result(error):
write_result = error.partial_result
if not write_result:
return None
return parse_client_bulk_write_result(write_result)
class NonLazyCursor:
"""A find cursor proxy that creates the remote cursor when initialized."""
@ -946,6 +1005,8 @@ def coerce_result(opname, result):
return {"acknowledged": False}
if opname == "bulkWrite":
return parse_bulk_write_result(result)
if opname == "clientBulkWrite":
return parse_client_bulk_write_result(result)
if opname == "insertOne":
return {"insertedId": result.inserted_id}
if opname == "insertMany":
@ -974,7 +1035,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
a class attribute ``TEST_SPEC``.
"""
SCHEMA_VERSION = Version.from_string("1.20")
SCHEMA_VERSION = Version.from_string("1.21")
RUN_ON_LOAD_BALANCER = True
RUN_ON_SERVERLESS = True
TEST_SPEC: Any
@ -1151,20 +1212,27 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
expect_result = spec.get("expectResult")
error_response = spec.get("errorResponse")
if error_response:
self.match_evaluator.match_result(error_response, exception.details)
if isinstance(exception, ClientBulkWriteException):
self.match_evaluator.match_result(error_response, exception.error.details)
else:
self.match_evaluator.match_result(error_response, exception.details)
if is_error:
# already satisfied because exception was raised
pass
if is_client_error:
if isinstance(exception, ClientBulkWriteException):
error = exception.error
else:
error = exception
# Connection errors are considered client errors.
if isinstance(exception, ConnectionFailure):
self.assertNotIsInstance(exception, NotPrimaryError)
elif isinstance(exception, (InvalidOperation, ConfigurationError, EncryptionError)):
if isinstance(error, ConnectionFailure):
self.assertNotIsInstance(error, NotPrimaryError)
elif isinstance(error, (InvalidOperation, ConfigurationError, EncryptionError)):
pass
else:
self.assertNotIsInstance(exception, PyMongoError)
self.assertNotIsInstance(error, PyMongoError)
if is_timeout_error:
self.assertIsInstance(exception, PyMongoError)
@ -1175,21 +1243,31 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
if error_contains:
if isinstance(exception, BulkWriteError):
errmsg = str(exception.details).lower()
elif isinstance(exception, ClientBulkWriteException):
errmsg = str(exception.details).lower()
else:
errmsg = str(exception).lower()
self.assertIn(error_contains.lower(), errmsg)
if error_code:
self.assertEqual(error_code, exception.details.get("code"))
if isinstance(exception, ClientBulkWriteException):
self.assertEqual(error_code, exception.error.details.get("code"))
else:
self.assertEqual(error_code, exception.details.get("code"))
if error_code_name:
self.assertEqual(error_code_name, exception.details.get("codeName"))
if isinstance(exception, ClientBulkWriteException):
self.assertEqual(error_code, exception.error.details.get("codeName"))
else:
self.assertEqual(error_code_name, exception.details.get("codeName"))
if error_labels_contain:
if isinstance(exception, ClientBulkWriteException):
error = exception.error
else:
error = exception
labels = [
err_label
for err_label in error_labels_contain
if exception.has_error_label(err_label)
err_label for err_label in error_labels_contain if error.has_error_label(err_label)
]
self.assertEqual(labels, error_labels_contain)
@ -1202,8 +1280,13 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
if isinstance(exception, BulkWriteError):
result = parse_bulk_write_error_result(exception)
self.match_evaluator.match_result(expect_result, result)
elif isinstance(exception, ClientBulkWriteException):
result = parse_client_bulk_write_error_result(exception)
self.match_evaluator.match_result(expect_result, result)
else:
self.fail(f"expectResult can only be specified with {BulkWriteError} exceptions")
self.fail(
f"expectResult can only be specified with {BulkWriteError} or {ClientBulkWriteException} exceptions"
)
return exception
@ -1481,6 +1564,8 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
target_opname = camel_to_snake(opname)
if target_opname == "iterate_once":
target_opname = "try_next"
if target_opname == "client_bulk_write":
target_opname = "bulk_write"
try:
cmd = getattr(target, target_opname)
except AttributeError:

6
test/utils.py
View File

@ -1251,10 +1251,10 @@ def prepare_spec_arguments(spec, arguments, opname, entity_map, with_txn_callbac
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = getattr(ReturnDocument, arguments.pop(arg_name).upper())
elif c2s == "requests":
elif "bulk_write" in opname and (c2s == "requests" or c2s == "models"):
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
for request in arguments[c2s]:
if "name" in request:
# CRUD v2 format
bulk_model = camel_to_upper_camel(request["name"])
@ -1266,7 +1266,7 @@ def prepare_spec_arguments(spec, arguments, opname, entity_map, with_txn_callbac
bulk_class = getattr(operations, camel_to_upper_camel(bulk_model))
bulk_arguments = camel_to_snake_args(spec)
requests.append(bulk_class(**dict(bulk_arguments)))
arguments["requests"] = requests
arguments[c2s] = requests
elif arg_name == "session":
arguments["session"] = entity_map[arguments["session"]]
elif opname == "open_download_stream" and arg_name == "id":

82
test/versioned-api/crud-api-version-1.json
View File

@ -50,7 +50,8 @@
},
"apiDeprecationErrors": true
}
]
],
"namespace": "versioned-api-tests.test"
},
"initialData": [
{
@ -426,6 +427,85 @@
}
]
},
{
"description": "client bulkWrite appends declared API version",
"runOnRequirements": [
{
"minServerVersion": "8.0"
}
],
"operations": [
{
"name": "clientBulkWrite",
"object": "client",
"arguments": {
"models": [
{
"insertOne": {
"namespace": "versioned-api-tests.test",
"document": {
"_id": 6,
"x": 6
}
}
}
],
"verboseResults": true
},
"expectResult": {
"insertedCount": 1,
"upsertedCount": 0,
"matchedCount": 0,
"modifiedCount": 0,
"deletedCount": 0,
"insertResults": {
"0": {
"insertedId": 6
}
},
"updateResults": {},
"deleteResults": {}
}
}
],
"expectEvents": [
{
"client": "client",
"events": [
{
"commandStartedEvent": {
"commandName": "bulkWrite",
"databaseName": "admin",
"command": {
"bulkWrite": 1,
"errorsOnly": false,
"ordered": true,
"ops": [
{
"insert": 0,
"document": {
"_id": 6,
"x": 6
}
}
],
"nsInfo": [
{
"ns": "versioned-api-tests.test"
}
],
"apiVersion": "1",
"apiStrict": {
"$$unsetOrMatches": false
},
"apiDeprecationErrors": true
}
}
}
]
}
]
},
{
"description": "countDocuments appends declared API version",
"operations": [

2
tools/synchro.py
View File

@ -39,6 +39,7 @@ replacements = {
"AsyncDatabaseChangeStream": "DatabaseChangeStream",
"AsyncClusterChangeStream": "ClusterChangeStream",
"_AsyncBulk": "_Bulk",
"_AsyncClientBulk": "_ClientBulk",
"AsyncConnection": "Connection",
"async_command": "command",
"async_receive_message": "receive_message",
@ -151,6 +152,7 @@ converted_tests = [
"pymongo_mocks.py",
"utils_spec_runner.py",
"test_client.py",
"test_client_bulk_write.py",
"test_collection.py",
"test_cursor.py",
"test_database.py",