mongodb/motor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MOTOR-643 Support PyMongo 4.0 (#141)

Browse Source 
Co-authored-by: Shane Harvey <shnhrv@gmail.com>
This commit is contained in:
Steven Silvester 2021-12-23 12:33:43 -06:00 committed by GitHub
parent 66fde7189d
commit 055f5e05ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 195 additions and 691 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
.evergreen/config.yml
View File

@ -473,87 +473,6 @@ tasks:
# Test tasks {{{
- name: "test-3.0-standalone"
tags: ["3.0", "standalone"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.0"
TOPOLOGY: "server"
- func: "run tox"
- name: "test-3.0-replica_set"
tags: ["3.0", "replica_set"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.0"
TOPOLOGY: "replica_set"
- func: "run tox"
- name: "test-3.0-sharded_cluster"
tags: ["3.0", "sharded_cluster"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.0"
TOPOLOGY: "sharded_cluster"
- func: "run tox"
- name: "test-3.2-standalone"
tags: ["3.2", "standalone"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.2"
TOPOLOGY: "server"
- func: "run tox"
- name: "test-3.2-replica_set"
tags: ["3.2", "replica_set"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.2"
TOPOLOGY: "replica_set"
- func: "run tox"
- name: "test-3.2-sharded_cluster"
tags: ["3.2", "sharded_cluster"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.2"
TOPOLOGY: "sharded_cluster"
- func: "run tox"
- name: "test-3.4-standalone"
tags: ["3.4", "standalone"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.4"
TOPOLOGY: "server"
- func: "run tox"
- name: "test-3.4-replica_set"
tags: ["3.4", "replica_set"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.4"
TOPOLOGY: "replica_set"
- func: "run tox"
- name: "test-3.4-sharded_cluster"
tags: ["3.4", "sharded_cluster"]
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "3.4"
TOPOLOGY: "sharded_cluster"
- func: "run tox"
- name: "test-3.6-standalone"
tags: ["3.6", "standalone"]
commands:
@ -844,9 +763,9 @@ axes:
variables:
TOX_ENV: "asyncio-py39"
PYTHON_BINARY: "/opt/python/3.9/bin/python3"
- id: "py3-pymongo-v3.12"
- id: "py3-pymongo-latest"
variables:
TOX_ENV: "py3-pymongo-v3.12"
TOX_ENV: "py3-pymongo-latest"
# Use 3.6 for now until 3.7 is on all Evergreen distros.
PYTHON_BINARY: "/opt/python/3.6/bin/python3"
- id: "synchro37"
@ -932,16 +851,6 @@ axes:
buildvariants:
# Test MongoDB 3.0 and Python only up to 3.6 on RHEL.
- matrix_name: "test-mongodb-3.0-rhel"
display_name: "${os}-${tox-env}-${ssl}"
matrix_spec:
os: "rhel"
tox-env: ["tornado5-py36", "py3-pymongo-v3.12"]
ssl: "*"
tasks:
- ".3.0"
# Main test matrix.
- matrix_name: "main"
display_name: "${os}-${tox-env}-${ssl}"
@ -962,8 +871,6 @@ buildvariants:
- ".4.2"
- ".4.0"
- ".3.6"
- ".3.4"
- ".3.2"
- matrix_name: "test-ubuntu"
display_name: "${os}-${tox-env-ubuntu}-${ssl}"
@ -989,8 +896,6 @@ buildvariants:
- ".4.2"
- ".4.0"
- ".3.6"
- ".3.4"
- ".3.2"
- matrix_name: "test-macos"
display_name: "${os}-${tox-env-osx}-${ssl}"

1
.gitignore vendored
View File

@ -12,3 +12,4 @@ setup.cfg
doc/_build/
.idea/
xunit-results
.eggs

6
doc/api-asyncio/asyncio_motor_collection.rst
View File

@ -71,8 +71,7 @@
- `partialFilterExpression`: A document that specifies a filter for
a partial index.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
See the MongoDB documentation for a full list of supported options by
server version.
@ -84,8 +83,7 @@
.. note:: `partialFilterExpression` requires server version **>= 3.2**
.. note:: The :attr:`~pymongo.collection.Collection.write_concern` of
this collection is automatically applied to this operation when using
MongoDB >= 3.4.
this collection is automatically applied to this operation.
:Parameters:
- `keys`: a single key or a list of (key, direction)

6
doc/api-tornado/motor_collection.rst
View File

@ -71,8 +71,7 @@
- `partialFilterExpression`: A document that specifies a filter for
a partial index.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
See the MongoDB documentation for a full list of supported options by
server version.
@ -84,8 +83,7 @@
.. note:: `partialFilterExpression` requires server version **>= 3.2**
.. note:: The :attr:`~pymongo.collection.Collection.write_concern` of
this collection is automatically applied to this operation when using
MongoDB >= 3.4.
this collection is automatically applied to this operation.
:Parameters:
- `keys`: a single key or a list of (key, direction)

2
doc/tutorial-asyncio.rst
View File

@ -36,7 +36,7 @@ Tutorial Prerequisites
----------------------
You can learn about MongoDB with the `MongoDB Tutorial`_ before you learn Motor.
Using Python 3.4 or later, do::
Using Python 3.5 or later, do::
$ python3 -m pip install motor

15
motor/aiohttp/__init__.py
View File

@ -26,6 +26,8 @@ import aiohttp.web
import gridfs
from motor.motor_asyncio import (AsyncIOMotorDatabase,
AsyncIOMotorGridFSBucket)
from motor.motor_gridfs import _hash_gridout
def get_gridfs_file(bucket, filename, request):
@ -190,7 +192,11 @@ class AIOHTTPGridFS:
raise aiohttp.web.HTTPNotFound(text=request.path)
resp = aiohttp.web.StreamResponse()
self._set_standard_headers(request.path, resp, gridout)
# Get the hash for the GridFS file.
checksum = _hash_gridout(gridout)
self._set_standard_headers(request.path, resp, gridout, checksum)
# Overridable method set_extra_headers.
self._set_extra_headers(resp, gridout)
@ -209,7 +215,7 @@ class AIOHTTPGridFS:
# Same for Etag
etag = request.headers.get("If-None-Match")
if etag is not None and etag.strip('"') == gridout.md5:
if etag is not None and etag.strip('"') == checksum:
resp.set_status(304)
return resp
@ -225,7 +231,7 @@ class AIOHTTPGridFS:
written += len(chunk)
return resp
def _set_standard_headers(self, path, resp, gridout):
def _set_standard_headers(self, path, resp, gridout, checksum):
resp.last_modified = gridout.upload_date
content_type = gridout.content_type
if content_type is None:
@ -234,8 +240,7 @@ class AIOHTTPGridFS:
if content_type:
resp.content_type = content_type
# MD5 is calculated on the MongoDB server when GridFS file is created.
resp.headers["Etag"] = '"%s"' % gridout.md5
resp.headers["Etag"] = '"%s"' % checksum
# Overridable method get_cache_time.
cache_time = self._get_cache_time(path,

42
motor/core.py
View File

@ -16,6 +16,7 @@
import functools
import sys
import time
import warnings
import pymongo
@ -24,8 +25,6 @@ import pymongo.common
import pymongo.database
import pymongo.errors
import pymongo.mongo_client
import pymongo.mongo_replica_set_client
import pymongo.monotonic
from pymongo.change_stream import ChangeStream
from pymongo.client_session import ClientSession
@ -66,7 +65,7 @@ _WITH_TRANSACTION_RETRY_TIME_LIMIT = 120
def _within_time_limit(start_time):
"""Are we within the with_transaction retry limit?"""
return (pymongo.monotonic.time() - start_time <
return (time.monotonic() - start_time <
_WITH_TRANSACTION_RETRY_TIME_LIMIT)
@ -106,8 +105,7 @@ class AgnosticClient(AgnosticBaseProperties):
close = DelegateMethod()
__hash__ = DelegateMethod()
drop_database = AsyncCommand().unwrap('MotorDatabase')
event_listeners = ReadOnlyProperty()
fsync = AsyncCommand(doc=fsync_doc)
options = ReadOnlyProperty()
get_database = DelegateMethod(doc=get_database_doc).wrap(Database)
get_default_database = DelegateMethod(doc=get_default_database_doc).wrap(Database)
HOST = ReadOnlyProperty()
@ -115,25 +113,14 @@ class AgnosticClient(AgnosticBaseProperties):
is_primary = ReadOnlyProperty()
list_databases = AsyncRead().wrap(CommandCursor)
list_database_names = AsyncRead()
local_threshold_ms = ReadOnlyProperty()
max_bson_size = ReadOnlyProperty()
max_idle_time_ms = ReadOnlyProperty()
max_message_size = ReadOnlyProperty()
max_pool_size = ReadOnlyProperty()
max_write_batch_size = ReadOnlyProperty()
min_pool_size = ReadOnlyProperty()
nodes = ReadOnlyProperty()
PORT = ReadOnlyProperty()
primary = ReadOnlyProperty()
read_concern = ReadOnlyProperty()
retry_reads = ReadOnlyProperty()
retry_writes = ReadOnlyProperty()
secondaries = ReadOnlyProperty()
server_info = AsyncRead()
server_selection_timeout = ReadOnlyProperty()
topology_description = ReadOnlyProperty()
start_session = AsyncCommand(doc=start_session_doc).wrap(ClientSession)
unlock = AsyncCommand(doc=unlock_doc)
def __init__(self, *args, **kwargs):
"""Create a new connection to a single MongoDB instance at *host:port*.
@ -405,7 +392,7 @@ class AgnosticClientSession(AgnosticBase):
.. versionadded:: 2.1
"""
start_time = pymongo.monotonic.time()
start_time = time.monotonic()
while True:
async with self.start_transaction(
read_concern, write_concern, read_preference,
@ -495,29 +482,21 @@ class AgnosticDatabase(AgnosticBaseProperties):
__hash__ = DelegateMethod()
command = AsyncCommand(doc=cmd_doc)
create_collection = AsyncCommand().wrap(Collection)
current_op = AsyncRead(doc=current_op_doc)
dereference = AsyncRead()
drop_collection = AsyncCommand().unwrap('MotorCollection')
get_collection = DelegateMethod().wrap(Collection)
list_collection_names = AsyncRead(doc=list_collection_names_doc)
list_collections = AsyncRead()
name = ReadOnlyProperty()
profiling_info = AsyncRead(doc=profiling_info_doc)
profiling_level = AsyncRead(doc=profiling_level_doc)
set_profiling_level = AsyncCommand(doc=set_profiling_level_doc)
validate_collection = AsyncRead().unwrap('MotorCollection')
with_options = DelegateMethod().wrap(Database)
incoming_manipulators = ReadOnlyProperty()
incoming_copying_manipulators = ReadOnlyProperty()
outgoing_manipulators = ReadOnlyProperty()
outgoing_copying_manipulators = ReadOnlyProperty()
_async_aggregate = AsyncRead(attr_name='aggregate')
def __init__(self, client, name, **kwargs):
self._client = client
delegate = kwargs.get('_delegate') or Database(
_delegate = kwargs.get('_delegate')
delegate = _delegate if _delegate is not None else Database(
client.delegate, name, **kwargs)
super().__init__(delegate)
@ -714,13 +693,10 @@ class AgnosticCollection(AgnosticBaseProperties):
find_one_and_update = AsyncCommand(doc=find_one_and_update_doc)
full_name = ReadOnlyProperty()
index_information = AsyncRead(doc=index_information_doc)
inline_map_reduce = AsyncRead()
insert_many = AsyncWrite(doc=insert_many_doc)
insert_one = AsyncCommand(doc=insert_one_doc)
map_reduce = AsyncCommand(doc=mr_doc).wrap(Collection)
name = ReadOnlyProperty()
options = AsyncRead()
reindex = AsyncCommand(doc=reindex_doc)
rename = AsyncCommand()
replace_one = AsyncCommand(doc=replace_one_doc)
update_many = AsyncCommand(doc=update_many_doc)
@ -741,7 +717,7 @@ class AgnosticCollection(AgnosticBaseProperties):
raise TypeError("First argument to MotorCollection must be "
"MotorDatabase, not %r" % database)
delegate = _delegate or Collection(
delegate = _delegate if _delegate is not None else Collection(
database.delegate, name, codec_options=codec_options,
read_preference=read_preference, write_concern=write_concern,
read_concern=read_concern)
@ -1419,7 +1395,6 @@ class AgnosticBaseCursor(AgnosticBase):
if future.done():
return
collection = self.collection
fix_outgoing = collection.database.delegate._fix_outgoing
if length is None:
n = result
@ -1427,8 +1402,7 @@ class AgnosticBaseCursor(AgnosticBase):
n = min(length - len(the_list), result)
for _ in range(n):
the_list.append(fix_outgoing(self._data().popleft(),
collection))
the_list.append(self._data().popleft())
reached_length = (length is not None and len(the_list) >= length)
if reached_length or not self.alive:

233
motor/docstrings.py
View File

@ -92,80 +92,6 @@ based only on the URI in a configuration file.
Removed this method.
"""
fsync_doc = """**DEPRECATED**: Flush all pending writes to datafiles.
Optional parameters can be passed as keyword arguments:
- `lock`: If True lock the server to disallow writes.
- `async`: If True don't block while synchronizing.
- `session` (optional): a
:class:`~pymongo.client_session.ClientSession`, created with
:meth:`~MotorClient.start_session`.
.. note:: Starting with Python 3.7 `async` is a reserved keyword.
The async option to the fsync command can be passed using a
dictionary instead::
options = {'async': True}
await client.fsync(**options)
Deprecated. Run the `fsync command`_ directly with
:meth:`~motor.motor_tornado.MotorDatabase.command` instead. For example::
await client.admin.command('fsync', lock=True)
.. versionchanged:: 2.2
Deprecated.
.. versionchanged:: 1.2
Added ``session`` parameter.
.. warning:: `async` and `lock` can not be used together.
.. warning:: MongoDB does not support the `async` option
on Windows and will raise an exception on that
platform.
.. _fsync command: https://docs.mongodb.com/manual/reference/command/fsync/
"""
current_op_doc = """
**DEPRECATED**: Get information on operations currently running.
Starting with MongoDB 3.6 this helper is obsolete. The functionality
provided by this helper is available in MongoDB 3.6+ using the
`$currentOp aggregation pipeline stage`_, which can be used with
:meth:`aggregate`. Note that, while this helper can only return
a single document limited to a 16MB result, :meth:`aggregate`
returns a cursor avoiding that limitation.
Users of MongoDB versions older than 3.6 can use the `currentOp command`_
directly::
# MongoDB 3.2 and 3.4
await client.admin.command("currentOp")
Or query the "inprog" virtual collection::
# MongoDB 2.6 and 3.0
await client.admin["$cmd.sys.inprog"].find_one()
:Parameters:
- `include_all` (optional): if ``True`` also list currently
idle operations in the result
- `session` (optional): a
:class:`~pymongo.client_session.ClientSession`, created with
:meth:`~MotorClient.start_session`.
.. versionchanged:: 2.1
Deprecated, use :meth:`aggregate` instead.
.. versionchanged:: 1.2
Added session parameter.
.. _$currentOp aggregation pipeline stage: https://docs.mongodb.com/manual/reference/operator/aggregation/currentOp/
.. _currentOp command: https://docs.mongodb.com/manual/reference/command/currentOp/
"""
list_collection_names_doc = """
Get a list of all the collection names in this database.
@ -287,8 +213,7 @@ This prints::
command (like maxTimeMS) can be passed as keyword arguments.
The :attr:`~pymongo.collection.Collection.write_concern` of
this collection is automatically applied to this operation when using
MongoDB >= 3.4.
this collection is automatically applied to this operation.
.. versionchanged:: 1.2
Added session parameter.
@ -362,8 +287,7 @@ This deletes all matching documents and prints "3".
:Parameters:
- `filter`: A query that matches the documents to delete.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
- `hint` (optional): An index used to support the query predicate specified
either by its string name, or in the same format as passed to
:meth:`~MotorDatabase.create_index` (e.g. ``[('field', ASCENDING)]``).
@ -394,8 +318,7 @@ This deletes one matching document and prints "1".
:Parameters:
- `filter`: A query that matches the document to delete.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
- `hint` (optional): An index used to support the query predicate specified
either by its string name, or in the same format as passed to
:meth:`~MotorDatabase.create_index` (e.g. ``[('field', ASCENDING)]``).
@ -903,8 +826,7 @@ This prints::
write to opt-out of document level validation. Default is
``False``.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
- `hint` (optional): An index to use to support the query
predicate specified either by its string name, or in the same
format as passed to
@ -962,8 +884,7 @@ This prints::
write to opt-out of document level validation. Default is
``False``.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
- `array_filters` (optional): A list of filters specifying which
array elements an update should apply. Requires MongoDB 3.6+.
- `hint` (optional): An index to use to support the query
@ -1023,8 +944,7 @@ This prints::
write to opt-out of document level validation. Default is
``False``.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`. This option is only supported
on MongoDB 3.4 and above.
:class:`~pymongo.collation.Collation`.
- `array_filters` (optional): A list of filters specifying which
array elements an update should apply. Requires MongoDB 3.6+.
- `hint` (optional): An index to use to support the query
@ -1193,53 +1113,6 @@ started it.
.. versionadded:: 1.2
"""
unlock_doc = """**DEPRECATED**: Unlock a previously locked server.
:Parameters:
- `session` (optional): a
:class:`~motor.motor_tornado.MotorClientSession`.
Deprecated. Users of MongoDB version 3.2 or newer can run the
`fsyncUnlock command`_ directly with
:meth:`~motor.motor_tornado.MotorDatabase.command`::
await motor_client.admin.command('fsyncUnlock')
Users of MongoDB version 3.0 can query the "unlock" virtual
collection::
await motor_client.admin["$cmd.sys.unlock"].find_one()
.. versionchanged:: 2.2
Deprecated.
.. _fsyncUnlock command: https://docs.mongodb.com/manual/reference/command/fsyncUnlock/
"""
reindex_doc = """**DEPRECATED**: Rebuild all indexes on this collection.
Deprecated. Use :meth:`~motor.motor_tornado.MotorDatabase.command`
to run the ``reIndex`` command directly instead::
await db.command({"reIndex": "<collection_name>"})
.. note:: Starting in MongoDB 4.6, the `reIndex` command can only be
run when connected to a standalone mongod.
:Parameters:
- `session` (optional): a
:class:`~motor.motor_tornado.MotorClientSession`.
- `**kwargs` (optional): optional arguments to the reIndex
command (like maxTimeMS) can be passed as keyword arguments.
.. warning:: reindex blocks all other operations (indexes
are built in the foreground) and will be slow for large
collections.
.. versionchanged:: 2.2
Deprecated.
"""
where_doc = """Adds a `$where`_ clause to this query.
The `code` argument must be an instance of :class:`str`
@ -1292,97 +1165,3 @@ Note that using this class in a with-statement will automatically call
encrypted = await client_encryption.encrypt(value, ...)
decrypted = await client_encryption.decrypt(encrypted)
"""
profiling_info_doc ="""**DEPRECATED**: Returns a list containing current profiling
information.
Starting with Motor 2.5, this helper is obsolete. Instead, users
can view the database profiler output by running
:meth:`~motor.motor_asyncio.AsyncIOMotorCollection.find` against the
``system.profile`` collection as detailed in the `profiler output`_
documentation::
profiling_info = await db["system.profile"].find().to_list()
:Parameters:
- `session` (optional): a
:class:`~motor.motor_asyncio.AsyncIOMotorClientSession`.
.. versionchanged:: 2.5
Deprecated.
.. mongodoc:: profiling
.. _profiler output: https://docs.mongodb.com/manual/reference/database-profiler/
"""
profiling_level_doc = """**DEPRECATED**: Get the database's current profiling level.
Starting with Motor 2.5, this helper is obsolete. Instead, users
can run the `profile command`_, using the :meth:`command`
helper to get the current profiler level. Running the
`profile command`_ with the level set to ``-1`` returns the current
profiler information without changing it::
res = await db.command("profile", -1)
profiling_level = res["was"]
The format of ``res`` depends on the version of MongoDB in use.
Returns one of (:data:`~pymongo.OFF`,
:data:`~pymongo.SLOW_ONLY`, :data:`~pymongo.ALL`).
:Parameters:
- `session` (optional): a
:class:`~motor.motor_asyncio.AsyncIOMotorClientSession`.
.. versionchanged:: 2.5
Deprecated.
.. mongodoc:: profiling
.. _profile command: https://docs.mongodb.com/manual/reference/command/profile/
"""
set_profiling_level_doc = """**DEPRECATED**: Set the database's profiling level.
Starting with Motor 2.5, this helper is obsolete. Instead, users
can directly run the `profile command`_, using the :meth:`command`
helper, e.g.::
res = await db.command("profile", 2, filter={"op": "query"})
:Parameters:
- `level`: Specifies a profiling level, see list of possible values
below.
- `slow_ms`: Optionally modify the threshold for the profile to
consider a query or operation. Even if the profiler is off queries
slower than the `slow_ms` level will get written to the logs.
- `session` (optional): a
:class:`~motor.motor_asyncio.AsyncIOMotorClientSession`.
- `sample_rate` (optional): The fraction of slow operations that
should be profiled or logged expressed as a float between 0 and 1.
- `filter` (optional): A filter expression that controls which
operations are profiled and logged.
Possible `level` values:
+----------------------------+------------------------------------+
| Level | Setting |
+============================+====================================+
| :data:`~pymongo.OFF` | Off. No profiling. |
+----------------------------+------------------------------------+
| :data:`~pymongo.SLOW_ONLY` | On. Only includes slow operations. |
+----------------------------+------------------------------------+
| :data:`~pymongo.ALL` | On. Includes all operations. |
+----------------------------+------------------------------------+
Raises :class:`ValueError` if level is not one of
(:data:`~pymongo.OFF`, :data:`~pymongo.SLOW_ONLY`,
:data:`~pymongo.ALL`).
.. versionchanged:: 2.5
Added the ``sample_rate`` and ``filter`` parameters.
Deprecated.
.. mongodoc:: profiling
.. _profile command: https://docs.mongodb.com/manual/reference/command/profile/
"""

2
motor/metaprogramming.py
View File

@ -178,7 +178,7 @@ class AsyncRead(Async):
class AsyncWrite(Async):
def __init__(self, attr_name=None, doc=None):
"""A descriptor that wraps a PyMongo write method like update() that
"""A descriptor that wraps a PyMongo write method like update_one() that
accepts getLastError options and returns a Future.
"""
Async.__init__(self, attr_name=attr_name, doc=doc)

37
motor/motor_gridfs.py
View File

@ -14,6 +14,7 @@
"""GridFS implementation for Motor, an asynchronous driver for MongoDB."""
import hashlib
import warnings
import gridfs
@ -93,7 +94,6 @@ class AgnosticGridOut(object):
content_type = MotorGridOutProperty()
filename = MotorGridOutProperty()
length = MotorGridOutProperty()
md5 = MotorGridOutProperty()
metadata = MotorGridOutProperty()
name = MotorGridOutProperty()
read = AsyncRead()
@ -217,7 +217,6 @@ class AgnosticGridIn(object):
content_type = ReadOnlyProperty()
filename = ReadOnlyProperty()
length = ReadOnlyProperty()
md5 = ReadOnlyProperty()
name = ReadOnlyProperty()
read = DelegateMethod()
readable = DelegateMethod()
@ -242,7 +241,7 @@ Metadata set on the file appears as attributes on a
""")
def __init__(self, root_collection, delegate=None, session=None,
disable_md5=False, **kwargs):
**kwargs):
"""
Class to write data to GridFS. Application developers should not
generally need to instantiate this class - see
@ -277,12 +276,11 @@ Metadata set on the file appears as attributes on a
- `session` (optional): a
:class:`~pymongo.client_session.ClientSession` to use for all
commands
- `disable_md5` (optional): When True, an MD5 checksum will not be
computed for the uploaded file. Useful in environments where
MD5 cannot be used for regulatory or other reasons. Defaults to
False.
- `**kwargs` (optional): file level options (see above)
.. versionchanged:: 3.0
Removed support for the `disable_md5` parameter (to match the
GridIn class in PyMongo).
.. versionchanged:: 0.2
``open`` method removed, no longer needed.
"""
@ -299,7 +297,6 @@ Metadata set on the file appears as attributes on a
self.delegate = delegate or self.__delegate_class__(
root_collection.delegate,
session=session,
disable_md5=disable_md5,
**kwargs)
# Support "async with bucket.open_upload_stream() as f:"
@ -328,7 +325,7 @@ class AgnosticGridFSBucket(object):
upload_from_stream = AsyncCommand()
upload_from_stream_with_id = AsyncCommand()
def __init__(self, database, bucket_name="fs", disable_md5=False,
def __init__(self, database, bucket_name="fs",
chunk_size_bytes=DEFAULT_CHUNK_SIZE, write_concern=None,
read_preference=None, collection=None):
"""Create a handle to a GridFS bucket.
@ -350,12 +347,12 @@ class AgnosticGridFSBucket(object):
(the default) db.write_concern is used.
- `read_preference` (optional): The read preference to use. If
``None`` (the default) db.read_preference is used.
- `disable_md5` (optional): When True, MD5 checksums will not be
computed for uploaded files. Useful in environments where MD5
cannot be used for regulatory or other reasons. Defaults to False.
- `collection` (optional): Deprecated, an alias for `bucket_name`
that exists solely to provide backwards compatibility.
.. versionchanged:: 3.0
Removed support for the `disable_md5` parameter (to match the
GridFSBucket class in PyMongo).
.. versionchanged:: 2.1
Added support for the `bucket_name`, `chunk_size_bytes`,
`write_concern`, and `read_preference` parameters.
@ -390,8 +387,7 @@ class AgnosticGridFSBucket(object):
bucket_name,
chunk_size_bytes=chunk_size_bytes,
write_concern=write_concern,
read_preference=read_preference,
disable_md5=disable_md5)
read_preference=read_preference)
def get_io_loop(self):
return self.io_loop
@ -479,3 +475,16 @@ class AgnosticGridFSBucket(object):
AgnosticGridOutCursor, self._framework, self.__module__)
return grid_out_cursor(cursor, self.collection)
def _hash_gridout(gridout):
"""Compute the effective hash of a GridOut object for use with an Etag header.
Create a FIPS-compliant Etag HTTP header hash using sha256
We use the _id + length + upload_date as a proxy for
uniqueness to avoid reading the entire file.
"""
grid_hash = hashlib.sha256(str(gridout._id).encode('utf8'))
grid_hash.update(str(gridout.length).encode('utf8'))
grid_hash.update(str(gridout.upload_date).encode('utf8'))
return grid_hash.hexdigest()

16
motor/web.py
View File

@ -16,6 +16,7 @@
import datetime
import email.utils
import hashlib
import mimetypes
import time
@ -24,15 +25,16 @@ from tornado import gen
import gridfs
import motor
from motor.motor_gridfs import _hash_gridout
# TODO: this class is not a drop-in replacement for StaticFileHandler.
# StaticFileHandler provides class method make_static_url, which appends
# an MD5 of the static file's contents. Templates thus can do
# an checksum of the static file's contents. Templates thus can do
# {{ static_url('image.png') }} and get "/static/image.png?v=1234abcdef",
# which is cached forever. Problem is, it calculates the MD5 synchronously.
# which is cached forever. Problem is, it calculates the checksum synchronously.
# Two options: keep a synchronous GridFS available to get each grid file's
# MD5 synchronously for every static_url call, or find some other idiom.
# checksum synchronously for every static_url call, or find some other idiom.
class GridFSHandler(tornado.web.RequestHandler):
@ -103,8 +105,10 @@ class GridFSHandler(tornado.web.RequestHandler):
modified = gridout.upload_date.replace(microsecond=0)
self.set_header("Last-Modified", modified)
# MD5 is calculated on the MongoDB server when GridFS file is created
self.set_header("Etag", '"%s"' % gridout.md5)
# Get the hash for the GridFS file.
checksum = _hash_gridout(gridout)
self.set_header("Etag", '"%s"' % checksum)
mime_type = gridout.content_type
@ -145,7 +149,7 @@ class GridFSHandler(tornado.web.RequestHandler):
# Same for Etag
etag = self.request.headers.get("If-None-Match")
if etag is not None and etag.strip('"') == gridout.md5:
if etag is not None and etag.strip('"') == checksum:
self.set_status(304)
return

2
setup.py
View File

@ -31,7 +31,7 @@ description = 'Non-blocking MongoDB driver for Tornado or asyncio'
with open("README.rst") as readme:
long_description = readme.read()
pymongo_ver = ">=3.12,<4"
pymongo_ver = ">=3.12,<5"
install_requires = ["pymongo" + pymongo_ver]

24
synchro/__init__.py
View File

@ -20,6 +20,7 @@ DO NOT USE THIS MODULE.
"""
import inspect
import time
import unittest
from tornado.ioloop import IOLoop
@ -44,13 +45,10 @@ from pymongo import (collation,
change_stream,
encryption_options,
errors,
monotonic,
operations,
server_selectors,
server_type,
son_manipulator,
saslprep,
ssl_match_hostname,
ssl_support,
write_concern)
from pymongo.auth import _build_credentials_tuple
@ -74,7 +72,6 @@ from pymongo.message import (_COMMAND_OVERHEAD,
from pymongo.monitor import *
from pymongo.monitoring import *
from pymongo.monitoring import _LISTENERS, _Listeners, _SENSITIVE_COMMANDS
from pymongo.monotonic import time
from pymongo.ocsp_cache import _OCSPCache
from pymongo.operations import *
from pymongo.pool import *
@ -91,7 +88,6 @@ from pymongo.server_selectors import *
from pymongo.settings import *
from pymongo.saslprep import *
from pymongo.ssl_support import *
from pymongo.son_manipulator import *
from pymongo.topology import *
from pymongo.topology_description import *
from pymongo.uri_parser import *
@ -126,7 +122,7 @@ def wrap_synchro(fn):
motor_obj = fn(*args, **kwargs)
# Not all Motor classes appear here, only those we need to return
# from methods like map_reduce() or create_collection()
# from methods like create_collection()
if isinstance(motor_obj, motor.MotorCollection):
client = MongoClient(delegate=motor_obj.database.client)
database = Database(client, motor_obj.database.name)
@ -345,7 +341,6 @@ class MongoClient(Synchro):
get_database = WrapOutgoing()
max_pool_size = SynchroProperty()
max_write_batch_size = SynchroProperty()
start_session = Sync()
watch = WrapOutgoing()
@ -361,12 +356,6 @@ class MongoClient(Synchro):
if not self.delegate:
self.delegate = self.__delegate_class__(host, port, *args, **kwargs)
@property
def is_locked(self):
# # MotorClient doesn't support the is_locked property.
# # Use the property directly from the underlying MongoClient.
return self.delegate.delegate.is_locked
# PyMongo expects this to return a real MongoClient, unwrap it.
def _duplicate(self, **kwargs):
client = self.delegate._duplicate(**kwargs)
@ -508,12 +497,6 @@ class Collection(Synchro):
return Collection(self.database, fullname,
delegate=self.delegate[name])
def count(self, *args, **kwargs):
raise unittest.SkipTest('count() is not supported in Motor')
def group(self, *args, **kwargs):
raise unittest.SkipTest('group() is not supported in Motor')
class ChangeStream(Synchro):
__delegate_class__ = motor.motor_tornado.MotorChangeStream
@ -707,7 +690,6 @@ class GridOut(Synchro):
content_type = SynchroGridOutProperty('content_type')
filename = SynchroGridOutProperty('filename')
length = SynchroGridOutProperty('length')
md5 = SynchroGridOutProperty('md5')
metadata = SynchroGridOutProperty('metadata')
name = SynchroGridOutProperty('name')
upload_date = SynchroGridOutProperty('upload_date')
@ -738,7 +720,7 @@ class GridOut(Synchro):
# to make PyMongo's assertRaises tests pass.
if key in (
"_id", "name", "content_type", "length", "chunk_size",
"upload_date", "aliases", "metadata", "md5"):
"upload_date", "aliases", "metadata"):
raise AttributeError()
super().__setattr__(key, value)

13
synchro/synchrotest.py
View File

@ -41,7 +41,6 @@ excluded_modules = [
'test.test_threads',
'test.test_pooling',
'test.test_legacy_api',
'test.test_monotonic',
'test.test_saslprep',
# Complex PyMongo-specific mocking.
@ -55,7 +54,6 @@ excluded_modules = [
# Deprecated in PyMongo, removed in Motor 2.0.
'test.test_gridfs',
'test.test_son_manipulator',
]
@ -82,7 +80,6 @@ excluded_tests = [
# Can't do MotorCollection(name, create=True), Motor constructors do no I/O.
'TestCollection.test_create',
'TestCollection.test_reindex',
# Motor doesn't support PyMongo's syntax, db.system_js['my_func'] = "code",
# users should just use system.js as a regular collection.
@ -162,20 +159,11 @@ excluded_tests = [
'TestCollection.test_parallel_scan',
'TestCollection.test_parallel_scan_max_time_ms',
'TestCollection.test_write_error_text_handling',
'TestCommandMonitoring.test_legacy_insert_many',
'TestCommandMonitoring.test_legacy_writes',
'TestClient.test_database_names',
'TestCollectionWCustomType.test_find_and_modify_w_custom_type_decoder',
# Tests that use "count", deprecated in PyMongo, removed in Motor 2.0.
'*.test_read_count_Deprecated_count_with_a_filter',
'*.test_read_count_Deprecated_count_without_a_filter',
'TestBinary.test_uuid_queries',
'TestCollection.test_count',
'TestCursor.test_comment',
'TestCursor.test_count',
'TestCursor.test_count_with_fields',
'TestCursor.test_count_with_hint',
'TestCursor.test_where',
'TestGridfs.test_gridfs_find',
@ -368,7 +356,6 @@ if __name__ == '__main__':
'pymongo.encryption_options',
'pymongo.mongo_client',
'pymongo.database',
'pymongo.mongo_replica_set_client',
'gridfs',
'gridfs.grid_file']:
sys.modules.pop(n)

10
test/asyncio_tests/test_aiohttp_gridfs.py
View File

@ -26,6 +26,7 @@ import aiohttp.web
import gridfs
from motor.aiohttp import AIOHTTPGridFS
from motor.motor_gridfs import _hash_gridout
import test
from test.asyncio_tests import AsyncIOTestCase, asyncio_test
@ -61,17 +62,18 @@ class AIOHTTPGridFSHandlerTestBase(AsyncIOTestCase):
# Make a 500k file in GridFS with filename 'foo'
cls.contents = b'Jesse' * 100 * 1024
cls.contents_hash = hashlib.md5(cls.contents).hexdigest()
# Record when we created the file, to check the Last-Modified header
cls.put_start = datetime.datetime.utcnow().replace(microsecond=0)
cls.file_id = 'id'
file_id = 'id'
cls.file_id = file_id
cls.fs.delete(cls.file_id)
cls.fs.put(cls.contents,
_id='id',
_id=file_id,
filename='foo',
content_type='my type')
item = cls.fs.get(file_id)
cls.contents_hash = _hash_gridout(item)
cls.put_end = datetime.datetime.utcnow().replace(microsecond=0)
cls.app = cls.srv = cls.app_handler = None

20
test/asyncio_tests/test_asyncio_basic.py
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest import SkipTest
from abc import ABC
import pymongo
from pymongo import WriteConcern
@ -43,14 +43,17 @@ class AIOMotorTestBasic(AsyncIOTestCase):
await self.collection.delete_many({})
await self.collection.insert_one({'_id': 0})
for gle_options in [
for wc_opts in [
{},
{'w': 0},
{'w': 1},
{'wtimeout': 1000},
{'wTimeoutMS': 1000},
]:
cx = self.asyncio_client(test.env.uri, **gle_options)
wc = WriteConcern(**gle_options)
cx = self.asyncio_client(test.env.uri, **wc_opts)
wtimeout = wc_opts.pop('wTimeoutMS', None)
if wtimeout:
wc_opts['wtimeout'] = wtimeout
wc = WriteConcern(**wc_opts)
self.assertEqual(wc, cx.write_concern)
db = cx.motor_test
@ -84,7 +87,7 @@ class AIOMotorTestBasic(AsyncIOTestCase):
self.assertEqual(ReadPreference.SECONDARY.mode, cx.read_preference.mode)
self.assertEqual([{'foo': 'bar'}], cx.read_preference.tag_sets)
self.assertEqual(42, cx.local_threshold_ms)
self.assertEqual(42, cx.options.local_threshold_ms)
# Make a MotorCursor and get its PyMongo Cursor
collection = cx.motor_test.test_collection.with_options(
@ -116,11 +119,6 @@ class AIOMotorTestBasic(AsyncIOTestCase):
self.collection._collection
def test_abc(self):
try:
from abc import ABC
except ImportError:
# Python < 3.4.
raise SkipTest()
class C(ABC):
db = self.db

3
test/asyncio_tests/test_asyncio_client.py
View File

@ -18,6 +18,7 @@ import asyncio
import os
import unittest
from unittest import SkipTest
from sys import platform
try:
import contextvars
@ -147,7 +148,7 @@ class TestAsyncIOClient(AsyncIOTestCase):
io_loop=self.loop)
cx = self.asyncio_client(maxPoolSize=100)
self.assertEqual(cx.max_pool_size, 100)
self.assertEqual(cx.options.pool_options.max_pool_size, 100)
cx.close()
@asyncio_test(timeout=30)

57
test/asyncio_tests/test_asyncio_collection.py
View File

@ -177,55 +177,6 @@ class TestAsyncIOCollection(AsyncIOTestCase):
while not (await coll.find_one({'a': 1})):
await asyncio.sleep(0.1)
@asyncio_test
async def test_map_reduce(self):
# Count number of documents with even and odd _id
await self.make_test_data()
expected_result = [{'_id': 0, 'value': 100}, {'_id': 1, 'value': 100}]
map_fn = bson.Code('function map() { emit(this._id % 2, 1); }')
reduce_fn = bson.Code('''
function reduce(key, values) {
r = 0;
values.forEach(function(value) { r += value; });
return r;
}''')
await self.db.tmp_mr.drop()
# First do a standard mapreduce, should return AsyncIOMotorCollection
collection = self.collection
tmp_mr = await collection.map_reduce(map_fn, reduce_fn, 'tmp_mr')
self.assertTrue(
isinstance(tmp_mr, motor_asyncio.AsyncIOMotorCollection),
'map_reduce should return AsyncIOMotorCollection, not %s' % tmp_mr)
result = await tmp_mr.find().sort([('_id', 1)]).to_list(
length=1000)
self.assertEqual(expected_result, result)
# Standard mapreduce with full response
await self.db.tmp_mr.drop()
response = await collection.map_reduce(
map_fn, reduce_fn, 'tmp_mr', full_response=True)
self.assertTrue(
isinstance(response, dict),
'map_reduce should return dict, not %s' % response)
self.assertEqual('tmp_mr', response['result'])
result = await tmp_mr.find().sort([('_id', 1)]).to_list(
length=1000)
self.assertEqual(expected_result, result)
# Inline mapreduce
await self.db.tmp_mr.drop()
result = await collection.inline_map_reduce(
map_fn, reduce_fn)
result.sort(key=lambda doc: doc['_id'])
self.assertEqual(expected_result, result)
@ignore_deprecations
@asyncio_test
async def test_indexes(self):
@ -275,6 +226,14 @@ class TestAsyncIOCollection(AsyncIOTestCase):
self.assertTrue('_unpack_response' in formatted
or '_check_command_response' in formatted)
@asyncio_test
async def test_aggregate_cursor_del(self):
cursor = self.db.test.aggregate(self.pipeline)
del cursor
cursor = self.db.test.aggregate(self.pipeline)
await cursor.close()
del cursor
def test_with_options(self):
coll = self.db.test
codec_options = CodecOptions(

14
test/asyncio_tests/test_asyncio_grid_file.py
View File

@ -61,8 +61,7 @@ class MotorGridFileTest(AsyncIOTestCase):
'chunk_size',
'upload_date',
'aliases',
'metadata',
'md5')
'metadata')
for attr_name in attr_names:
self.assertRaises(InvalidOperation, getattr, g, attr_name)
@ -136,7 +135,6 @@ class MotorGridFileTest(AsyncIOTestCase):
async def test_alternate_collection(self):
await self.db.alt.files.delete_many({})
await self.db.alt.chunks.delete_many({})
f = motor_asyncio.AsyncIOMotorGridIn(self.db.alt)
await f.write(b"hello world")
await f.close()
@ -147,9 +145,6 @@ class MotorGridFileTest(AsyncIOTestCase):
g = motor_asyncio.AsyncIOMotorGridOut(self.db.alt, f._id)
self.assertEqual(b"hello world", (await g.read()))
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
@asyncio_test
async def test_grid_in_default_opts(self):
self.assertRaises(TypeError, motor_asyncio.AsyncIOMotorGridIn, "foo")
@ -192,8 +187,6 @@ class MotorGridFileTest(AsyncIOTestCase):
await a.set("metadata", {"foo": 1})
self.assertEqual({"foo": 1}, a.metadata)
self.assertRaises(AttributeError, setattr, a, "md5", 5)
await a.close()
self.assertTrue(isinstance(a._id, ObjectId))
@ -216,9 +209,6 @@ class MotorGridFileTest(AsyncIOTestCase):
self.assertEqual({"foo": 1}, a.metadata)
self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", a.md5)
self.assertRaises(AttributeError, setattr, a, "md5", 5)
@asyncio_test
async def test_grid_in_custom_opts(self):
self.assertRaises(TypeError, motor_asyncio.AsyncIOMotorGridIn, "foo")
@ -266,7 +256,6 @@ class MotorGridFileTest(AsyncIOTestCase):
self.assertTrue(isinstance(b.upload_date, datetime.datetime))
self.assertEqual(None, b.aliases)
self.assertEqual(None, b.metadata)
self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5)
@asyncio_test
async def test_grid_out_custom_opts(self):
@ -288,7 +277,6 @@ class MotorGridFileTest(AsyncIOTestCase):
self.assertEqual(["foo"], two.aliases)
self.assertEqual({"foo": 1, "bar": 2}, two.metadata)
self.assertEqual(3, two.bar)
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5)
@asyncio_test
async def test_grid_out_file_document(self):

2
test/asyncio_tests/test_asyncio_gridfsbucket.py
View File

@ -67,7 +67,7 @@ class TestAsyncIOGridFSBucket(AsyncIOTestCase):
rp = ReadPreference.SECONDARY
size = 8
bucket = AsyncIOMotorGridFSBucket(
self.db, name, disable_md5=True, chunk_size_bytes=size,
self.db, name, chunk_size_bytes=size,
write_concern=wc, read_preference=rp)
self.assertEqual(name, bucket.collection.name)
self.assertEqual(wc, bucket.collection.write_concern)

3
test/asyncio_tests/test_asyncio_replica_set.py
View File

@ -18,11 +18,10 @@ import unittest
import pymongo
import pymongo.errors
import pymongo.mongo_replica_set_client
import test
from motor import motor_asyncio
from test import env, SkipTest
from test import SkipTest
from test.asyncio_tests import AsyncIOTestCase, asyncio_test
from test.test_environment import env

2
test/asyncio_tests/test_asyncio_session.py
View File

@ -36,7 +36,7 @@ class TestAsyncIOSession(AsyncIOTestCase):
raise SkipTest("Sessions not supported")
async def _test_ops(self, client, *ops):
listener = client.event_listeners()[0][0]
listener = client.options.event_listeners[0]
for f, args, kw in ops:
s = await client.start_session()

56
test/asyncio_tests/test_asyncio_ssl.py
View File

@ -59,25 +59,19 @@ class TestAsyncIOSSL(unittest.TestCase):
self.assertRaises(ValueError,
AsyncIOMotorClient,
io_loop=self.loop,
ssl='foo')
tls='foo')
self.assertRaises(ConfigurationError,
AsyncIOMotorClient,
io_loop=self.loop,
ssl=False,
ssl_certfile=CLIENT_PEM)
tls=False,
tlsCertificateKeyFile=CLIENT_PEM)
self.assertRaises(IOError, AsyncIOMotorClient,
io_loop=self.loop, ssl_certfile="NoFile")
io_loop=self.loop, tlsCertificateKeyFile="NoFile")
self.assertRaises(TypeError, AsyncIOMotorClient,
io_loop=self.loop, ssl_certfile=True)
self.assertRaises(IOError, AsyncIOMotorClient,
io_loop=self.loop, ssl_keyfile="NoFile")
self.assertRaises(TypeError, AsyncIOMotorClient,
io_loop=self.loop, ssl_keyfile=True)
io_loop=self.loop, tlsCertificateKeyFile=True)
@asyncio_test
async def test_cert_ssl(self):
@ -88,8 +82,8 @@ class TestAsyncIOSSL(unittest.TestCase):
raise SkipTest("Can't test with auth")
client = AsyncIOMotorClient(env.host, env.port,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.loop)
await client.db.collection.find_one()
@ -97,9 +91,9 @@ class TestAsyncIOSSL(unittest.TestCase):
if 'setName' in response:
client = AsyncIOMotorClient(
env.host, env.port,
ssl=True,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
tls=True,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
replicaSet=response['setName'],
io_loop=self.loop)
@ -114,9 +108,8 @@ class TestAsyncIOSSL(unittest.TestCase):
raise SkipTest("Can't test with auth")
client = AsyncIOMotorClient(env.host, env.port,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.loop)
await client.db.collection.find_one()
@ -126,9 +119,8 @@ class TestAsyncIOSSL(unittest.TestCase):
client = AsyncIOMotorClient(
env.host, env.port,
replicaSet=response['setName'],
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.loop)
await client.db.collection.find_one()
@ -142,9 +134,9 @@ class TestAsyncIOSSL(unittest.TestCase):
raise SkipTest("Can't test with auth")
client = AsyncIOMotorClient(test.env.fake_hostname_uri,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_NONE,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsAllowInvalidCertificates=True,
tlsCAFile=CA_PEM,
io_loop=self.loop)
await client.admin.command('ismaster')
@ -158,8 +150,8 @@ class TestAsyncIOSSL(unittest.TestCase):
raise SkipTest("Can't test with auth")
client = AsyncIOMotorClient(env.host, env.port,
ssl=True, ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
tls=True, tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.loop)
response = await client.admin.command('ismaster')
@ -168,9 +160,8 @@ class TestAsyncIOSSL(unittest.TestCase):
# which is what the server cert presents.
client = AsyncIOMotorClient(test.env.fake_hostname_uri,
serverSelectionTimeoutMS=1000,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.loop)
await client.db.collection.find_one()
@ -181,9 +172,8 @@ class TestAsyncIOSSL(unittest.TestCase):
test.env.fake_hostname_uri,
serverSelectionTimeoutMS=1000,
replicaSet=response['setName'],
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.loop)
await client.db.collection.find_one()

5
test/test_environment.py
View File

@ -148,6 +148,7 @@ class TestEnvironment(object):
host, port,
username=db_user,
password=db_password,
directConnection=True,
connectTimeoutMS=connectTimeoutMS,
socketTimeoutMS=socketTimeoutMS,
serverSelectionTimeoutMS=serverSelectionTimeoutMS,
@ -161,6 +162,7 @@ class TestEnvironment(object):
host, port,
username=db_user,
password=db_password,
directConnection=True,
connectTimeoutMS=connectTimeoutMS,
socketTimeoutMS=socketTimeoutMS,
serverSelectionTimeoutMS=serverSelectionTimeoutMS,
@ -174,6 +176,7 @@ class TestEnvironment(object):
host, port,
username=db_user,
password=db_password,
directConnection=True,
connectTimeoutMS=connectTimeoutMS,
socketTimeoutMS=socketTimeoutMS,
serverSelectionTimeoutMS=serverSelectionTimeoutMS))
@ -202,6 +205,7 @@ class TestEnvironment(object):
host, port,
username=db_user,
password=db_password,
directConnection=True,
tlsCAFile=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM))
else:
@ -209,6 +213,7 @@ class TestEnvironment(object):
host, port,
username=db_user,
password=db_password,
directConnection=True,
ssl=False))
self.sync_cx = client

21
test/tornado_tests/test_motor_basic.py
View File

@ -14,6 +14,8 @@
"""Test Motor, an asynchronous driver for MongoDB and Tornado."""
from abc import ABC
import pymongo
from pymongo import WriteConcern
from pymongo.errors import ConfigurationError
@ -43,14 +45,17 @@ class MotorTestBasic(MotorTest):
await self.collection.delete_many({})
await self.collection.insert_one({'_id': 0})
for gle_options in [
for wc_opts in [
{},
{'w': 0},
{'w': 1},
{'wtimeout': 1000},
{'wTimeoutMS': 1000},
]:
cx = self.motor_client(test.env.uri, **gle_options)
wc = WriteConcern(**gle_options)
cx = self.motor_client(test.env.uri, **wc_opts)
wtimeout = wc_opts.pop('wTimeoutMS', None)
if wtimeout:
wc_opts['wtimeout'] = wtimeout
wc = WriteConcern(**wc_opts)
self.assertEqual(wc, cx.write_concern)
db = cx.motor_test
@ -83,7 +88,7 @@ class MotorTestBasic(MotorTest):
self.assertEqual(ReadPreference.SECONDARY.mode, cx.read_preference.mode)
self.assertEqual([{'foo': 'bar'}], cx.read_preference.tag_sets)
self.assertEqual(42, cx.local_threshold_ms)
self.assertEqual(42, cx.options.local_threshold_ms)
# Make a MotorCursor and get its PyMongo Cursor
collection = cx.motor_test.test_collection.with_options(
@ -115,12 +120,6 @@ class MotorTestBasic(MotorTest):
self.collection._collection
def test_abc(self):
try:
from abc import ABC
except ImportError:
# Python < 3.4.
raise SkipTest()
class C(ABC):
db = self.db
collection = self.collection

2
test/tornado_tests/test_motor_client.py
View File

@ -120,7 +120,7 @@ class MotorClientTest(MotorTest):
motor.MotorClient(maxPoolSize='foo')
cx = self.motor_client(maxPoolSize=100)
self.assertEqual(cx.max_pool_size, 100)
self.assertEqual(cx.options.pool_options.max_pool_size, 100)
cx.close()
@gen_test(timeout=30)

55
test/tornado_tests/test_motor_collection.py
View File

@ -176,53 +176,6 @@ class MotorCollectionTest(MotorTest):
while not (await coll.find_one({'a': 1})):
await gen.sleep(0.1)
@gen_test
async def test_map_reduce(self):
# Count number of documents with even and odd _id
await self.make_test_data()
expected_result = [{'_id': 0, 'value': 100}, {'_id': 1, 'value': 100}]
map_fn = bson.Code('function map() { emit(this._id % 2, 1); }')
reduce_fn = bson.Code('''
function reduce(key, values) {
r = 0;
values.forEach(function(value) { r += value; });
return r;
}''')
await self.db.tmp_mr.drop()
# First do a standard mapreduce, should return MotorCollection
collection = self.collection
tmp_mr = await collection.map_reduce(map_fn, reduce_fn, 'tmp_mr')
self.assertTrue(
isinstance(tmp_mr, motor.MotorCollection),
'map_reduce should return MotorCollection, not %s' % tmp_mr)
result = await tmp_mr.find().sort([('_id', 1)]).to_list(length=1000)
self.assertEqual(expected_result, result)
# Standard mapreduce with full response
await self.db.tmp_mr.drop()
response = await collection.map_reduce(
map_fn, reduce_fn, 'tmp_mr', full_response=True)
self.assertTrue(
isinstance(response, dict),
'map_reduce should return dict, not %s' % response)
self.assertEqual('tmp_mr', response['result'])
result = await tmp_mr.find().sort([('_id', 1)]).to_list(length=1000)
self.assertEqual(expected_result, result)
# Inline mapreduce
await self.db.tmp_mr.drop()
result = await collection.inline_map_reduce(
map_fn, reduce_fn)
result.sort(key=lambda doc: doc['_id'])
self.assertEqual(expected_result, result)
@ignore_deprecations
@gen_test
async def test_indexes(self):
@ -273,6 +226,14 @@ class MotorCollectionTest(MotorTest):
self.assertTrue('_unpack_response' in formatted
or '_check_command_response' in formatted)
@gen_test
async def test_aggregate_cursor_del(self):
cursor = self.db.test.aggregate(self.pipeline)
del cursor
cursor = self.db.test.aggregate(self.pipeline)
await cursor.close()
del cursor
def test_with_options(self):
coll = self.db.test
codec_options = CodecOptions(

46
test/tornado_tests/test_motor_core.py
View File

@ -36,39 +36,11 @@ pymongo_only = set(['next'])
motor_client_only = motor_only.union(['open'])
pymongo_client_only = set([
'close_cursor',
'database_names',
'is_locked',
'set_cursor_manager',
'kill_cursors']).union(pymongo_only)
pymongo_client_only = set([]).union(pymongo_only)
pymongo_database_only = set([
'add_user',
'collection_names',
'remove_user',
'system_js',
'last_status',
'reset_error_history',
'eval',
'add_son_manipulator',
'logout',
'error',
'authenticate',
'previous_error']).union(pymongo_only)
pymongo_database_only = set([]).union(pymongo_only)
pymongo_collection_only = set([
'count',
'ensure_index',
'group',
'initialize_ordered_bulk_op',
'initialize_unordered_bulk_op',
'save',
'remove',
'insert',
'update',
'find_and_modify',
'parallel_scan']).union(pymongo_only)
pymongo_collection_only = set([]).union(pymongo_only)
motor_cursor_only = set([
'fetch_next',
@ -79,7 +51,6 @@ motor_cursor_only = set([
'closed']).union(motor_only)
pymongo_cursor_only = set([
'count',
'retrieved'])
@ -157,9 +128,10 @@ class MotorCoreTestGridFS(MotorTest):
def test_gridin_attrs(self):
motor_gridin_only = set(['set']).union(motor_only)
gridin_only = set(['md5'])
self.assertEqual(
attrs(GridIn(env.sync_cx.test.fs)),
attrs(GridIn(env.sync_cx.test.fs)) - gridin_only,
attrs(MotorGridIn(self.cx.test.fs)) - motor_gridin_only)
@gen_test
@ -169,10 +141,14 @@ class MotorCoreTestGridFS(MotorTest):
'stream_to_handler'
]).union(motor_only)
gridin_only = set([
'md5', 'readlines', 'truncate', 'flush', 'fileno', 'closed', 'writelines',
'isatty', 'writable'])
fs = MotorGridFSBucket(self.cx.test)
motor_gridout = await fs.open_download_stream(1)
self.assertEqual(
attrs(self.sync_fs.open_download_stream(1)),
attrs(self.sync_fs.open_download_stream(1)) - gridin_only,
attrs(motor_gridout) - motor_gridout_only)
def test_gridout_cursor_attrs(self):

13
test/tornado_tests/test_motor_grid_file.py
View File

@ -63,8 +63,7 @@ class MotorGridFileTest(MotorTest):
'chunk_size',
'upload_date',
'aliases',
'metadata',
'md5')
'metadata')
for attr_name in attr_names:
self.assertRaises(InvalidOperation, getattr, g, attr_name)
@ -149,9 +148,6 @@ class MotorGridFileTest(MotorTest):
g = motor.MotorGridOut(self.db.alt, f._id)
self.assertEqual(b"hello world", (await g.read()))
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
@gen_test
async def test_grid_in_default_opts(self):
self.assertRaises(TypeError, motor.MotorGridIn, "foo")
@ -194,8 +190,6 @@ class MotorGridFileTest(MotorTest):
await a.set("metadata", {"foo": 1})
self.assertEqual({"foo": 1}, a.metadata)
self.assertRaises(AttributeError, setattr, a, "md5", 5)
await a.close()
self.assertTrue(isinstance(a._id, ObjectId))
@ -218,9 +212,6 @@ class MotorGridFileTest(MotorTest):
self.assertEqual({"foo": 1}, a.metadata)
self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", a.md5)
self.assertRaises(AttributeError, setattr, a, "md5", 5)
@gen_test
async def test_grid_in_custom_opts(self):
self.assertRaises(TypeError, motor.MotorGridIn, "foo")
@ -268,7 +259,6 @@ class MotorGridFileTest(MotorTest):
self.assertTrue(isinstance(b.upload_date, datetime.datetime))
self.assertEqual(None, b.aliases)
self.assertEqual(None, b.metadata)
self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5)
@gen_test
async def test_grid_out_custom_opts(self):
@ -290,7 +280,6 @@ class MotorGridFileTest(MotorTest):
self.assertEqual(["foo"], two.aliases)
self.assertEqual({"foo": 1, "bar": 2}, two.metadata)
self.assertEqual(3, two.bar)
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5)
@gen_test
async def test_grid_out_file_document(self):

4
test/tornado_tests/test_motor_replica_set.py
View File

@ -19,7 +19,6 @@ import unittest
import pymongo
import pymongo.auth
import pymongo.errors
import pymongo.mongo_replica_set_client
from tornado import gen
from tornado.testing import gen_test
@ -27,9 +26,8 @@ import motor
import motor.core
import test
from test import SkipTest
from test.test_environment import db_user, db_password, env
from test.test_environment import env
from test.tornado_tests import MotorReplicaSetTestBase, MotorTest
from test.utils import one, get_primary_pool
class MotorReplicaSetTest(MotorReplicaSetTestBase):

2
test/tornado_tests/test_motor_session.py
View File

@ -37,7 +37,7 @@ class MotorSessionTest(MotorTest):
raise SkipTest("Sessions not supported")
async def _test_ops(self, client, *ops):
listener = client.event_listeners()[0][0]
listener = client.options.event_listeners[0]
for f, args, kw in ops:
# Simulate "async with" on all Pythons.

50
test/tornado_tests/test_motor_ssl.py
View File

@ -55,16 +55,14 @@ class MotorSSLTest(MotorTest):
super().setUp()
def test_config_ssl(self):
self.assertRaises(ValueError, motor.MotorClient, ssl='foo')
self.assertRaises(ValueError, motor.MotorClient, tls='foo')
self.assertRaises(ConfigurationError,
motor.MotorClient,
ssl=False,
ssl_certfile=CLIENT_PEM)
tls=False,
tlsCertificateKeyFile=CLIENT_PEM)
self.assertRaises(IOError, motor.MotorClient, ssl_certfile="NoFile")
self.assertRaises(TypeError, motor.MotorClient, ssl_certfile=True)
self.assertRaises(IOError, motor.MotorClient, ssl_keyfile="NoFile")
self.assertRaises(TypeError, motor.MotorClient, ssl_keyfile=True)
self.assertRaises(IOError, motor.MotorClient, tlsCertificateKeyFile="NoFile")
self.assertRaises(TypeError, motor.MotorClient, tlsCertificateKeyFile=True)
@gen_test
async def test_cert_ssl(self):
@ -75,15 +73,15 @@ class MotorSSLTest(MotorTest):
raise SkipTest("can't test with auth")
client = motor.MotorClient(env.host, env.port,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
await client.db.collection.find_one()
response = await client.admin.command('ismaster')
if 'setName' in response:
client = self.motor_rsc(ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM)
client = self.motor_rsc(tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM)
await client.db.collection.find_one()
@gen_test
@ -96,9 +94,8 @@ class MotorSSLTest(MotorTest):
client = motor.MotorClient(
env.host, env.port,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
await client.db.collection.find_one()
@ -108,9 +105,8 @@ class MotorSSLTest(MotorTest):
client = motor.MotorClient(
env.host, env.port,
replicaSet=response['setName'],
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
await client.db.collection.find_one()
@ -125,9 +121,9 @@ class MotorSSLTest(MotorTest):
client = motor.MotorClient(
test.env.fake_hostname_uri,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_NONE,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsAllowInvalidCertificates=True,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
await client.admin.command('ismaster')
@ -142,8 +138,8 @@ class MotorSSLTest(MotorTest):
client = motor.MotorClient(
env.host, env.port,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
response = await client.admin.command('ismaster')
@ -153,9 +149,8 @@ class MotorSSLTest(MotorTest):
client = motor.MotorClient(
test.env.fake_hostname_uri,
serverSelectionTimeoutMS=100,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
await client.db.collection.find_one()
@ -166,9 +161,8 @@ class MotorSSLTest(MotorTest):
test.env.fake_hostname_uri,
serverSelectionTimeoutMS=100,
replicaSet=response['setName'],
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
tlsCertificateKeyFile=CLIENT_PEM,
tlsCAFile=CA_PEM,
io_loop=self.io_loop)
await client.db.collection.find_one()

11
test/tornado_tests/test_motor_web.py
View File

@ -27,6 +27,7 @@ from tornado.web import Application
import motor
import motor.web
from motor.motor_gridfs import _hash_gridout
import test
from test.test_environment import env, CA_PEM, CLIENT_PEM
@ -41,15 +42,17 @@ class GridFSHandlerTestBase(AsyncHTTPTestCase):
# Make a 500k file in GridFS with filename 'foo'
self.contents = b'Jesse' * 100 * 1024
self.contents_hash = hashlib.md5(self.contents).hexdigest()
# Record when we created the file, to check the Last-Modified header
self.put_start = datetime.datetime.utcnow().replace(microsecond=0)
self.file_id = 'id'
file_id = 'id'
self.file_id = file_id
self.fs.delete(self.file_id)
self.fs.put(
self.contents, _id='id', filename='foo', content_type='my type')
self.contents, _id=file_id, filename='foo', content_type='my type')
item = self.fs.get(file_id)
self.contents_hash = _hash_gridout(item)
self.put_end = datetime.datetime.utcnow().replace(microsecond=0)
self.assertTrue(self.fs.get_last_version('foo'))

12
tox.ini
View File

@ -23,8 +23,8 @@ envlist =
# asyncio without Tornado.
asyncio-{pypy35,pypy36,py35,py36,py37,py38,py39,py310},
# Test PyMongo 3.12 because 4.x breaks some APIs
py3-pymongo-v3.12,
# Test with the latest PyMongo.
py3-pymongo-latest,
# Apply PyMongo's test suite to Motor via Synchro.
synchro37
@ -62,7 +62,7 @@ deps =
sphinx: aiohttp
sphinx: git+https://github.com/tornadoweb/tornado.git
py3-pymongo-v3.12: tornado>=5,<6
py3-pymongo-latest: tornado>=5,<6
synchro37: tornado>=6,<7
synchro37: nose
@ -85,9 +85,9 @@ changedir = doc
commands =
sphinx-build -q -E -b doctest . {envtmpdir}/doctest {posargs}
[testenv:py3-pymongo-v3.12]
[testenv:py3-pymongo-latest]
commands =
pip install git+https://github.com/mongodb/mongo-python-driver.git@v3.12#egg=pymongo[encryption]
pip install git+https://github.com/mongodb/mongo-python-driver.git@master#egg=pymongo[encryption]
python --version
python -c "import pymongo; print('PyMongo %s' % (pymongo.version,))"
python setup.py test --xunit-output=xunit-results {posargs}
@ -98,5 +98,5 @@ allowlist_externals =
setenv =
PYTHONPATH = {envtmpdir}/mongo-python-driver
commands =
git clone --depth 1 --branch v3.12 https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
git clone --depth 1 --branch master https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
python3 -m synchro.synchrotest --with-xunit --xunit-file=xunit-synchro-results -v -w {envtmpdir}/mongo-python-driver {posargs}