mongodb/motor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MOTOR-228 Fix Python 3 module loading

Browse Source 
Make sys.meta_path import hook Python 3 compatible.
Make synchro ChangeStream iterable.
This commit is contained in:
Shane Harvey 2019-10-22 15:22:08 -07:00
parent a300b5e0d5
commit 163c56f6cc
2 changed files with 149 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
synchro/__init__.py
View File

@ -33,7 +33,12 @@ from motor.metaprogramming import MotorAttributeFactory
# Make e.g. "from pymongo.errors import AutoReconnect" work. Note that
# importing * won't pick up underscore-prefixed attrs.
from gridfs import *
from gridfs.errors import *
from gridfs.grid_file import (DEFAULT_CHUNK_SIZE,
_SEEK_CUR,
_SEEK_END,
_clear_entity_type_registry)
from pymongo import *
from pymongo import (collation,
compression_support,
@ -94,7 +99,6 @@ from pymongo.write_concern import *
from pymongo import auth
from pymongo.auth import *
from pymongo.auth import _password_digest
from gridfs.grid_file import DEFAULT_CHUNK_SIZE, _SEEK_CUR, _SEEK_END
from pymongo import GEOSPHERE, HASHED
from pymongo.pool import SocketInfo, Pool
@ -282,7 +286,15 @@ class SynchroMeta(type):
return new_class
class Synchro(object):
def with_metaclass(metaclass, *bases):
"""Python 2/3 compatible metaclass helper."""
class _metaclass(metaclass):
def __new__(mcls, name, _bases, attrs):
return metaclass(name, bases, attrs)
return type.__new__(_metaclass, str('dummy'), (), {})
class Synchro(with_metaclass(SynchroMeta)):
"""
Wraps a MotorClient, MotorDatabase, MotorCollection, etc. and
makes it act like the synchronous pymongo equivalent
@ -291,8 +303,17 @@ class Synchro(object):
__delegate_class__ = None
def __cmp__(self, other):
"""Implements == and != on Python 2."""
return cmp(self.delegate, other.delegate)
def __eq__(self, other):
"""Implements == and != on Python 3."""
if (isinstance(other, self.__class__)
and hasattr(self, 'delegate')
and hasattr(other, 'delegate')):
return self.delegate == other.delegate
return NotImplemented
def synchronize(self, async_method):
"""
@param async_method: Bound method of a MotorClient, MotorDatabase, etc.
@ -314,7 +335,6 @@ class MongoClient(Synchro):
HOST = 'localhost'
PORT = 27017
_cache_credentials = SynchroProperty()
get_database = WrapOutgoing()
max_pool_size = SynchroProperty()
max_write_batch_size = SynchroProperty()
@ -352,8 +372,11 @@ class MongoClient(Synchro):
def __getitem__(self, name):
return Database(self, name, delegate=self.delegate[name])
# For PyMongo tests that access client internals.
_MongoClient__all_credentials = SynchroProperty()
_MongoClient__options = SynchroProperty()
_cache_credentials = SynchroProperty()
_close_cursor_now = SynchroProperty()
_get_topology = SynchroProperty()
_topology = SynchroProperty()
_kill_cursors_executor = SynchroProperty()
@ -390,6 +413,7 @@ class ClientSession(Synchro):
def __exit__(self, exc_type, exc_val, exc_tb):
self.synchronize(self.delegate.end_session)
# For PyMongo tests that access session internals.
_client = SynchroProperty()
_in_transaction = SynchroProperty()
_pinned_address = SynchroProperty()
@ -495,10 +519,16 @@ class Collection(Synchro):
class ChangeStream(Synchro):
__delegate_class__ = motor.motor_tornado.MotorChangeStream
next = Sync('next')
_next = Sync('next')
try_next = Sync('try_next')
close = Sync('close')
def next(self):
try:
return self._next()
except StopAsyncIteration:
raise StopIteration()
def __init__(self, motor_change_stream):
self.delegate = motor_change_stream
@ -508,6 +538,24 @@ class ChangeStream(Synchro):
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def __iter__(self):
return self
__next__ = next
# For PyMongo tests that access change stream internals.
@property
def _cursor(self):
raise unittest.SkipTest('test accesses internal _cursor field')
_batch_size = SynchroProperty()
_client = SynchroProperty()
_full_document = SynchroProperty()
_max_await_time_ms = SynchroProperty()
_pipeline = SynchroProperty()
_target = SynchroProperty()
class Cursor(Synchro):
__delegate_class__ = motor.motor_tornado.MotorCursor
@ -592,7 +640,7 @@ class GridOutCursor(Cursor):
raise TypeError(
"Expected MotorGridOutCursor, got %r" % delegate)
self.delegate = delegate
super(GridOutCursor, self).__init__(delegate)
def next(self):
motor_grid_out = super(GridOutCursor, self).next()
@ -712,3 +760,18 @@ class GridOut(Synchro):
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def __next__(self):
if sys.version_info >= (3, 5):
try:
return self.synchronize(self.delegate.__anext__)()
except StopAsyncIteration:
raise StopIteration()
else:
chunk = self.readchunk()
if chunk:
return chunk
raise StopIteration()
def __iter__(self):
return self

97
synchro/synchrotest.py
View File

@ -210,6 +210,15 @@ excluded_tests = [
'TestTransactionsConvenientAPI.*',
]
if sys.version_info[:2] >= (3, 5):
excluded_tests.extend([
# Motor's change streams need Python 3.5 to support async iteration but
# these change streams tests spawn threads which don't work without an
# IO loop.
'*.test_next_blocks',
'*.test_aggregate_cursor_blocks',
])
excluded_modules_matched = set()
excluded_tests_matched = set()
@ -294,27 +303,67 @@ class SynchroNosePlugin(Plugin):
return True
# So that e.g. 'from pymongo.mongo_client import MongoClient' gets the
# Synchro MongoClient, not the real one.
class SynchroModuleFinder(object):
def find_module(self, fullname, path=None):
parts = fullname.split('.')
if parts[-1] in ('gridfs', 'pymongo'):
return SynchroModuleLoader(path)
elif len(parts) >= 2 and parts[-2] in ('gridfs', 'pymongo'):
return SynchroModuleLoader(path)
if sys.version_info[0] < 3:
# So that e.g. 'from pymongo.mongo_client import MongoClient' gets the
# Synchro MongoClient, not the real one.
class SynchroModuleFinder(object):
def find_module(self, fullname, path=None):
parts = fullname.split('.')
if parts[-1] in ('gridfs', 'pymongo'):
# E.g. "import pymongo"
return SynchroModuleLoader(path)
elif len(parts) >= 2 and parts[-2] in ('gridfs', 'pymongo'):
# E.g. "import pymongo.mongo_client"
return SynchroModuleLoader(path)
# Let regular module search continue.
return None
# Let regular module search continue.
return None
class SynchroModuleLoader(object):
def __init__(self, path):
self.path = path
class SynchroModuleLoader(object):
def __init__(self, path):
self.path = path
def load_module(self, fullname):
return synchro
def load_module(self, fullname):
return synchro
else:
import importlib
import importlib.abc
import importlib.machinery
class SynchroModuleFinder(importlib.abc.MetaPathFinder):
def __init__(self):
self._loader = SynchroModuleLoader()
def find_spec(self, fullname, path, target=None):
if self._loader.patch_spec(fullname):
return importlib.machinery.ModuleSpec(fullname, self._loader)
# Let regular module search continue.
return None
class SynchroModuleLoader(importlib.abc.Loader):
def patch_spec(self, fullname):
parts = fullname.split('.')
if parts[-1] in ('gridfs', 'pymongo'):
# E.g. "import pymongo"
return True
elif len(parts) >= 2 and parts[-2] in ('gridfs', 'pymongo'):
# E.g. "import pymongo.mongo_client"
return True
return False
def exec_module(self, module):
pass
def create_module(self, spec):
if self.patch_spec(spec.name):
return synchro
# Let regular module search continue.
return None
if __name__ == '__main__':
try:
@ -333,6 +382,22 @@ if __name__ == '__main__':
# Monkey-patch all pymongo's unittests so they think Synchro is the
# real PyMongo.
sys.meta_path[0:0] = [SynchroModuleFinder()]
# Delete the cached pymongo/gridfs modules so that SynchroModuleFinder will
# be invoked in Python 3, see
# https://docs.python.org/3/reference/import.html#import-hooks
for n in ['pymongo',
'pymongo.collection',
'pymongo.client_session',
'pymongo.command_cursor',
'pymongo.change_stream',
'pymongo.cursor',
'pymongo.mongo_client',
'pymongo.database',
'pymongo.mongo_replica_set_client',
'gridfs',
'gridfs.grid_file',
'pymongo.encryption']:
sys.modules.pop(n)
if '--check-exclude-patterns' in sys.argv:
check_exclude_patterns = True