mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-1769 Re-define TypeCodecBase as an AbstractBaseClass

Browse Source 
(cherry picked from commit 65f85f648c)
This commit is contained in:
Prashant Mital 2019-03-18 16:19:51 -05:00
parent 713419ef4e
commit 2867fe544c
No known key found for this signature in database
GPG Key ID: 3D2DAA9E483ABE51
5 changed files with 288 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
bson/codec_options.py
View File

@ -16,9 +16,10 @@
import datetime
from abc import abstractmethod
from collections import namedtuple
from bson.py3compat import abc, string_type
from bson.py3compat import ABC, abc, abstractproperty, string_type
from bson.binary import (ALL_UUID_REPRESENTATIONS,
PYTHON_LEGACY,
UUID_REPRESENTATION_NAMES)
@ -33,33 +34,53 @@ def _raw_document_class(document_class):
return marker == _RAW_BSON_DOCUMENT_MARKER
class TypeCodecBase(object):
class TypeEncoder(ABC):
"""Base class for defining type codec classes which describe how a
custom type can be transformed to one of the types BSON understands.
Codec classes must implement the ``python_type`` attribute, and the
``transform_python`` method to support encoding.
"""
@abstractproperty
def python_type(self):
"""The Python type to be converted into something serializable."""
pass
@abstractmethod
def transform_python(self, value):
"""Convert the given Python object into something serializable."""
pass
class TypeDecoder(ABC):
"""Base class for defining type codec classes which describe how a
BSON type can be transformed to a custom type.
Codec classes must implement the ``bson_type`` attribute, and the
``transform_bson`` method to support decoding.
"""
@abstractproperty
def bson_type(self):
"""The BSON type to be converted into our own type."""
pass
@abstractmethod
def transform_bson(self, value):
"""Convert the given BSON value into our own type."""
pass
class TypeCodec(TypeEncoder, TypeDecoder):
"""Base class for defining type codec classes which describe how a
custom type can be transformed to/from one of the types BSON already
understands, and can encode/decode.
Codec classes must implement the ``python_type`` property, and the
``transform_python`` method to support encoding, or the ``bson_type``
property and ``transform_bson`` method to support decoding. Note that a
single codec class may support both encoding and decoding.
Codec classes must implement the ``python_type`` attribute, and the
``transform_python`` method to support encoding, as well as the
``bson_type`` attribute, and the ``transform_bson`` method to support
decoding.
"""
@property
def python_type(self):
"""The Python type to be converted into something serializable."""
raise NotImplementedError
@property
def bson_type(self):
"""The BSON type to be converted into our own type."""
raise NotImplementedError
def transform_bson(self, value):
"""Convert the given BSON value into our own type."""
raise NotImplementedError
def transform_python(self, value):
"""Convert the given Python object into something serializable."""
raise NotImplementedError
pass
class TypeRegistry(object):
@ -95,23 +116,18 @@ class TypeRegistry(object):
fallback_encoder))
for codec in self.__type_codecs:
if not isinstance(codec, TypeCodecBase):
is_valid_codec = False
if isinstance(codec, TypeEncoder):
is_valid_codec = True
self._encoder_map[codec.python_type] = codec.transform_python
if isinstance(codec, TypeDecoder):
is_valid_codec = True
self._decoder_map[codec.bson_type] = codec.transform_bson
if not is_valid_codec:
raise TypeError(
"Expected an instance of %s, got %r instead" % (
TypeCodecBase.__name__, codec))
try:
python_type = codec.python_type
except NotImplementedError:
pass
else:
self._encoder_map[python_type] = codec.transform_python
try:
bson_type = codec.bson_type
except NotImplementedError:
pass
else:
self._decoder_map[bson_type] = codec.transform_bson
"Expected an instance of %s, %s, or %s, got %r instead" % (
TypeEncoder.__name__, TypeDecoder.__name__,
TypeCodec.__name__, codec))
def __repr__(self):
return ('%s(type_codecs=%r, fallback_encoder=%r)' % (

7
bson/py3compat.py
View File

@ -22,8 +22,12 @@ if PY3:
import codecs
import collections.abc as abc
import _thread as thread
from abc import ABC, abstractmethod
from io import BytesIO as StringIO
def abstractproperty(func):
return property(abstractmethod(func))
MAXSIZE = sys.maxsize
imap = map
@ -60,6 +64,7 @@ if PY3:
else:
import collections as abc
import thread
from abc import ABCMeta, abstractproperty
from itertools import imap
try:
@ -67,6 +72,8 @@ else:
except ImportError:
from StringIO import StringIO
ABC = ABCMeta('ABC', (object,), {})
MAXSIZE = sys.maxint
def b(s):

44
doc/examples/custom_type.rst
View File

@ -2,7 +2,7 @@ Custom Type Example
===================
This is an example of using a custom type with PyMongo. The example here shows
how to subclass :class:`~bson.codec_options.TypeCodecBase` to write a type
how to subclass :class:`~bson.codec_options.TypeCodec` to write a type
codec, which is used to populate a :class:`~bson.codec_options.TypeRegistry`.
The type registry can then be used to create a custom-type-aware
:class:`~pymongo.collection.Collection`. Read and write operations
@ -51,39 +51,39 @@ The Type Codec
In order to encode custom types, we must first define a **type codec** for our
type. A type codec describes how an instance of a custom type can be
*transformed* to/from one of the types :mod:`~bson` already understands, and
can encode/decode. Type codecs must inherit from
:class:`~bson.codec_options.TypeCodecBase`. In order to encode a custom type,
a codec must implement the ``python_type`` property and the
``transform_python`` method. Similarly, in order to decode a custom type,
a codec must implement the ``bson_type`` property and the ``transform_bson``
method. Note that a type codec need not support both encoding and decoding.
*transformed* to and/or from one of the types :mod:`~bson` already understands.
Depending on the desired functionality, users must choose from the following
base classes when defining type codecs:
* :class:`~bson.codec_options.TypeEncoder`: subclass this to define a codec that
encodes a custom Python type to a known BSON type. Users must implement the
``python_type`` property/attribute and the ``transform_python`` method.
* :class:`~bson.codec_options.TypeDecoder`: subclass this to define a codec that
decodes a specified BSON type into a custom Python type. Users must implement
the ``bson_type`` property/attribute and the ``transform_bson`` method.
* :class:`~bson.codec_options.TypeCodec`: subclass this to define a codec that
can both encode from and decode to a custom type. Users must implement the
``python_type`` and ``bson_type`` properties/attributes, as well as the
``transform_python`` and ``transform_bson`` methods.
The type codec for our custom type simply needs to define how a
:py:class:`~decimal.Decimal` instance can be converted into a
:class:`~bson.decimal128.Decimal128` instance and vice-versa:
:class:`~bson.decimal128.Decimal128` instance and vice-versa. Since we are
interested in both encoding and decoding our custom type, we use the
``TypeCodec`` base class to define our codec:
.. doctest::
>>> from bson.decimal128 import Decimal128
>>> from bson.codec_options import TypeCodecBase
>>> class DecimalCodec(TypeCodecBase):
... @property
... def python_type(self):
... """The Python type acted upon by this type codec."""
... return Decimal
...
>>> from bson.codec_options import TypeCodec
>>> class DecimalCodec(TypeCodec):
... python_type = Decimal # the Python type acted upon by this type codec
... bson_type = Decimal128 # the BSON type acted upon by this type codec
... def transform_python(self, value):
... """Function that transforms a custom type value into a type
... that BSON can encode."""
... return Decimal128(value)
...
... @property
... def bson_type(self):
... """The BSON type acted upon by this type codec."""
... return Decimal128
...
... def transform_bson(self, value):
... """Function that transforms a vanilla BSON type value into our
... custom type."""

118
test/test_bson.py
View File

@ -34,14 +34,13 @@ from bson import (BSON,
Regex)
from bson.binary import Binary, UUIDLegacy
from bson.code import Code
from bson.codec_options import CodecOptions, TypeCodecBase, TypeRegistry
from bson.codec_options import CodecOptions
from bson.int64 import Int64
from bson.objectid import ObjectId
from bson.dbref import DBRef
from bson.py3compat import abc, iteritems, PY3, StringIO, text_type
from bson.son import SON
from bson.timestamp import Timestamp
from bson.tz_util import FixedOffset
from bson.errors import (InvalidBSON,
InvalidDocument,
InvalidStringData)
@ -906,121 +905,6 @@ class TestBSON(unittest.TestCase):
BSON.encode({"_id": {'$oid': "52d0b971b3ba219fdeb4170e"}})
class TestTypeRegistry(unittest.TestCase):
@classmethod
def setUpClass(cls):
class MyIntType(object):
def __init__(self, x):
assert isinstance(x, int)
self.x = x
class MyStrType(object):
def __init__(self, x):
assert isinstance(x, str)
self.x = x
class MyIntCodec(TypeCodecBase):
@property
def python_type(self):
return MyIntType
@property
def bson_type(self):
return int
def transform_python(self, value):
return value.x
def transform_bson(self, value):
return MyIntType(value)
class MyStrCodec(TypeCodecBase):
@property
def python_type(self):
return MyStrType
@property
def bson_type(self):
return str
def transform_python(self, value):
return value.x
def transform_bson(self, value):
return MyStrType(value)
def fallback_encoder(value):
return value
cls.types = (MyIntType, MyStrType)
cls.codecs = (MyIntCodec, MyStrCodec)
cls.fallback_encoder = fallback_encoder
def test_simple(self):
codec_instances = [codec() for codec in self.codecs]
def assert_proper_initialization(type_registry, codec_instances):
self.assertEqual(type_registry._encoder_map, {
self.types[0]: codec_instances[0].transform_python,
self.types[1]: codec_instances[1].transform_python})
self.assertEqual(type_registry._decoder_map, {
int: codec_instances[0].transform_bson,
str: codec_instances[1].transform_bson})
self.assertEqual(
type_registry._fallback_encoder, self.fallback_encoder)
type_registry = TypeRegistry(codec_instances, self.fallback_encoder)
assert_proper_initialization(type_registry, codec_instances)
type_registry = TypeRegistry(
fallback_encoder=self.fallback_encoder, type_codecs=codec_instances)
assert_proper_initialization(type_registry, codec_instances)
# Ensure codec list held by the type registry doesn't change if we
# mutate the initial list.
codec_instances_copy = list(codec_instances)
codec_instances.pop(0)
self.assertListEqual(
type_registry._TypeRegistry__type_codecs, codec_instances_copy)
def test_initialize_fail(self):
err_msg = "Expected an instance of TypeCodecBase, got .* instead"
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry(self.codecs)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry([type('AnyType', (object,), {})()])
err_msg = "fallback_encoder %r is not a callable" % (True,)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry([], True)
err_msg = "fallback_encoder %r is not a callable" % ('hello',)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry(fallback_encoder='hello')
def test_not_implemented(self):
type_registry = TypeRegistry([type("codec1", (TypeCodecBase, ), {})(),
type("codec2", (TypeCodecBase, ), {})()])
self.assertEqual(type_registry._encoder_map, {})
self.assertEqual(type_registry._decoder_map, {})
def test_type_registry_repr(self):
codec_instances = [codec() for codec in self.codecs]
type_registry = TypeRegistry(codec_instances)
r = ("TypeRegistry(type_codecs=%r, fallback_encoder=%r)" % (
codec_instances, None))
self.assertEqual(r, repr(type_registry))
def test_type_registry_eq(self):
codec_instances = [codec() for codec in self.codecs]
self.assertEqual(
TypeRegistry(codec_instances), TypeRegistry(codec_instances))
codec_instances_2 = [codec() for codec in self.codecs]
self.assertNotEqual(
TypeRegistry(codec_instances), TypeRegistry(codec_instances_2))
class TestCodecOptions(unittest.TestCase):
def test_document_class(self):
self.assertRaises(TypeError, CodecOptions, document_class=object)

218
test/test_bson_custom_types.py
View File

@ -28,35 +28,36 @@ from bson import (BSON,
decode_iter,
_dict_to_bson,
_bson_to_dict)
from bson.codec_options import CodecOptions, TypeCodecBase, TypeRegistry
from bson.codec_options import (CodecOptions, TypeCodec, TypeDecoder,
TypeEncoder, TypeRegistry)
from bson.errors import InvalidDocument
from test import unittest
class DecimalCodec(TypeCodecBase):
@property
def bson_type(self):
return Decimal128
class DecimalEncoder(TypeEncoder):
@property
def python_type(self):
return Decimal
def transform_bson(self, value):
return value.to_decimal()
def transform_python(self, value):
return Decimal128(value)
class TestCustomPythonTypeToBSON(unittest.TestCase):
@classmethod
def setUpClass(cls):
type_registry = TypeRegistry((DecimalCodec(),))
codec_options = CodecOptions(type_registry=type_registry)
cls.codecopts = codec_options
class DecimalDecoder(TypeDecoder):
@property
def bson_type(self):
return Decimal128
def transform_bson(self, value):
return value.to_decimal()
class DecimalCodec(DecimalDecoder, DecimalEncoder):
pass
class CustomTypeTests(object):
def test_encode_decode_roundtrip(self):
document = {'average': Decimal('56.47')}
bsonbytes = BSON().encode(document, codec_options=self.codecopts)
@ -114,6 +115,23 @@ class TestCustomPythonTypeToBSON(unittest.TestCase):
fileobj.close()
class TestCustomPythonTypeToBSONMonolithicCodec(CustomTypeTests,
unittest.TestCase):
@classmethod
def setUpClass(cls):
type_registry = TypeRegistry((DecimalCodec(),))
codec_options = CodecOptions(type_registry=type_registry)
cls.codecopts = codec_options
class TestCustomPythonTypeToBSONMultiplexedCodec(CustomTypeTests,
unittest.TestCase):
@classmethod
def setUpClass(cls):
type_registry = TypeRegistry((DecimalEncoder(), DecimalDecoder()))
codec_options = CodecOptions(type_registry=type_registry)
cls.codecopts = codec_options
class TestFallbackEncoder(unittest.TestCase):
def _get_codec_options(self, fallback_encoder):
@ -160,5 +178,175 @@ class TestFallbackEncoder(unittest.TestCase):
BSON().encode(document, codec_options=codecopts)
class TestTypeEnDeCodecs(unittest.TestCase):
def test_instantiation(self):
msg = "Can't instantiate abstract class .* with abstract methods .*"
def run_test(base, attrs, fail):
codec = type('testcodec', (base,), attrs)
if fail:
with self.assertRaisesRegex(TypeError, msg):
codec()
else:
codec()
run_test(TypeEncoder, {'python_type': int,}, fail=True)
run_test(TypeEncoder, {'transform_python': lambda s, x: x}, fail=True)
run_test(TypeEncoder, {'transform_python': lambda s, x: x,
'python_type': int}, fail=False)
run_test(TypeDecoder, {'bson_type': Decimal128, }, fail=True)
run_test(TypeDecoder, {'transform_bson': lambda s, x: x}, fail=True)
run_test(TypeDecoder, {'transform_bson': lambda s, x: x,
'bson_type': Decimal128}, fail=False)
run_test(TypeCodec, {'bson_type': Decimal128,
'python_type': int}, fail=True)
run_test(TypeCodec, {'transform_bson': lambda s, x: x,
'transform_python': lambda s, x: x}, fail=True)
run_test(TypeCodec, {'python_type': int,
'transform_python': lambda s, x: x,
'transform_bson': lambda s, x: x,
'bson_type': Decimal128}, fail=False)
def test_type_checks(self):
self.assertTrue(issubclass(TypeCodec, TypeEncoder))
self.assertTrue(issubclass(TypeCodec, TypeDecoder))
self.assertFalse(issubclass(TypeDecoder, TypeEncoder))
self.assertFalse(issubclass(TypeEncoder, TypeDecoder))
class TestTypeRegistry(unittest.TestCase):
@classmethod
def setUpClass(cls):
class MyIntType(object):
def __init__(self, x):
assert isinstance(x, int)
self.x = x
class MyStrType(object):
def __init__(self, x):
assert isinstance(x, str)
self.x = x
class MyIntCodec(TypeCodec):
@property
def python_type(self):
return MyIntType
@property
def bson_type(self):
return int
def transform_python(self, value):
return value.x
def transform_bson(self, value):
return MyIntType(value)
class MyStrCodec(TypeCodec):
@property
def python_type(self):
return MyStrType
@property
def bson_type(self):
return str
def transform_python(self, value):
return value.x
def transform_bson(self, value):
return MyStrType(value)
def fallback_encoder(value):
return value
cls.types = (MyIntType, MyStrType)
cls.codecs = (MyIntCodec, MyStrCodec)
cls.fallback_encoder = fallback_encoder
def test_simple(self):
codec_instances = [codec() for codec in self.codecs]
def assert_proper_initialization(type_registry, codec_instances):
self.assertEqual(type_registry._encoder_map, {
self.types[0]: codec_instances[0].transform_python,
self.types[1]: codec_instances[1].transform_python})
self.assertEqual(type_registry._decoder_map, {
int: codec_instances[0].transform_bson,
str: codec_instances[1].transform_bson})
self.assertEqual(
type_registry._fallback_encoder, self.fallback_encoder)
type_registry = TypeRegistry(codec_instances, self.fallback_encoder)
assert_proper_initialization(type_registry, codec_instances)
type_registry = TypeRegistry(
fallback_encoder=self.fallback_encoder, type_codecs=codec_instances)
assert_proper_initialization(type_registry, codec_instances)
# Ensure codec list held by the type registry doesn't change if we
# mutate the initial list.
codec_instances_copy = list(codec_instances)
codec_instances.pop(0)
self.assertListEqual(
type_registry._TypeRegistry__type_codecs, codec_instances_copy)
def test_simple_separate_codecs(self):
class MyIntEncoder(TypeEncoder):
python_type = self.types[0]
def transform_python(self, value):
return value.x
class MyIntDecoder(TypeDecoder):
bson_type = int
def transform_bson(self, value):
return self.types[0](value)
codec_instances = [MyIntDecoder(), MyIntEncoder()]
type_registry = TypeRegistry(codec_instances)
self.assertEqual(
type_registry._encoder_map,
{MyIntEncoder.python_type: codec_instances[1].transform_python})
self.assertEqual(
type_registry._decoder_map,
{MyIntDecoder.bson_type: codec_instances[0].transform_bson})
def test_initialize_fail(self):
err_msg = ("Expected an instance of TypeEncoder, TypeDecoder, "
"or TypeCodec, got .* instead")
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry(self.codecs)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry([type('AnyType', (object,), {})()])
err_msg = "fallback_encoder %r is not a callable" % (True,)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry([], True)
err_msg = "fallback_encoder %r is not a callable" % ('hello',)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry(fallback_encoder='hello')
def test_type_registry_repr(self):
codec_instances = [codec() for codec in self.codecs]
type_registry = TypeRegistry(codec_instances)
r = ("TypeRegistry(type_codecs=%r, fallback_encoder=%r)" % (
codec_instances, None))
self.assertEqual(r, repr(type_registry))
def test_type_registry_eq(self):
codec_instances = [codec() for codec in self.codecs]
self.assertEqual(
TypeRegistry(codec_instances), TypeRegistry(codec_instances))
codec_instances_2 = [codec() for codec in self.codecs]
self.assertNotEqual(
TypeRegistry(codec_instances), TypeRegistry(codec_instances_2))
if __name__ == "__main__":
unittest.main()