diff --git a/bson/codec_options.py b/bson/codec_options.py index 53331832c..9631b1fbf 100644 --- a/bson/codec_options.py +++ b/bson/codec_options.py @@ -16,9 +16,10 @@ import datetime +from abc import abstractmethod from collections import namedtuple -from bson.py3compat import abc, string_type +from bson.py3compat import ABC, abc, abstractproperty, string_type from bson.binary import (ALL_UUID_REPRESENTATIONS, PYTHON_LEGACY, UUID_REPRESENTATION_NAMES) @@ -33,33 +34,53 @@ def _raw_document_class(document_class): return marker == _RAW_BSON_DOCUMENT_MARKER -class TypeCodecBase(object): +class TypeEncoder(ABC): + """Base class for defining type codec classes which describe how a + custom type can be transformed to one of the types BSON understands. + + Codec classes must implement the ``python_type`` attribute, and the + ``transform_python`` method to support encoding. + """ + @abstractproperty + def python_type(self): + """The Python type to be converted into something serializable.""" + pass + + @abstractmethod + def transform_python(self, value): + """Convert the given Python object into something serializable.""" + pass + + +class TypeDecoder(ABC): + """Base class for defining type codec classes which describe how a + BSON type can be transformed to a custom type. + + Codec classes must implement the ``bson_type`` attribute, and the + ``transform_bson`` method to support decoding. + """ + @abstractproperty + def bson_type(self): + """The BSON type to be converted into our own type.""" + pass + + @abstractmethod + def transform_bson(self, value): + """Convert the given BSON value into our own type.""" + pass + + +class TypeCodec(TypeEncoder, TypeDecoder): """Base class for defining type codec classes which describe how a custom type can be transformed to/from one of the types BSON already understands, and can encode/decode. - Codec classes must implement the ``python_type`` property, and the - ``transform_python`` method to support encoding, or the ``bson_type`` - property and ``transform_bson`` method to support decoding. Note that a - single codec class may support both encoding and decoding. + Codec classes must implement the ``python_type`` attribute, and the + ``transform_python`` method to support encoding, as well as the + ``bson_type`` attribute, and the ``transform_bson`` method to support + decoding. """ - @property - def python_type(self): - """The Python type to be converted into something serializable.""" - raise NotImplementedError - - @property - def bson_type(self): - """The BSON type to be converted into our own type.""" - raise NotImplementedError - - def transform_bson(self, value): - """Convert the given BSON value into our own type.""" - raise NotImplementedError - - def transform_python(self, value): - """Convert the given Python object into something serializable.""" - raise NotImplementedError + pass class TypeRegistry(object): @@ -95,23 +116,18 @@ class TypeRegistry(object): fallback_encoder)) for codec in self.__type_codecs: - if not isinstance(codec, TypeCodecBase): + is_valid_codec = False + if isinstance(codec, TypeEncoder): + is_valid_codec = True + self._encoder_map[codec.python_type] = codec.transform_python + if isinstance(codec, TypeDecoder): + is_valid_codec = True + self._decoder_map[codec.bson_type] = codec.transform_bson + if not is_valid_codec: raise TypeError( - "Expected an instance of %s, got %r instead" % ( - TypeCodecBase.__name__, codec)) - try: - python_type = codec.python_type - except NotImplementedError: - pass - else: - self._encoder_map[python_type] = codec.transform_python - - try: - bson_type = codec.bson_type - except NotImplementedError: - pass - else: - self._decoder_map[bson_type] = codec.transform_bson + "Expected an instance of %s, %s, or %s, got %r instead" % ( + TypeEncoder.__name__, TypeDecoder.__name__, + TypeCodec.__name__, codec)) def __repr__(self): return ('%s(type_codecs=%r, fallback_encoder=%r)' % ( diff --git a/bson/py3compat.py b/bson/py3compat.py index 0b7378eec..84d1ea00f 100644 --- a/bson/py3compat.py +++ b/bson/py3compat.py @@ -22,8 +22,12 @@ if PY3: import codecs import collections.abc as abc import _thread as thread + from abc import ABC, abstractmethod from io import BytesIO as StringIO + def abstractproperty(func): + return property(abstractmethod(func)) + MAXSIZE = sys.maxsize imap = map @@ -60,6 +64,7 @@ if PY3: else: import collections as abc import thread + from abc import ABCMeta, abstractproperty from itertools import imap try: @@ -67,6 +72,8 @@ else: except ImportError: from StringIO import StringIO + ABC = ABCMeta('ABC', (object,), {}) + MAXSIZE = sys.maxint def b(s): diff --git a/doc/examples/custom_type.rst b/doc/examples/custom_type.rst index 2311baca0..2cdf22cb3 100644 --- a/doc/examples/custom_type.rst +++ b/doc/examples/custom_type.rst @@ -2,7 +2,7 @@ Custom Type Example =================== This is an example of using a custom type with PyMongo. The example here shows -how to subclass :class:`~bson.codec_options.TypeCodecBase` to write a type +how to subclass :class:`~bson.codec_options.TypeCodec` to write a type codec, which is used to populate a :class:`~bson.codec_options.TypeRegistry`. The type registry can then be used to create a custom-type-aware :class:`~pymongo.collection.Collection`. Read and write operations @@ -51,39 +51,39 @@ The Type Codec In order to encode custom types, we must first define a **type codec** for our type. A type codec describes how an instance of a custom type can be -*transformed* to/from one of the types :mod:`~bson` already understands, and -can encode/decode. Type codecs must inherit from -:class:`~bson.codec_options.TypeCodecBase`. In order to encode a custom type, -a codec must implement the ``python_type`` property and the -``transform_python`` method. Similarly, in order to decode a custom type, -a codec must implement the ``bson_type`` property and the ``transform_bson`` -method. Note that a type codec need not support both encoding and decoding. +*transformed* to and/or from one of the types :mod:`~bson` already understands. +Depending on the desired functionality, users must choose from the following +base classes when defining type codecs: + +* :class:`~bson.codec_options.TypeEncoder`: subclass this to define a codec that + encodes a custom Python type to a known BSON type. Users must implement the + ``python_type`` property/attribute and the ``transform_python`` method. +* :class:`~bson.codec_options.TypeDecoder`: subclass this to define a codec that + decodes a specified BSON type into a custom Python type. Users must implement + the ``bson_type`` property/attribute and the ``transform_bson`` method. +* :class:`~bson.codec_options.TypeCodec`: subclass this to define a codec that + can both encode from and decode to a custom type. Users must implement the + ``python_type`` and ``bson_type`` properties/attributes, as well as the + ``transform_python`` and ``transform_bson`` methods. The type codec for our custom type simply needs to define how a :py:class:`~decimal.Decimal` instance can be converted into a -:class:`~bson.decimal128.Decimal128` instance and vice-versa: +:class:`~bson.decimal128.Decimal128` instance and vice-versa. Since we are +interested in both encoding and decoding our custom type, we use the +``TypeCodec`` base class to define our codec: .. doctest:: >>> from bson.decimal128 import Decimal128 - >>> from bson.codec_options import TypeCodecBase - >>> class DecimalCodec(TypeCodecBase): - ... @property - ... def python_type(self): - ... """The Python type acted upon by this type codec.""" - ... return Decimal - ... + >>> from bson.codec_options import TypeCodec + >>> class DecimalCodec(TypeCodec): + ... python_type = Decimal # the Python type acted upon by this type codec + ... bson_type = Decimal128 # the BSON type acted upon by this type codec ... def transform_python(self, value): ... """Function that transforms a custom type value into a type ... that BSON can encode.""" ... return Decimal128(value) - ... - ... @property - ... def bson_type(self): - ... """The BSON type acted upon by this type codec.""" - ... return Decimal128 - ... ... def transform_bson(self, value): ... """Function that transforms a vanilla BSON type value into our ... custom type.""" diff --git a/test/test_bson.py b/test/test_bson.py index 9684eec08..e2af46e59 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -34,14 +34,13 @@ from bson import (BSON, Regex) from bson.binary import Binary, UUIDLegacy from bson.code import Code -from bson.codec_options import CodecOptions, TypeCodecBase, TypeRegistry +from bson.codec_options import CodecOptions from bson.int64 import Int64 from bson.objectid import ObjectId from bson.dbref import DBRef from bson.py3compat import abc, iteritems, PY3, StringIO, text_type from bson.son import SON from bson.timestamp import Timestamp -from bson.tz_util import FixedOffset from bson.errors import (InvalidBSON, InvalidDocument, InvalidStringData) @@ -906,121 +905,6 @@ class TestBSON(unittest.TestCase): BSON.encode({"_id": {'$oid': "52d0b971b3ba219fdeb4170e"}}) -class TestTypeRegistry(unittest.TestCase): - @classmethod - def setUpClass(cls): - class MyIntType(object): - def __init__(self, x): - assert isinstance(x, int) - self.x = x - - class MyStrType(object): - def __init__(self, x): - assert isinstance(x, str) - self.x = x - - class MyIntCodec(TypeCodecBase): - @property - def python_type(self): - return MyIntType - - @property - def bson_type(self): - return int - - def transform_python(self, value): - return value.x - - def transform_bson(self, value): - return MyIntType(value) - - class MyStrCodec(TypeCodecBase): - @property - def python_type(self): - return MyStrType - - @property - def bson_type(self): - return str - - def transform_python(self, value): - return value.x - - def transform_bson(self, value): - return MyStrType(value) - - def fallback_encoder(value): - return value - - cls.types = (MyIntType, MyStrType) - cls.codecs = (MyIntCodec, MyStrCodec) - cls.fallback_encoder = fallback_encoder - - def test_simple(self): - codec_instances = [codec() for codec in self.codecs] - def assert_proper_initialization(type_registry, codec_instances): - self.assertEqual(type_registry._encoder_map, { - self.types[0]: codec_instances[0].transform_python, - self.types[1]: codec_instances[1].transform_python}) - self.assertEqual(type_registry._decoder_map, { - int: codec_instances[0].transform_bson, - str: codec_instances[1].transform_bson}) - self.assertEqual( - type_registry._fallback_encoder, self.fallback_encoder) - - type_registry = TypeRegistry(codec_instances, self.fallback_encoder) - assert_proper_initialization(type_registry, codec_instances) - - type_registry = TypeRegistry( - fallback_encoder=self.fallback_encoder, type_codecs=codec_instances) - assert_proper_initialization(type_registry, codec_instances) - - # Ensure codec list held by the type registry doesn't change if we - # mutate the initial list. - codec_instances_copy = list(codec_instances) - codec_instances.pop(0) - self.assertListEqual( - type_registry._TypeRegistry__type_codecs, codec_instances_copy) - - def test_initialize_fail(self): - err_msg = "Expected an instance of TypeCodecBase, got .* instead" - with self.assertRaisesRegex(TypeError, err_msg): - TypeRegistry(self.codecs) - - with self.assertRaisesRegex(TypeError, err_msg): - TypeRegistry([type('AnyType', (object,), {})()]) - - err_msg = "fallback_encoder %r is not a callable" % (True,) - with self.assertRaisesRegex(TypeError, err_msg): - TypeRegistry([], True) - - err_msg = "fallback_encoder %r is not a callable" % ('hello',) - with self.assertRaisesRegex(TypeError, err_msg): - TypeRegistry(fallback_encoder='hello') - - def test_not_implemented(self): - type_registry = TypeRegistry([type("codec1", (TypeCodecBase, ), {})(), - type("codec2", (TypeCodecBase, ), {})()]) - self.assertEqual(type_registry._encoder_map, {}) - self.assertEqual(type_registry._decoder_map, {}) - - def test_type_registry_repr(self): - codec_instances = [codec() for codec in self.codecs] - type_registry = TypeRegistry(codec_instances) - r = ("TypeRegistry(type_codecs=%r, fallback_encoder=%r)" % ( - codec_instances, None)) - self.assertEqual(r, repr(type_registry)) - - def test_type_registry_eq(self): - codec_instances = [codec() for codec in self.codecs] - self.assertEqual( - TypeRegistry(codec_instances), TypeRegistry(codec_instances)) - - codec_instances_2 = [codec() for codec in self.codecs] - self.assertNotEqual( - TypeRegistry(codec_instances), TypeRegistry(codec_instances_2)) - - class TestCodecOptions(unittest.TestCase): def test_document_class(self): self.assertRaises(TypeError, CodecOptions, document_class=object) diff --git a/test/test_bson_custom_types.py b/test/test_bson_custom_types.py index 16d07031a..0feb04527 100644 --- a/test/test_bson_custom_types.py +++ b/test/test_bson_custom_types.py @@ -28,35 +28,36 @@ from bson import (BSON, decode_iter, _dict_to_bson, _bson_to_dict) -from bson.codec_options import CodecOptions, TypeCodecBase, TypeRegistry +from bson.codec_options import (CodecOptions, TypeCodec, TypeDecoder, + TypeEncoder, TypeRegistry) from bson.errors import InvalidDocument from test import unittest -class DecimalCodec(TypeCodecBase): - @property - def bson_type(self): - return Decimal128 - +class DecimalEncoder(TypeEncoder): @property def python_type(self): return Decimal - def transform_bson(self, value): - return value.to_decimal() - def transform_python(self, value): return Decimal128(value) -class TestCustomPythonTypeToBSON(unittest.TestCase): - @classmethod - def setUpClass(cls): - type_registry = TypeRegistry((DecimalCodec(),)) - codec_options = CodecOptions(type_registry=type_registry) - cls.codecopts = codec_options +class DecimalDecoder(TypeDecoder): + @property + def bson_type(self): + return Decimal128 + def transform_bson(self, value): + return value.to_decimal() + + +class DecimalCodec(DecimalDecoder, DecimalEncoder): + pass + + +class CustomTypeTests(object): def test_encode_decode_roundtrip(self): document = {'average': Decimal('56.47')} bsonbytes = BSON().encode(document, codec_options=self.codecopts) @@ -114,6 +115,23 @@ class TestCustomPythonTypeToBSON(unittest.TestCase): fileobj.close() +class TestCustomPythonTypeToBSONMonolithicCodec(CustomTypeTests, + unittest.TestCase): + @classmethod + def setUpClass(cls): + type_registry = TypeRegistry((DecimalCodec(),)) + codec_options = CodecOptions(type_registry=type_registry) + cls.codecopts = codec_options + + +class TestCustomPythonTypeToBSONMultiplexedCodec(CustomTypeTests, + unittest.TestCase): + @classmethod + def setUpClass(cls): + type_registry = TypeRegistry((DecimalEncoder(), DecimalDecoder())) + codec_options = CodecOptions(type_registry=type_registry) + cls.codecopts = codec_options + class TestFallbackEncoder(unittest.TestCase): def _get_codec_options(self, fallback_encoder): @@ -160,5 +178,175 @@ class TestFallbackEncoder(unittest.TestCase): BSON().encode(document, codec_options=codecopts) +class TestTypeEnDeCodecs(unittest.TestCase): + def test_instantiation(self): + msg = "Can't instantiate abstract class .* with abstract methods .*" + def run_test(base, attrs, fail): + codec = type('testcodec', (base,), attrs) + if fail: + with self.assertRaisesRegex(TypeError, msg): + codec() + else: + codec() + + run_test(TypeEncoder, {'python_type': int,}, fail=True) + run_test(TypeEncoder, {'transform_python': lambda s, x: x}, fail=True) + run_test(TypeEncoder, {'transform_python': lambda s, x: x, + 'python_type': int}, fail=False) + + run_test(TypeDecoder, {'bson_type': Decimal128, }, fail=True) + run_test(TypeDecoder, {'transform_bson': lambda s, x: x}, fail=True) + run_test(TypeDecoder, {'transform_bson': lambda s, x: x, + 'bson_type': Decimal128}, fail=False) + + run_test(TypeCodec, {'bson_type': Decimal128, + 'python_type': int}, fail=True) + run_test(TypeCodec, {'transform_bson': lambda s, x: x, + 'transform_python': lambda s, x: x}, fail=True) + run_test(TypeCodec, {'python_type': int, + 'transform_python': lambda s, x: x, + 'transform_bson': lambda s, x: x, + 'bson_type': Decimal128}, fail=False) + + def test_type_checks(self): + self.assertTrue(issubclass(TypeCodec, TypeEncoder)) + self.assertTrue(issubclass(TypeCodec, TypeDecoder)) + self.assertFalse(issubclass(TypeDecoder, TypeEncoder)) + self.assertFalse(issubclass(TypeEncoder, TypeDecoder)) + + +class TestTypeRegistry(unittest.TestCase): + @classmethod + def setUpClass(cls): + class MyIntType(object): + def __init__(self, x): + assert isinstance(x, int) + self.x = x + + class MyStrType(object): + def __init__(self, x): + assert isinstance(x, str) + self.x = x + + class MyIntCodec(TypeCodec): + @property + def python_type(self): + return MyIntType + + @property + def bson_type(self): + return int + + def transform_python(self, value): + return value.x + + def transform_bson(self, value): + return MyIntType(value) + + class MyStrCodec(TypeCodec): + @property + def python_type(self): + return MyStrType + + @property + def bson_type(self): + return str + + def transform_python(self, value): + return value.x + + def transform_bson(self, value): + return MyStrType(value) + + def fallback_encoder(value): + return value + + cls.types = (MyIntType, MyStrType) + cls.codecs = (MyIntCodec, MyStrCodec) + cls.fallback_encoder = fallback_encoder + + def test_simple(self): + codec_instances = [codec() for codec in self.codecs] + def assert_proper_initialization(type_registry, codec_instances): + self.assertEqual(type_registry._encoder_map, { + self.types[0]: codec_instances[0].transform_python, + self.types[1]: codec_instances[1].transform_python}) + self.assertEqual(type_registry._decoder_map, { + int: codec_instances[0].transform_bson, + str: codec_instances[1].transform_bson}) + self.assertEqual( + type_registry._fallback_encoder, self.fallback_encoder) + + type_registry = TypeRegistry(codec_instances, self.fallback_encoder) + assert_proper_initialization(type_registry, codec_instances) + + type_registry = TypeRegistry( + fallback_encoder=self.fallback_encoder, type_codecs=codec_instances) + assert_proper_initialization(type_registry, codec_instances) + + # Ensure codec list held by the type registry doesn't change if we + # mutate the initial list. + codec_instances_copy = list(codec_instances) + codec_instances.pop(0) + self.assertListEqual( + type_registry._TypeRegistry__type_codecs, codec_instances_copy) + + def test_simple_separate_codecs(self): + class MyIntEncoder(TypeEncoder): + python_type = self.types[0] + + def transform_python(self, value): + return value.x + + class MyIntDecoder(TypeDecoder): + bson_type = int + + def transform_bson(self, value): + return self.types[0](value) + + codec_instances = [MyIntDecoder(), MyIntEncoder()] + type_registry = TypeRegistry(codec_instances) + + self.assertEqual( + type_registry._encoder_map, + {MyIntEncoder.python_type: codec_instances[1].transform_python}) + self.assertEqual( + type_registry._decoder_map, + {MyIntDecoder.bson_type: codec_instances[0].transform_bson}) + + def test_initialize_fail(self): + err_msg = ("Expected an instance of TypeEncoder, TypeDecoder, " + "or TypeCodec, got .* instead") + with self.assertRaisesRegex(TypeError, err_msg): + TypeRegistry(self.codecs) + + with self.assertRaisesRegex(TypeError, err_msg): + TypeRegistry([type('AnyType', (object,), {})()]) + + err_msg = "fallback_encoder %r is not a callable" % (True,) + with self.assertRaisesRegex(TypeError, err_msg): + TypeRegistry([], True) + + err_msg = "fallback_encoder %r is not a callable" % ('hello',) + with self.assertRaisesRegex(TypeError, err_msg): + TypeRegistry(fallback_encoder='hello') + + def test_type_registry_repr(self): + codec_instances = [codec() for codec in self.codecs] + type_registry = TypeRegistry(codec_instances) + r = ("TypeRegistry(type_codecs=%r, fallback_encoder=%r)" % ( + codec_instances, None)) + self.assertEqual(r, repr(type_registry)) + + def test_type_registry_eq(self): + codec_instances = [codec() for codec in self.codecs] + self.assertEqual( + TypeRegistry(codec_instances), TypeRegistry(codec_instances)) + + codec_instances_2 = [codec() for codec in self.codecs] + self.assertNotEqual( + TypeRegistry(codec_instances), TypeRegistry(codec_instances_2)) + + if __name__ == "__main__": unittest.main()