From 85e80bcc8b2c4d30e656af863d40884aac93205c Mon Sep 17 00:00:00 2001 From: "A. Jesse Jiryu Davis" Date: Sat, 29 Jul 2017 16:01:19 -0400 Subject: [PATCH] PYTHON-1308 - Finish deprecating SON manipulators --- doc/examples/custom_type.rst | 251 ----------------------------------- doc/examples/index.rst | 1 - pymongo/cursor.py | 3 + pymongo/database.py | 34 ++++- pymongo/son_manipulator.py | 29 ++-- test/test_legacy_api.py | 6 +- test/test_son_manipulator.py | 11 ++ 7 files changed, 65 insertions(+), 270 deletions(-) delete mode 100644 doc/examples/custom_type.rst diff --git a/doc/examples/custom_type.rst b/doc/examples/custom_type.rst deleted file mode 100644 index d7115c4b1..000000000 --- a/doc/examples/custom_type.rst +++ /dev/null @@ -1,251 +0,0 @@ -Custom Type Example -=================== - -.. warning:: The following examples document a deprecated feature. The - :class:`~pymongo.son_manipulator.SONManipulator` API has limitations as a - technique for transforming your data. Instead, it is more flexible and - straightforward to transform outgoing documents in your own code before - passing them to PyMongo, and transform incoming documents after receiving - them from PyMongo. - - Thus the :meth:`~pymongo.database.Database.add_son_manipulator` method is - deprecated. PyMongo 3's new CRUD API does **not** apply SON manipulators to - documents passed to :meth:`~pymongo.collection.Collection.bulk_write`, - :meth:`~pymongo.collection.Collection.insert_one`, - :meth:`~pymongo.collection.Collection.insert_many`, - :meth:`~pymongo.collection.Collection.update_one`, or - :meth:`~pymongo.collection.Collection.update_many`. SON manipulators are - **not** applied to documents returned by the new methods - :meth:`~pymongo.collection.Collection.find_one_and_delete`, - :meth:`~pymongo.collection.Collection.find_one_and_replace`, and - :meth:`~pymongo.collection.Collection.find_one_and_update`. - - -This is an example of using a custom type with PyMongo. The example -here is a bit contrived, but shows how to use a -:class:`~pymongo.son_manipulator.SONManipulator` to manipulate -documents as they are saved or retrieved from MongoDB. More -specifically, it shows a couple different mechanisms for working with -custom datatypes in PyMongo. - -Setup ------ - -We'll start by getting a clean database to use for the example: - -.. doctest:: - - >>> from pymongo.mongo_client import MongoClient - >>> client = MongoClient() - >>> client.drop_database("custom_type_example") - >>> db = client.custom_type_example - -Since the purpose of the example is to demonstrate working with custom -types, we'll need a custom datatype to use. Here we define the aptly -named :class:`Custom` class, which has a single method, :meth:`x`: - -.. doctest:: - - >>> class Custom(object): - ... def __init__(self, x): - ... self.__x = x - ... - ... def x(self): - ... return self.__x - ... - >>> foo = Custom(10) - >>> foo.x() - 10 - -When we try to save an instance of :class:`Custom` with PyMongo, we'll -get an :class:`~bson.errors.InvalidDocument` exception: - -.. doctest:: - - >>> db.test.insert({"custom": Custom(5)}) - Traceback (most recent call last): - InvalidDocument: cannot convert value of type to bson - -Manual Encoding ---------------- - -One way to work around this is to manipulate our data into something -we *can* save with PyMongo. To do so we define two methods, -:meth:`encode_custom` and :meth:`decode_custom`: - -.. doctest:: - - >>> def encode_custom(custom): - ... return {"_type": "custom", "x": custom.x()} - ... - >>> def decode_custom(document): - ... assert document["_type"] == "custom" - ... return Custom(document["x"]) - ... - -We can now manually encode and decode :class:`Custom` instances and -use them with PyMongo: - -.. doctest:: - - >>> import pprint - >>> db.test.insert({"custom": encode_custom(Custom(5))}) - ObjectId('...') - >>> pprint.pprint(db.test.find_one()) - {u'_id': ObjectId('...'), - u'custom': {u'_type': u'custom', u'x': 5}} - >>> decode_custom(db.test.find_one()["custom"]) - - >>> decode_custom(db.test.find_one()["custom"]).x() - 5 - -Automatic Encoding and Decoding -------------------------------- - -Needless to say, that was a little unwieldy. Let's make this a bit -more seamless by creating a new -:class:`~pymongo.son_manipulator.SONManipulator`. -:class:`~pymongo.son_manipulator.SONManipulator` instances allow you -to specify transformations to be applied automatically by PyMongo: - -.. doctest:: - - >>> from pymongo.son_manipulator import SONManipulator - >>> class Transform(SONManipulator): - ... def transform_incoming(self, son, collection): - ... for (key, value) in son.items(): - ... if isinstance(value, Custom): - ... son[key] = encode_custom(value) - ... elif isinstance(value, dict): # Make sure we recurse into sub-docs - ... son[key] = self.transform_incoming(value, collection) - ... return son - ... - ... def transform_outgoing(self, son, collection): - ... for (key, value) in son.items(): - ... if isinstance(value, dict): - ... if "_type" in value and value["_type"] == "custom": - ... son[key] = decode_custom(value) - ... else: # Again, make sure to recurse into sub-docs - ... son[key] = self.transform_outgoing(value, collection) - ... return son - ... - -Now we add our manipulator to the :class:`~pymongo.database.Database`: - -.. doctest:: - - >>> db.add_son_manipulator(Transform()) - -After doing so we can save and restore :class:`Custom` instances seamlessly: - -.. doctest:: - - >>> db.test.remove() # remove whatever has already been saved - {...} - >>> db.test.insert({"custom": Custom(5)}) - ObjectId('...') - >>> pprint.pprint(db.test.find_one()) - {u'_id': ObjectId('...'), - u'custom': } - >>> db.test.find_one()["custom"].x() - 5 - -If we get a new :class:`~pymongo.database.Database` instance we'll -clear out the :class:`~pymongo.son_manipulator.SONManipulator` -instance we added: - -.. doctest:: - - >>> db = client.custom_type_example - -This allows us to see what was actually saved to the database: - -.. doctest:: - - >>> pprint.pprint(db.test.find_one()) - {u'_id': ObjectId('...'), - u'custom': {u'_type': u'custom', u'x': 5}} - -which is the same format that we encode to with our -:meth:`encode_custom` method! - -Binary Encoding ---------------- - -We can take this one step further by encoding to binary, using a user -defined subtype. This allows us to identify what to decode without -resorting to tricks like the ``_type`` field used above. - -We'll start by defining the methods :meth:`to_binary` and -:meth:`from_binary`, which convert :class:`Custom` instances to and -from :class:`~bson.binary.Binary` instances: - -.. note:: You could just pickle the instance and save that. What we do - here is a little more lightweight. - -.. doctest:: - - >>> from bson.binary import Binary - >>> def to_binary(custom): - ... return Binary(str(custom.x()).encode(), 128) - ... - >>> def from_binary(binary): - ... return Custom(int(binary)) - ... - -Next we'll create another -:class:`~pymongo.son_manipulator.SONManipulator`, this time using the -methods we just defined: - -.. doctest:: - - >>> class TransformToBinary(SONManipulator): - ... def transform_incoming(self, son, collection): - ... for (key, value) in son.items(): - ... if isinstance(value, Custom): - ... son[key] = to_binary(value) - ... elif isinstance(value, dict): - ... son[key] = self.transform_incoming(value, collection) - ... return son - ... - ... def transform_outgoing(self, son, collection): - ... for (key, value) in son.items(): - ... if isinstance(value, Binary) and value.subtype == 128: - ... son[key] = from_binary(value) - ... elif isinstance(value, dict): - ... son[key] = self.transform_outgoing(value, collection) - ... return son - ... - -Now we'll empty the :class:`~pymongo.database.Database` and add the -new manipulator: - -.. doctest:: - - >>> db.test.remove() - {...} - >>> db.add_son_manipulator(TransformToBinary()) - -After doing so we can save and restore :class:`Custom` instances -seamlessly: - -.. doctest:: - - >>> db.test.insert({"custom": Custom(5)}) - ObjectId('...') - >>> pprint.pprint(db.test.find_one()) - {u'_id': ObjectId('...'), - u'custom': } - >>> db.test.find_one()["custom"].x() - 5 - -We can see what's actually being saved to the database (and verify -that it is using a :class:`~bson.binary.Binary` instance) by -clearing out the manipulators and repeating our -:meth:`~pymongo.collection.Collection.find_one`: - -.. doctest:: - - >>> db = client.custom_type_example - >>> pprint.pprint(db.test.find_one()) - {u'_id': ObjectId('...'), u'custom': Binary('5', 128)} diff --git a/doc/examples/index.rst b/doc/examples/index.rst index c873092bc..c33024caf 100644 --- a/doc/examples/index.rst +++ b/doc/examples/index.rst @@ -21,7 +21,6 @@ MongoDB, you can start it like so: collations copydb bulk - custom_type datetimes geo gevent diff --git a/pymongo/cursor.py b/pymongo/cursor.py index 11137bf8a..0563b4c63 100644 --- a/pymongo/cursor.py +++ b/pymongo/cursor.py @@ -140,6 +140,9 @@ class Cursor(object): warnings.warn("the 'modifiers' parameter is deprecated", DeprecationWarning, stacklevel=2) validate_is_mapping("modifiers", modifiers) + if manipulate: + warnings.warn("the 'manipulate' parameter is deprecated", + DeprecationWarning, stacklevel=2) if not isinstance(batch_size, integer_types): raise TypeError("batch_size must be an integer") if batch_size < 0: diff --git a/pymongo/database.py b/pymongo/database.py index b997eddc2..cc7b25822 100644 --- a/pymongo/database.py +++ b/pymongo/database.py @@ -160,39 +160,61 @@ class Database(common.BaseObject): @property def incoming_manipulators(self): - """All incoming SON manipulators installed on this instance. + """**DEPRECATED**: All incoming SON manipulators. + + .. versionchanged:: 3.5 + Deprecated. .. versionadded:: 2.0 """ + warnings.warn("Database.incoming_manipulators() is deprecated", + DeprecationWarning, stacklevel=2) + return [manipulator.__class__.__name__ for manipulator in self.__incoming_manipulators] @property def incoming_copying_manipulators(self): - """All incoming SON copying manipulators installed on this instance. + """**DEPRECATED**: All incoming SON copying manipulators. + + .. versionchanged:: 3.5 + Deprecated. .. versionadded:: 2.0 """ + warnings.warn("Database.incoming_copying_manipulators() is deprecated", + DeprecationWarning, stacklevel=2) + return [manipulator.__class__.__name__ for manipulator in self.__incoming_copying_manipulators] @property def outgoing_manipulators(self): - """List all outgoing SON manipulators - installed on this instance. + """**DEPRECATED**: All outgoing SON manipulators. + + .. versionchanged:: 3.5 + Deprecated. .. versionadded:: 2.0 """ + warnings.warn("Database.outgoing_manipulators() is deprecated", + DeprecationWarning, stacklevel=2) + return [manipulator.__class__.__name__ for manipulator in self.__outgoing_manipulators] @property def outgoing_copying_manipulators(self): - """List all outgoing SON copying manipulators - installed on this instance. + """**DEPRECATED**: All outgoing SON copying manipulators. + + .. versionchanged:: 3.5 + Deprecated. .. versionadded:: 2.0 """ + warnings.warn("Database.outgoing_copying_manipulators() is deprecated", + DeprecationWarning, stacklevel=2) + return [manipulator.__class__.__name__ for manipulator in self.__outgoing_copying_manipulators] diff --git a/pymongo/son_manipulator.py b/pymongo/son_manipulator.py index f1dab33cb..1515a4f31 100644 --- a/pymongo/son_manipulator.py +++ b/pymongo/son_manipulator.py @@ -12,11 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Manipulators that can edit SON objects as they enter and exit a database. +"""**DEPRECATED**: Manipulators that can edit SON objects as they enter and exit +a database. -New manipulators should be defined as subclasses of SONManipulator and can be -installed on a database by calling -`pymongo.database.Database.add_son_manipulator`.""" +The :class:`~pymongo.son_manipulator.SONManipulator` API has limitations as a +technique for transforming your data. Instead, it is more flexible and +straightforward to transform outgoing documents in your own code before passing +them to PyMongo, and transform incoming documents after receiving them from +PyMongo. SON Manipulators will be removed from PyMongo in 4.0. + +PyMongo does **not** apply SON manipulators to documents passed to +the modern methods :meth:`~pymongo.collection.Collection.bulk_write`, +:meth:`~pymongo.collection.Collection.insert_one`, +:meth:`~pymongo.collection.Collection.insert_many`, +:meth:`~pymongo.collection.Collection.update_one`, or +:meth:`~pymongo.collection.Collection.update_many`. SON manipulators are +**not** applied to documents returned by the modern methods +:meth:`~pymongo.collection.Collection.find_one_and_delete`, +:meth:`~pymongo.collection.Collection.find_one_and_replace`, and +:meth:`~pymongo.collection.Collection.find_one_and_update`. +""" import collections @@ -175,9 +190,3 @@ class AutoReference(SONManipulator): return object return transform_dict(SON(son)) - -# TODO make a generic translator for custom types. Take encode, decode, -# should_encode and should_decode functions and just encode and decode where -# necessary. See examples/custom_type.py for where this would be useful. -# Alternatively it could take a should_encode, to_binary, from_binary and -# binary subtype. diff --git a/test/test_legacy_api.py b/test/test_legacy_api.py index 362f6906e..8f559c430 100644 --- a/test/test_legacy_api.py +++ b/test/test_legacy_api.py @@ -47,7 +47,8 @@ from pymongo.son_manipulator import (AutoReference, from pymongo.write_concern import WriteConcern from test import client_context, qcheck, unittest, SkipTest from test.test_client import IntegrationTest -from test.utils import (joinall, +from test.utils import (ignore_deprecations, + joinall, rs_or_single_client, wait_until) @@ -1196,7 +1197,8 @@ class TestLegacy(IntegrationTest): for name in db.incoming_copying_manipulators: self.assertTrue(name in ('ObjectIdShuffler', 'AutoReference')) self.assertEqual([], db.outgoing_manipulators) - self.assertEqual(['AutoReference'], db.outgoing_copying_manipulators) + self.assertEqual(['AutoReference'], + db.outgoing_copying_manipulators) def test_ensure_index(self): db = self.db diff --git a/test/test_son_manipulator.py b/test/test_son_manipulator.py index a4192c602..5117ca81a 100644 --- a/test/test_son_manipulator.py +++ b/test/test_son_manipulator.py @@ -16,6 +16,8 @@ """ import sys +import warnings + sys.path[0:0] = [""] from bson.son import SON @@ -31,10 +33,19 @@ class TestSONManipulator(unittest.TestCase): @classmethod def setUpClass(cls): + cls.warn_context = warnings.catch_warnings() + cls.warn_context.__enter__() + warnings.simplefilter("ignore", DeprecationWarning) + client = MongoClient( client_context.host, client_context.port, connect=False) cls.db = client.pymongo_test + @classmethod + def tearDownClass(cls): + cls.warn_context.__exit__() + cls.warn_context = None + def test_basic(self): manip = SONManipulator() collection = self.db.test