diff --git a/doc/api/pymongo/collection.rst b/doc/api/pymongo/collection.rst index b40e341d2..49e32383a 100644 --- a/doc/api/pymongo/collection.rst +++ b/doc/api/pymongo/collection.rst @@ -61,12 +61,12 @@ .. automethod:: drop .. automethod:: rename .. automethod:: options - .. automethod:: group .. automethod:: map_reduce .. automethod:: inline_map_reduce .. automethod:: parallel_scan .. automethod:: initialize_unordered_bulk_op .. automethod:: initialize_ordered_bulk_op + .. automethod:: group .. automethod:: insert(doc_or_docs, manipulate=True, check_keys=True, continue_on_error=False, **kwargs) .. automethod:: save(to_save, manipulate=True, check_keys=True, **kwargs) .. automethod:: update(spec, document, upsert=False, manipulate=False, multi=False, check_keys=True, **kwargs) diff --git a/doc/changelog.rst b/doc/changelog.rst index 5407c4863..e3428940c 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,15 +1,24 @@ Changelog ========= -Changes in Next Version ------------------------ +Changes in Version 3.5 +---------------------- -If a custom :class:`~bson.codec_options.CodecOptions` is passed to -:class:`RawBSONDocument`, its `document_class` must be :class:`RawBSONDocument`. +Highlights: -Increase the performance of -:meth:`~pymongo.mongo_client.MongoClient.database_names` by using the -`nameOnly` option for listDatabases. +- Increase the performance of + :meth:`~pymongo.mongo_client.MongoClient.database_names` by using the + `nameOnly` option for listDatabases when available. + +Changes and Deprecations: + +- Deprecated :meth:`~pymongo.collection.Collection.group`. The group command + was deprecated in MongoDB 3.4 and is expected to be removed in MongoDB 3.6. + Applications should use :meth:`~pymongo.collection.Collection.aggregate` + with the `$group` pipeline stage instead. +- If a custom :class:`~bson.codec_options.CodecOptions` is passed to + :class:`RawBSONDocument`, its `document_class` must be + :class:`RawBSONDocument`. Changes in Version 3.4 ---------------------- diff --git a/doc/examples/aggregation.rst b/doc/examples/aggregation.rst index 365c2191d..5a1cae565 100644 --- a/doc/examples/aggregation.rst +++ b/doc/examples/aggregation.rst @@ -177,34 +177,3 @@ specify a different database to store the result collection: u'timeMillis': ...} .. seealso:: The full list of options for MongoDB's `map reduce engine `_ - -Group ------ - -The :meth:`~pymongo.collection.Collection.group` method provides some of the -same functionality as SQL's GROUP BY. Simpler than a map reduce you need to -provide a key to group by, an initial value for the aggregation and a -reduce function. - -.. note:: Doesn't work with sharded MongoDB configurations, use aggregation or - map/reduce instead of group(). - -Here we are doing a simple group and count of the occurrences of ``x`` values: - -.. doctest:: - - >>> from bson.code import Code - >>> reducer = Code(""" - ... function(obj, prev){ - ... prev.count++; - ... } - ... """) - ... - >>> results = db.things.group(key={"x":1}, condition={}, initial={"count": 0}, reduce=reducer) - >>> for doc in results: - ... pprint.pprint(doc) - {u'count': 1.0, u'x': 1.0} - {u'count': 2.0, u'x': 2.0} - {u'count': 1.0, u'x': 3.0} - -.. seealso:: The full list of options for MongoDB's `group method `_ diff --git a/pymongo/collection.py b/pymongo/collection.py index 52bbe4df9..d2171afeb 100644 --- a/pymongo/collection.py +++ b/pymongo/collection.py @@ -1886,41 +1886,25 @@ class Collection(common.BaseObject): return CommandCursor( self, cursor, sock_info.address).batch_size(batch_size or 0) - # key and condition ought to be optional, but deprecation - # would be painful as argument order would have to change. def group(self, key, condition, initial, reduce, finalize=None, **kwargs): """Perform a query similar to an SQL *group by* operation. - Returns an array of grouped items. - - The `key` parameter can be: - - - ``None`` to use the entire document as a key. - - A :class:`list` of keys (each a :class:`basestring` - (:class:`str` in python 3)) to group by. - - A :class:`basestring` (:class:`str` in python 3), or - :class:`~bson.code.Code` instance containing a JavaScript - function to be applied to each document, returning the key - to group by. - - The :meth:`group` method obeys the :attr:`read_preference` of this - :class:`Collection`. - - :Parameters: - - `key`: fields to group by (see above description) - - `condition`: specification of rows to be - considered (as a :meth:`find` query specification) - - `initial`: initial value of the aggregation counter object - - `reduce`: aggregation function as a JavaScript string - - `finalize`: function to be called on each object in output list. - - `**kwargs` (optional): additional arguments to the group command - may be passed as keyword arguments to this helper method + **DEPRECATED** - The group command was deprecated in MongoDB 3.4. The + :meth:`~group` method is deprecated and will be removed in PyMongo 4.0. + Use :meth:`~aggregate` with the `$group` stage or :meth:`~map_reduce` + instead. + .. versionchanged:: 3.5 + Deprecated the group method. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 2.2 Removed deprecated argument: command """ + warnings.warn("The group method is deprecated and will be removed in " + "PyMongo 4.0. Use the aggregate method with the $group " + "stage or the map_reduce method instead.", + DeprecationWarning, stacklevel=2) group = {} if isinstance(key, string_type): group["$keyf"] = Code(key) diff --git a/test/test_collation.py b/test/test_collation.py index 7ce5dbc44..9b863fc01 100644 --- a/test/test_collation.py +++ b/test/test_collation.py @@ -15,6 +15,7 @@ """Test the collation module.""" import functools +import warnings from test import unittest, client_context from test.utils import EventListener, rs_or_single_client @@ -177,10 +178,12 @@ class TestCollation(unittest.TestCase): @raisesConfigurationErrorForOldMongoDB def test_group(self): - self.db.test.group('foo', {'foo': {'$gt': 42}}, {}, - 'function(a, b) { return a; }', - collation=self.collation) - self.assertCollationInLastCommand() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.db.test.group('foo', {'foo': {'$gt': 42}}, {}, + 'function(a, b) { return a; }', + collation=self.collation) + self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_map_reduce(self): diff --git a/test/test_collection.py b/test/test_collection.py index 5d6db1aa3..991a13b51 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -1641,101 +1641,6 @@ class TestCollection(IntegrationTest): "maxTimeAlwaysTimeOut", mode="off") - def test_group(self): - db = self.db - db.drop_collection("test") - - self.assertEqual([], - db.test.group([], {}, {"count": 0}, - "function (obj, prev) { prev.count++; }" - )) - - db.test.insert_many([{"a": 2}, {"b": 5}, {"a": 1}]) - - self.assertEqual([{"count": 3}], - db.test.group([], {}, {"count": 0}, - "function (obj, prev) { prev.count++; }" - )) - - self.assertEqual([{"count": 1}], - db.test.group([], {"a": {"$gt": 1}}, {"count": 0}, - "function (obj, prev) { prev.count++; }" - )) - - db.test.insert_one({"a": 2, "b": 3}) - - self.assertEqual([{"a": 2, "count": 2}, - {"a": None, "count": 1}, - {"a": 1, "count": 1}], - db.test.group(["a"], {}, {"count": 0}, - "function (obj, prev) { prev.count++; }" - )) - - # modifying finalize - self.assertEqual([{"a": 2, "count": 3}, - {"a": None, "count": 2}, - {"a": 1, "count": 2}], - db.test.group(["a"], {}, {"count": 0}, - "function (obj, prev) " - "{ prev.count++; }", - "function (obj) { obj.count++; }")) - - # returning finalize - self.assertEqual([2, 1, 1], - db.test.group(["a"], {}, {"count": 0}, - "function (obj, prev) " - "{ prev.count++; }", - "function (obj) { return obj.count; }")) - - # keyf - self.assertEqual([2, 2], - db.test.group("function (obj) { if (obj.a == 2) " - "{ return {a: true} }; " - "return {b: true}; }", {}, {"count": 0}, - "function (obj, prev) " - "{ prev.count++; }", - "function (obj) { return obj.count; }")) - - # no key - self.assertEqual([{"count": 4}], - db.test.group(None, {}, {"count": 0}, - "function (obj, prev) { prev.count++; }" - )) - - self.assertRaises(OperationFailure, db.test.group, - [], {}, {}, "5 ++ 5") - - def test_group_with_scope(self): - db = self.db - db.drop_collection("test") - db.test.insert_many([{"a": 1}, {"b": 1}]) - - reduce_function = "function (obj, prev) { prev.count += inc_value; }" - - self.assertEqual(2, db.test.group([], {}, {"count": 0}, - Code(reduce_function, - {"inc_value": 1}))[0]['count']) - self.assertEqual(4, db.test.group([], {}, {"count": 0}, - Code(reduce_function, - {"inc_value": 2}))[0]['count']) - - self.assertEqual(1, - db.test.group([], {}, {"count": 0}, - Code(reduce_function, - {"inc_value": 0.5}))[0]['count']) - - self.assertEqual(2, db.test.group( - [], {}, {"count": 0}, - Code(reduce_function, {"inc_value": 1}))[0]['count']) - - self.assertEqual(4, db.test.group( - [], {}, {"count": 0}, - Code(reduce_function, {"inc_value": 2}))[0]['count']) - - self.assertEqual(1, db.test.group( - [], {}, {"count": 0}, - Code(reduce_function, {"inc_value": 0.5}))[0]['count']) - def test_large_limit(self): db = self.db db.drop_collection("test_large_limit") diff --git a/test/test_common.py b/test/test_common.py index 6153d11d1..c2f7af4f3 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -147,22 +147,6 @@ class TestCommon(IntegrationTest): self.db.drop_collection("result") coll.drop() - # Test group - coll.insert_one({"_id": uu, "a": 2}) - coll.insert_one({"_id": uuid.uuid4(), "a": 1}) - - reduce = "function (obj, prev) { prev.count++; }" - coll = self.db.get_collection( - "uuid", CodecOptions(uuid_representation=STANDARD)) - self.assertEqual([], - coll.group([], {"_id": uu}, - {"count": 0}, reduce)) - coll = self.db.get_collection( - "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) - self.assertEqual([{"count": 1}], - coll.group([], {"_id": uu}, - {"count": 0}, reduce)) - def test_write_concern(self): c = rs_or_single_client(connect=False) self.assertEqual(WriteConcern(), c.write_concern) diff --git a/test/test_legacy_api.py b/test/test_legacy_api.py index 918fa0717..cf06af664 100644 --- a/test/test_legacy_api.py +++ b/test/test_legacy_api.py @@ -18,10 +18,13 @@ import itertools import sys import threading import time +import uuid import warnings sys.path[0:0] = [""] +from bson.binary import PYTHON_LEGACY, STANDARD +from bson.code import Code from bson.codec_options import CodecOptions from bson.dbref import DBRef from bson.objectid import ObjectId @@ -44,7 +47,6 @@ from pymongo.write_concern import WriteConcern from test import client_context, qcheck, unittest, SkipTest from test.test_client import IntegrationTest from test.utils import (joinall, - oid_generated_on_client, rs_or_single_client, wait_until) @@ -904,6 +906,121 @@ class TestLegacy(IntegrationTest): c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True)) + def test_group(self): + db = self.db + db.drop_collection("test") + + self.assertEqual([], + db.test.group([], {}, {"count": 0}, + "function (obj, prev) { prev.count++; }" + )) + + db.test.insert_many([{"a": 2}, {"b": 5}, {"a": 1}]) + + self.assertEqual([{"count": 3}], + db.test.group([], {}, {"count": 0}, + "function (obj, prev) { prev.count++; }" + )) + + self.assertEqual([{"count": 1}], + db.test.group([], {"a": {"$gt": 1}}, {"count": 0}, + "function (obj, prev) { prev.count++; }" + )) + + db.test.insert_one({"a": 2, "b": 3}) + + self.assertEqual([{"a": 2, "count": 2}, + {"a": None, "count": 1}, + {"a": 1, "count": 1}], + db.test.group(["a"], {}, {"count": 0}, + "function (obj, prev) { prev.count++; }" + )) + + # modifying finalize + self.assertEqual([{"a": 2, "count": 3}, + {"a": None, "count": 2}, + {"a": 1, "count": 2}], + db.test.group(["a"], {}, {"count": 0}, + "function (obj, prev) " + "{ prev.count++; }", + "function (obj) { obj.count++; }")) + + # returning finalize + self.assertEqual([2, 1, 1], + db.test.group(["a"], {}, {"count": 0}, + "function (obj, prev) " + "{ prev.count++; }", + "function (obj) { return obj.count; }")) + + # keyf + self.assertEqual([2, 2], + db.test.group("function (obj) { if (obj.a == 2) " + "{ return {a: true} }; " + "return {b: true}; }", {}, {"count": 0}, + "function (obj, prev) " + "{ prev.count++; }", + "function (obj) { return obj.count; }")) + + # no key + self.assertEqual([{"count": 4}], + db.test.group(None, {}, {"count": 0}, + "function (obj, prev) { prev.count++; }" + )) + + self.assertRaises(OperationFailure, db.test.group, + [], {}, {}, "5 ++ 5") + + def test_group_with_scope(self): + db = self.db + db.drop_collection("test") + db.test.insert_many([{"a": 1}, {"b": 1}]) + + reduce_function = "function (obj, prev) { prev.count += inc_value; }" + + self.assertEqual(2, db.test.group([], {}, {"count": 0}, + Code(reduce_function, + {"inc_value": 1}))[0]['count']) + self.assertEqual(4, db.test.group([], {}, {"count": 0}, + Code(reduce_function, + {"inc_value": 2}))[0]['count']) + + self.assertEqual(1, + db.test.group([], {}, {"count": 0}, + Code(reduce_function, + {"inc_value": 0.5}))[0]['count']) + + self.assertEqual(2, db.test.group( + [], {}, {"count": 0}, + Code(reduce_function, {"inc_value": 1}))[0]['count']) + + self.assertEqual(4, db.test.group( + [], {}, {"count": 0}, + Code(reduce_function, {"inc_value": 2}))[0]['count']) + + self.assertEqual(1, db.test.group( + [], {}, {"count": 0}, + Code(reduce_function, {"inc_value": 0.5}))[0]['count']) + + def test_group_uuid_representation(self): + db = self.db + coll = db.uuid + coll.drop() + uu = uuid.uuid4() + coll.insert_one({"_id": uu, "a": 2}) + coll.insert_one({"_id": uuid.uuid4(), "a": 1}) + + reduce = "function (obj, prev) { prev.count++; }" + coll = self.db.get_collection( + "uuid", CodecOptions(uuid_representation=STANDARD)) + self.assertEqual([], + coll.group([], {"_id": uu}, + {"count": 0}, reduce)) + coll = self.db.get_collection( + "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) + self.assertEqual([{"count": 1}], + coll.group([], {"_id": uu}, + {"count": 0}, reduce)) + def test_last_status(self): # Tests many legacy API elements. # We must call getlasterror on same socket as the last operation. diff --git a/test/test_read_preferences.py b/test/test_read_preferences.py index 6c9a05614..629040e55 100644 --- a/test/test_read_preferences.py +++ b/test/test_read_preferences.py @@ -16,9 +16,10 @@ import contextlib import copy +import pickle import random import sys -import pickle +import warnings sys.path[0:0] = [""] @@ -416,8 +417,10 @@ class TestCommandAndReadPreference(TestReplicaSetClientBase): lambda: self.c.pymongo_test.some_collection.drop()) def test_group(self): - self._test_coll_helper(True, self.c.pymongo_test.test, 'group', - {'a': 1}, {}, {}, 'function() { }') + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self._test_coll_helper(True, self.c.pymongo_test.test, 'group', + {'a': 1}, {}, {}, 'function() { }') def test_map_reduce(self): self._test_coll_helper(False, self.c.pymongo_test.test, 'map_reduce',