BUMP 2.8.1

Log4mongo-python is actively maintained and upaded to the latest pymongo versions (2.8 and 3.X).

.travis.yml

@@ -1,11 +1,11 @@
 language: python
 
 python:
-  - 2.5
   - 2.6
   - 2.7
   - 3.2
   - 3.3
+  - 3.4
   - pypy
 
 services:
@@ -14,5 +14,5 @@ services:
 script: python setup.py test
 
 install:
-    #Temporary solution for Travis CI mutiprocessing issue #155
+    # Temporary solution for Travis CI multiprocessing issue #943
     - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm

README.rst

@@ -38,6 +38,24 @@ case in our issue management tool, JIRA:
 Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the
 Core Server (i.e. SERVER) project are **public**.
 
+How To Ask For Help
+-------------------
+
+Please include all of the following information when opening an issue:
+
+- Detailed steps to reproduce the problem, including full traceback, if possible.
+- The exact python version used, with patch level::
+
+  $ python -c "import sys; print(sys.version)"
+
+- The exact version of PyMongo used, with patch level::
+
+  $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())"
+
+- The operating system and version (e.g. Windows 7, OSX 10.8, ...)
+- Web framework or asynchronous network library used, if any, with version (e.g.
+  Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...)
+
 Security Vulnerabilities
 ------------------------
 
@@ -54,6 +72,10 @@ should be able to do **easy_install pymongo** to install
 PyMongo. Otherwise you can download the project source and do **python
 setup.py install** to install.
 
+Do **not** install the "bson" package. PyMongo comes with its own bson package;
+doing "easy_install bson" installs a third-party package that is incompatible
+with PyMongo.
+
 Dependencies
 ============
 

bson/__init__.py

@@ -333,7 +333,10 @@ def _elements_to_dict(data, as_class, tz_aware, uuid_subtype, compile_re):
     return result
 
 def _bson_to_dict(data, as_class, tz_aware, uuid_subtype, compile_re):
-    obj_size = struct.unpack("<i", data[:4])[0]
+    try:
+        obj_size = struct.unpack("<i", data[:4])[0]
+    except struct.error, e:
+        raise InvalidBSON(str(e))
     length = len(data)
     if length < obj_size:
         raise InvalidBSON("objsize too large")
@@ -493,7 +496,6 @@ if _use_c:
     _dict_to_bson = _cbson._dict_to_bson
 
 
-
 def decode_all(data, as_class=dict,
                tz_aware=True, uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True):
     """Decode BSON data to multiple documents.
@@ -507,6 +509,8 @@ def decode_all(data, as_class=dict,
         documents
       - `tz_aware` (optional): if ``True``, return timezone-aware
         :class:`~datetime.datetime` instances
+      - `uuid_subtype` (optional): The BSON representation to use for UUIDs.
+        See the :mod:`bson.binary` module for all options.
       - `compile_re` (optional): if ``False``, don't attempt to compile
         BSON regular expressions into Python regular expressions. Return
         instances of :class:`~bson.regex.Regex` instead. Can avoid
@@ -542,6 +546,82 @@ if _use_c:
     decode_all = _cbson.decode_all
 
 
+def decode_iter(data, as_class=dict, tz_aware=True,
+                uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True):
+    """Decode BSON data to multiple documents as a generator.
+
+    Works similarly to the decode_all function, but yields one document at a
+    time.
+
+    `data` must be a string of concatenated, valid, BSON-encoded
+    documents.
+
+    :Parameters:
+      - `data`: BSON data
+      - `as_class` (optional): the class to use for the resulting
+        documents
+      - `tz_aware` (optional): if ``True``, return timezone-aware
+        :class:`~datetime.datetime` instances
+      - `uuid_subtype` (optional): The BSON representation to use for UUIDs.
+        See the :mod:`bson.binary` module for all options.
+      - `compile_re` (optional): if ``False``, don't attempt to compile
+        BSON regular expressions into Python regular expressions. Return
+        instances of
+        :class:`~bson.regex.Regex` instead. Can avoid
+        :exc:`~bson.errors.InvalidBSON` errors when receiving
+        Python-incompatible regular expressions, for example from
+        ``currentOp``
+
+    .. versionadded:: 2.8
+    """
+    position = 0
+    end = len(data) - 1
+    while position < end:
+        obj_size = struct.unpack("<i", data[position:position + 4])[0]
+        elements = data[position:position + obj_size]
+        position += obj_size
+        yield _bson_to_dict(elements, as_class,
+                            tz_aware, uuid_subtype, compile_re)[0]
+
+
+def decode_file_iter(file_obj, as_class=dict, tz_aware=True,
+                     uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True):
+    """Decode bson data from a file to multiple documents as a generator.
+
+    Works similarly to the decode_all function, but reads from the file object
+    in chunks and parses bson in chunks, yielding one document at a time.
+
+    :Parameters:
+      - `file_obj`: A file object containing BSON data.
+      - `as_class` (optional): the class to use for the resulting
+        documents
+      - `tz_aware` (optional): if ``True``, return timezone-aware
+        :class:`~datetime.datetime` instances
+      - `uuid_subtype` (optional): The BSON representation to use for UUIDs.
+        See the :mod:`bson.binary` module for all options.
+      - `compile_re` (optional): if ``False``, don't attempt to compile
+        BSON regular expressions into Python regular expressions. Return
+        instances of
+        :class:`~bson.regex.Regex` instead. Can avoid
+        :exc:`~bson.errors.InvalidBSON` errors when receiving
+        Python-incompatible regular expressions, for example from
+        ``currentOp``
+
+    .. versionadded:: 2.8
+    """
+    while True:
+        # Read size of next object.
+        size_data = file_obj.read(4)
+        if len(size_data) == 0:
+            break  # Finished with file normaly.
+        elif len(size_data) != 4:
+            raise InvalidBSON("cut off in middle of objsize")
+        obj_size = struct.unpack("<i", size_data)[0] - 4
+        elements = size_data + file_obj.read(obj_size)
+        yield _bson_to_dict(elements, as_class,
+                            tz_aware, uuid_subtype, compile_re)[0]
+
+
 def is_valid(bson):
     """Check that the given string represents valid :class:`BSON` data.
 
@@ -584,6 +664,8 @@ class BSON(binary_type):
           - `check_keys` (optional): check if keys start with '$' or
             contain '.', raising :class:`~bson.errors.InvalidDocument` in
             either case
+          - `uuid_subtype` (optional): The BSON representation to use for
+            UUIDs. See the :mod:`bson.binary` module for all options.
 
         .. versionadded:: 1.9
         """
@@ -610,6 +692,8 @@ class BSON(binary_type):
             document
           - `tz_aware` (optional): if ``True``, return timezone-aware
             :class:`~datetime.datetime` instances
+          - `uuid_subtype` (optional): The BSON representation to use for
+            UUIDs. See the :mod:`bson.binary` module for all options.
           - `compile_re` (optional): if ``False``, don't attempt to compile
             BSON regular expressions into Python regular expressions. Return
             instances of

bson/_cbsonmodule.c

@@ -554,6 +554,7 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer,
                                     unsigned char check_keys,
                                     unsigned char uuid_subtype) {
     struct module_state *state = GETSTATE(self);
+    PyObject* type_marker = NULL;
 
     /*
      * Don't use PyObject_IsInstance for our custom types. It causes
@@ -561,26 +562,32 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer,
      * have a _type_marker attribute, which we can switch on instead.
      */
     if (PyObject_HasAttrString(value, "_type_marker")) {
-        long type;
-        PyObject* type_marker = PyObject_GetAttrString(value, "_type_marker");
-        if (type_marker == NULL)
+        type_marker = PyObject_GetAttrString(value, "_type_marker");
+        if (type_marker == NULL) {
             return 0;
+        }
+    }
+    /*
+     * Python objects with broken __getattr__ implementations could return
+     * arbitrary types for a call to PyObject_GetAttrString. For example
+     * pymongo.database.Database returns a new Collection instance for
+     * __getattr__ calls with names that don't match an existing attribute
+     * or method. In some cases "value" could be a subtype of something
+     * we know how to serialize. Make a best effort to encode these types.
+     */
 #if PY_MAJOR_VERSION >= 3
-        type = PyLong_AsLong(type_marker);
+    if (type_marker && PyLong_CheckExact(type_marker)) {
+        long type = PyLong_AsLong(type_marker);
 #else
-        type = PyInt_AsLong(type_marker);
+    if (type_marker && PyInt_CheckExact(type_marker)) {
+        long type = PyInt_AsLong(type_marker);
 #endif
         Py_DECREF(type_marker);
         /* 
          * Py(Long|Int)_AsLong returns -1 for error but -1 is a valid value
          * so we call PyErr_Occurred to differentiate.
-         *
-         * One potential reason for an error is the user passing an invalid
-         * type that overrides __getattr__ (e.g. pymongo.collection.Collection)
          */
         if (type == -1 && PyErr_Occurred()) {
-            PyErr_Clear();
-            _set_cannot_encode(value);
             return 0;
         }
         switch (type) {
@@ -792,6 +799,8 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer,
                 return 1;
             }
         }
+    } else {
+        Py_XDECREF(type_marker);
     }
 
     /* No _type_marker attibute or not one of our types. */
@@ -1775,7 +1784,7 @@ static PyObject* get_value(PyObject* self, const char* buffer, unsigned* positio
                 Py_DECREF(args);
                 goto invalid;
             }
-            utc_type = _get_object(state->UTC, "bson.tz_util", "UTC");
+            utc_type = _get_object(state->UTC, "bson.tz_util", "utc");
             if (!utc_type || PyDict_SetItemString(kwargs, "tzinfo", utc_type) == -1) {
                 Py_DECREF(replace);
                 Py_DECREF(args);

bson/json_util.py

@@ -47,6 +47,16 @@ It won't handle :class:`~bson.binary.Binary` and :class:`~bson.code.Code`
 instances (as they are extended strings you can't provide custom defaults),
 but it will be faster as there is less recursion.
 
+.. versionchanged:: 2.8
+   The output format for :class:`~bson.timestamp.Timestamp` has changed from
+   '{"t": <int>, "i": <int>}' to '{"$timestamp": {"t": <int>, "i": <int>}}'.
+   This new format will be decoded to an instance of
+   :class:`~bson.timestamp.Timestamp`. The old format will continue to be
+   decoded to a python dict as before. Encoding to the old format is no longer
+   supported as it was never correct and loses type information.
+   Added support for $numberLong and $undefined - new in MongoDB 2.6 - and
+   parsing $date in ISO-8601 format.
+
 .. versionchanged:: 2.7
    Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
    instances. (But not in Python 2.4.)
@@ -76,6 +86,7 @@ import base64
 import calendar
 import datetime
 import re
+import time
 
 json_lib = True
 try:
@@ -96,6 +107,7 @@ from bson.min_key import MinKey
 from bson.objectid import ObjectId
 from bson.regex import Regex
 from bson.timestamp import Timestamp
+from bson.tz_util import utc
 
 from bson.py3compat import PY3, binary_type, string_types
 
@@ -166,7 +178,39 @@ def object_hook(dct, compile_re=True):
     if "$ref" in dct:
         return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None))
     if "$date" in dct:
-        secs = float(dct["$date"]) / 1000.0
+        dtm = dct["$date"]
+        # mongoexport 2.6 and newer
+        if isinstance(dtm, basestring):
+            # datetime.datetime.strptime is new in python 2.5
+            naive = datetime.datetime(
+                *(time.strptime(dtm[:19], "%Y-%m-%dT%H:%M:%S")[0:6]))
+            # The %f format is new in python 2.6
+            micros = int(dtm[20:23]) * 1000
+            aware = naive.replace(microsecond=micros, tzinfo=utc)
+            offset = dtm[23:]
+            if not offset or offset == 'Z':
+                # UTC
+                return aware
+            else:
+                if len(offset) == 5:
+                    # Offset from mongoexport is in format (+|-)HHMM
+                    secs = (int(offset[1:3]) * 3600 + int(offset[3:]) * 60)
+                elif ':' in offset and len(offset) == 6:
+                    # RFC-3339 format (+|-)HH:MM
+                    hours, minutes = offset[1:].split(':')
+                    secs = (int(hours) * 3600 + int(minutes) * 60)
+                else:
+                    # Not RFC-3339 compliant or mongoexport output.
+                    raise ValueError("invalid format for offset")
+                if offset[0] == "-":
+                    secs *= -1
+                return aware - datetime.timedelta(seconds=secs)
+        # mongoexport 2.6 and newer, time before the epoch (SERVER-15275)
+        elif isinstance(dtm, dict):
+            secs = float(dtm["$numberLong"]) / 1000.0
+        # mongoexport before 2.6
+        else:
+            secs = float(dtm) / 1000.0
         return EPOCH_AWARE + datetime.timedelta(seconds=secs)
     if "$regex" in dct:
         flags = 0
@@ -193,6 +237,15 @@ def object_hook(dct, compile_re=True):
         return Code(dct["$code"], dct.get("$scope"))
     if bson.has_uuid() and "$uuid" in dct:
         return bson.uuid.UUID(dct["$uuid"])
+    if "$undefined" in dct:
+        return None
+    if "$numberLong" in dct:
+        # 2to3 will change this to int. PyMongo 3.0 supports
+        # a new type, Int64, to avoid round trip issues.
+        return long(dct["$numberLong"])
+    if "$timestamp" in dct:
+        tsp = dct["$timestamp"]
+        return Timestamp(tsp["t"], tsp["i"])
     return dct
 
 
@@ -240,7 +293,7 @@ def default(obj):
     if isinstance(obj, MaxKey):
         return {"$maxKey": 1}
     if isinstance(obj, Timestamp):
-        return SON([("t", obj.time), ("i", obj.inc)])
+        return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])}
     if isinstance(obj, Code):
         return SON([('$code', str(obj)), ('$scope', obj.scope)])
     if isinstance(obj, Binary):

bson/objectid.py

@@ -54,6 +54,13 @@ def _machine_bytes():
     return machine_hash.digest()[0:3]
 
 
+def _raise_invalid_id(oid):
+    raise InvalidId(
+        "%r is not a valid ObjectId, it must be a 12-byte input"
+        " of type %r or a 24-character hex string" % (
+            oid, binary_type.__name__))
+
+
 class ObjectId(object):
     """A MongoDB ObjectId.
     """
@@ -70,19 +77,41 @@ class ObjectId(object):
     def __init__(self, oid=None):
         """Initialize a new ObjectId.
 
-        If `oid` is ``None``, create a new (unique) ObjectId. If `oid`
-        is an instance of (:class:`basestring` (:class:`str` or :class:`bytes`
-        in python 3), :class:`ObjectId`) validate it and use that.  Otherwise,
-        a :class:`TypeError` is raised. If `oid` is invalid,
-        :class:`~bson.errors.InvalidId` is raised.
+        An ObjectId is a 12-byte unique identifier consisting of:
+
+          - a 4-byte value representing the seconds since the Unix epoch,
+          - a 3-byte machine identifier,
+          - a 2-byte process id, and
+          - a 3-byte counter, starting with a random value.
+
+        By default, ``ObjectId()`` creates a new unique identifier. The
+        optional parameter `oid` can be an :class:`ObjectId`, or any 12
+        :class:`bytes` or, in Python 2, any 12-character :class:`str`.
+
+        For example, the 12 bytes b'foo-bar-quux' do not follow the ObjectId
+        specification but they are acceptable input::
+
+          >>> ObjectId(b'foo-bar-quux')
+          ObjectId('666f6f2d6261722d71757578')
+
+        `oid` can also be a :class:`unicode` or :class:`str` of 24 hex digits::
+
+          >>> ObjectId('0123456789ab0123456789ab')
+          ObjectId('0123456789ab0123456789ab')
+          >>>
+          >>> # A u-prefixed unicode literal:
+          >>> ObjectId(u'0123456789ab0123456789ab')
+          ObjectId('0123456789ab0123456789ab')
+
+        Raises :class:`~bson.errors.InvalidId` if `oid` is not 12 bytes nor
+        24 hex digits, or :class:`TypeError` if `oid` is not an accepted type.
 
         :Parameters:
-          - `oid` (optional): a valid ObjectId (12 byte binary or 24 character
-            hex string)
+          - `oid` (optional): a valid ObjectId.
 
         .. versionadded:: 1.2.1
            The `oid` parameter can be a ``unicode`` instance (that contains
-           only hexadecimal digits).
+           24 hexadecimal digits).
 
         .. mongodoc:: objectids
         """
@@ -140,6 +169,9 @@ class ObjectId(object):
 
         .. versionadded:: 2.3
         """
+        if not oid:
+            return False
+
         try:
             ObjectId(oid)
             return True
@@ -186,14 +218,14 @@ class ObjectId(object):
                 if isinstance(oid, binary_type):
                     self.__id = oid
                 else:
-                    raise InvalidId("%s is not a valid ObjectId" % oid)
+                    _raise_invalid_id(oid)
             elif len(oid) == 24:
                 try:
                     self.__id = bytes_from_hex(oid)
                 except (TypeError, ValueError):
-                    raise InvalidId("%s is not a valid ObjectId" % oid)
+                    _raise_invalid_id(oid)
             else:
-                raise InvalidId("%s is not a valid ObjectId" % oid)
+                _raise_invalid_id(oid)
         else:
             raise TypeError("id must be an instance of (%s, %s, ObjectId), "
                             "not %s" % (binary_type.__name__,

bson/son.py

@@ -100,7 +100,7 @@ class SON(dict):
         return "SON([%s])" % ", ".join(result)
 
     def __setitem__(self, key, value):
-        if key not in self:
+        if key not in self.__keys:
             self.__keys.append(key)
         dict.__setitem__(self, key, value)
 
@@ -120,14 +120,11 @@ class SON(dict):
     # efficient.
     # second level definitions support higher levels
     def __iter__(self):
-        for k in self.keys():
+        for k in self.__keys:
             yield k
 
     def has_key(self, key):
-        return key in self.keys()
-
-    def __contains__(self, key):
-        return key in self.keys()
+        return key in self.__keys
 
     # third level takes advantage of second level definitions
     def iteritems(self):
@@ -149,8 +146,8 @@ class SON(dict):
         return [(key, self[key]) for key in self]
 
     def clear(self):
-        for key in self.keys():
-            del self[key]
+        self.__keys = []
+        super(SON, self).clear()
 
     def setdefault(self, key, default=None):
         try:
@@ -214,7 +211,7 @@ class SON(dict):
         return not self == other
 
     def __len__(self):
-        return len(self.keys())
+        return len(self.__keys)
 
     def to_dict(self):
         """Convert a SON document to a normal Python dictionary instance.
@@ -226,12 +223,12 @@ class SON(dict):
         def transform_value(value):
             if isinstance(value, list):
                 return [transform_value(v) for v in value]
-            if isinstance(value, SON):
-                value = dict(value)
-            if isinstance(value, dict):
-                for k, v in value.iteritems():
-                    value[k] = transform_value(v)
-            return value
+            elif isinstance(value, dict):
+                return dict([
+                    (k, transform_value(v))
+                    for k, v in value.iteritems()])
+            else:
+                return value
 
         return transform_value(dict(self))
 

doc/api/pymongo/collection.rst

@@ -10,6 +10,7 @@
    .. autodata:: pymongo.GEOHAYSTACK
    .. autodata:: pymongo.GEOSPHERE
    .. autodata:: pymongo.HASHED
+   .. autodata:: pymongo.TEXT
 
    .. autoclass:: pymongo.collection.Collection(database, name[, create=False[, **kwargs]]])
 

doc/api/pymongo/connection.rst

@@ -42,6 +42,7 @@
       .. automethod:: get_default_database
       .. automethod:: server_info
       .. automethod:: start_request
+      .. automethod:: in_request
       .. automethod:: end_request
       .. automethod:: close_cursor
       .. automethod:: kill_cursors

doc/api/pymongo/mongo_client.rst

@@ -41,6 +41,7 @@
       .. automethod:: get_default_database
       .. automethod:: server_info
       .. automethod:: start_request
+      .. automethod:: in_request
       .. automethod:: end_request
       .. automethod:: close_cursor
       .. automethod:: kill_cursors

doc/changelog.rst

@@ -1,6 +1,102 @@
 Changelog
 =========
 
+Changes in Version 2.8.1
+------------------------
+
+Version 2.8.1 fixes a number of issues reported since the release of PyMongo
+2.8. It is a recommended upgrade for all users of PyMongo 2.x.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.8.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.8.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/15324
+
+Changes in Version 2.8
+----------------------
+
+Version 2.8 is a major release that provides full support for MongoDB 3.0 and
+fixes a number of bugs.
+
+Special thanks to Don Mitchell, Ximing, Can Zhang, Sergey Azovskov, and Heewa
+Barfchin for their contributions to this release.
+
+Highlights include:
+
+- Support for the SCRAM-SHA-1 authentication mechanism (new in MongoDB 3.0).
+- JSON decoder support for the new $numberLong and $undefined types.
+- JSON decoder support for the $date type as an ISO-8601 string.
+- Support passing an index name to :meth:`~pymongo.cursor.Cursor.hint`.
+- The :meth:`~pymongo.cursor.Cursor.count` method will use a hint if one
+  has been provided through :meth:`~pymongo.cursor.Cursor.hint`.
+- A new socketKeepAlive option for the connection pool.
+- New generator based BSON decode functions, :func:`~bson.decode_iter`
+  and :func:`~bson.decode_file_iter`.
+- Internal changes to support alternative storage engines like wiredtiger.
+
+.. note:: There are a number of deprecations in this release for features that
+  will be removed in PyMongo 3.0. These include:
+
+  - :meth:`~pymongo.mongo_client.MongoClient.start_request`
+  - :meth:`~pymongo.mongo_client.MongoClient.in_request`
+  - :meth:`~pymongo.mongo_client.MongoClient.end_request`
+  - :meth:`~pymongo.mongo_client.MongoClient.copy_database`
+  - :meth:`~pymongo.database.Database.error`
+  - :meth:`~pymongo.database.Database.last_status`
+  - :meth:`~pymongo.database.Database.previous_error`
+  - :meth:`~pymongo.database.Database.reset_error_history`
+  - :class:`~pymongo.master_slave_connection.MasterSlaveConnection`
+
+  The JSON format for :class:`~bson.timestamp.Timestamp` has changed from
+  '{"t": <int>, "i": <int>}' to '{"$timestamp": {"t": <int>, "i": <int>}}'.
+  This new format will be decoded to an instance of
+  :class:`~bson.timestamp.Timestamp`. The old format will continue to be
+  decoded to a python dict as before. Encoding to the old format is no
+  longer supported as it was never correct and loses type information.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.8 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.8 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14223
+
+Changes in Version 2.7.2
+------------------------
+
+Version 2.7.2 includes fixes for upsert reporting in the bulk API for MongoDB
+versions previous to 2.6, a regression in how son manipulators are applied in
+:meth:`~pymongo.collection.Collection.insert`, a few obscure connection pool
+semaphore leaks, and a few other minor issues. See the list of issues resolved
+for full details.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.7.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.7.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14005
+
+Changes in Version 2.7.1
+------------------------
+
+Version 2.7.1 fixes a number of issues reported since the release of 2.7,
+most importantly a fix for creating indexes and manipulating users through
+mongos versions older than 2.4.0.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.7.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.7.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/13823
+
 Changes in Version 2.7
 ----------------------
 

doc/compatibility-policy.rst

@@ -0,0 +1,61 @@
+Compatibility Policy
+====================
+
+Semantic Versioning
+-------------------
+
+PyMongo's version numbers follow `semantic versioning`_: each version number
+is structured "major.minor.patch". Patch releases fix bugs, minor releases
+add features (and may fix bugs), and major releases include API changes that
+break backwards compatibility (and may add features and fix bugs).
+
+Deprecation
+-----------
+
+Before we remove a feature in a major release, PyMongo's maintainers make an
+effort to release at least one minor version that *deprecates* it. We add
+"**DEPRECATED**" to the feature's documentation, and update the code to raise a
+`DeprecationWarning`_. You can ensure your code is future-proof by running
+your code with the latest PyMongo release and looking for DeprecationWarnings.
+
+Starting with Python 2.7, the interpreter silences DeprecationWarnings by
+default. For example, the following code uses the deprecated ``slave_okay``
+option but does not raise any warning:
+
+.. code-block:: python
+
+    # "slave_okay.py"
+    from pymongo import MongoClient
+
+    client = MongoClient(slave_okay=True)
+
+To print deprecation warnings to stderr, run python with "-Wd"::
+
+  $ python -Wd slave_okay.py
+  slave_okay.py:4: DeprecationWarning: slave_okay is deprecated. Please use read_preference instead.
+    client = MongoClient(slave_okay=True)
+
+You can turn warnings into exceptions with "python -We"::
+
+  $ python -We slave_okay.py
+  Traceback (most recent call last):
+    File "slave_okay.py", line 4, in <module>
+      client = MongoClient(slave_okay=True)
+    File "/Users/emptysquare/.virtualenvs/official/mongo-python-driver/pymongo/mongo_client.py", line 373, in __init__
+      stacklevel=2)
+  DeprecationWarning: slave_okay is deprecated. Please use read_preference instead.
+
+If your own code's test suite passes with "python -We" then it uses no
+deprecated PyMongo features.
+
+.. seealso:: The Python documentation on `the warnings module`_,
+   and `the -W command line option`_.
+
+.. _semantic versioning: http://semver.org/
+
+.. _DeprecationWarning:
+  https://docs.python.org/2/library/exceptions.html#exceptions.DeprecationWarning
+
+.. _the warnings module: https://docs.python.org/2/library/warnings.html
+
+.. _the -W command line option: https://docs.python.org/2/using/cmdline.html#cmdoption-W

doc/contributors.rst

@@ -69,3 +69,10 @@ The following is a list of people who have contributed to
 - Yuchen Ying (yegle)
 - Kyle Erf (3rf)
 - Luke Lovett (lovett89)
+- Jaroslav Semančík (girogiro)
+- Don Mitchell (dmitchell)
+- Ximing (armnotstrong)
+- Can Zhang (cannium)
+- Sergey Azovskov (last-g)
+- Heewa Barfchin (heewa)
+- Len Buckens (buckensl)

doc/examples/aggregation.rst

@@ -74,7 +74,7 @@ Map/Reduce
 ----------
 
 Another option for aggregation is to use the map reduce framework.  Here we
-will define **map** and **reduce** functions to also count he number of
+will define **map** and **reduce** functions to also count the number of
 occurrences for each tag in the ``tags`` array, across the entire collection.
 
 Our **map** function just emits a single `(key, 1)` pair for each tag in
@@ -171,17 +171,17 @@ reduce function.
 .. note:: Doesn't work with sharded MongoDB configurations, use aggregation or
           map/reduce instead of group().
 
-Here we are doing a simple group and count of the occurrences ``x`` values:
+Here we are doing a simple group and count of the occurrences of ``x`` values:
 
 .. doctest::
 
+  >>> from bson.code import Code
   >>> reducer = Code("""
   ...                function(obj, prev){
   ...                  prev.count++;
   ...                }
   ...                """)
   ...
-  >>> from bson.son import SON
   >>> results = db.things.group(key={"x":1}, condition={}, initial={"count": 0}, reduce=reducer)
   >>> for doc in results:
   ...   print doc

doc/examples/authentication.rst

@@ -5,24 +5,69 @@ MongoDB supports several different authentication mechanisms. These examples
 cover all authentication methods currently supported by PyMongo, documenting
 Python module and MongoDB version dependencies.
 
-MONGODB-CR
-----------
-MONGODB-CR is the default authentication mechanism supported by a MongoDB
-cluster configured for authentication. Authentication is per-database and
-credentials can be specified through the MongoDB URI or passed to the
-:meth:`~pymongo.database.Database.authenticate` method::
+Support For Special Characters In Usernames And Passwords
+---------------------------------------------------------
+
+If your username or password contains special characters (e.g. '/', ' ',
+or '@') you must ``%xx`` escape them for use in the MongoDB URI. PyMongo
+uses :meth:`~urllib.unquote_plus` to decode them. For example::
+
+  >>> import urllib
+  >>> password = urllib.quote_plus('pass/word')
+  >>> password
+  'pass%2Fword'
+  >>> MongoClient('mongodb://user:' + password + '@127.0.0.1')
+  MongoClient('127.0.0.1', 27017)
+
+SCRAM-SHA-1 (RFC 5802)
+----------------------
+.. versionadded:: 2.8
+
+SCRAM-SHA-1 is the default authentication mechanism supported by a cluster
+configured for authentication with MongoDB 3.0 or later. Authentication is
+per-database and credentials can be specified through the MongoDB URI or
+passed to the :meth:`~pymongo.database.Database.authenticate` method::
 
   >>> from pymongo import MongoClient
   >>> client = MongoClient('example.com')
-  >>> client.the_database.authenticate('user', 'password')
+  >>> client.the_database.authenticate('user', 'password', mechanism='SCRAM-SHA-1')
   True
   >>>
-  >>> uri = "mongodb://user:password@example.com/the_database"
+  >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=SCRAM-SHA-1"
   >>> client = MongoClient(uri)
-  >>>
 
-When using MongoDB's delegated authentication features, a separate
-authentication source can be specified (using PyMongo 2.5 or newer)::
+For best performance install `backports.pbkdf2`_, especially on Python older
+than 2.7.8, or on Python 3 before Python 3.4.
+
+.. _backports.pbkdf2: https://pypi.python.org/pypi/backports.pbkdf2/
+
+MONGODB-CR
+----------
+
+Before MongoDB 3.0 the default authentication mechanism was MONGODB-CR,
+the "MongoDB Challenge-Response" protocol::
+
+  >>> from pymongo import MongoClient
+  >>> client = MongoClient('example.com')
+  >>> client.the_database.authenticate('user', 'password', mechanism='MONGODB-CR')
+  True
+  >>>
+  >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=MONGODB-CR"
+  >>> client = MongoClient(uri)
+
+Default Authentication Mechanism
+--------------------------------
+
+If no mechanism is specified, PyMongo automatically uses MONGODB-CR when
+connected to a pre-3.0 version of MongoDB, and SCRAM-SHA-1 when connected to
+a recent version.
+
+Delegated Authentication
+------------------------
+.. versionadded: 2.5
+
+In MongoDB 2.4.x a separate authentication source can be specified.
+This feature was introduced in MongoDB 2.4 and removed in 2.6::
 
   >>> from pymongo import MongoClient
   >>> client = MongoClient('example.com')
@@ -113,15 +158,15 @@ or using :meth:`~pymongo.database.Database.authenticate`::
   True
 
 The default service name used by MongoDB and PyMongo is `mongodb`. You can
-specify a custom service name with the ``gssapiServiceName`` option::
+specify a custom service name with the ``authMechanismProperties`` option::
 
   >>> from pymongo import MongoClient
-  >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&gssapiServiceName=myservicename"
+  >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&authMechanismProperties=SERVICE_NAME:myservicename"
   >>> client = MongoClient(uri)
   >>>
   >>> client = MongoClient('example.com')
   >>> db = client.test
-  >>> db.authenticate('mongodbuser@EXAMPLE.COM', mechanism='GSSAPI', gssapiServiceName='myservicename')
+  >>> db.authenticate('mongodbuser@EXAMPLE.COM', mechanism='GSSAPI', authMechanismProperties='SERVICE_NAME:myservicename')
   True
 
 .. note::
@@ -177,4 +222,3 @@ the SASL PLAIN mechanism::
   ...                      ssl_cert_reqs=ssl.CERT_REQUIRED,
   ...                      ssl_ca_certs='/path/to/ca.pem')
   >>>
-

doc/examples/copydb.rst

@@ -0,0 +1,71 @@
+Copying a Database
+==================
+
+Raw command
+-----------
+
+To copy a database within a single mongod process, or between mongod
+servers, simply connect to the target mongod and use the
+:meth:`~pymongo.database.Database.command` method::
+
+  >>> from pymongo import MongoClient
+  >>> client = MongoClient('target.example.com')
+  >>> client.admin.command('copydb',
+                           fromdb='source_db_name',
+                           todb='target_db_name')
+
+To copy from a different mongod server that is not password-protected::
+
+  >>> client.admin.command('copydb',
+                           fromdb='source_db_name',
+                           todb='target_db_name',
+                           fromhost='source.example.com')
+
+If the target server is password-protected, authenticate to the "admin"
+database first::
+
+  >>> client.admin.authenticate('administrator', 'pwd')
+  True
+  >>> client.admin.command('copydb',
+                           fromdb='source_db_name',
+                           todb='target_db_name',
+                           fromhost='source.example.com')
+
+See the :doc:`authentication examples </examples/authentication>`.
+
+``copy_database`` method
+------------------------
+
+The current version of PyMongo provides a helper method,
+:meth:`~pymongo.mongo_client.MongoClient.copy_database`, to copy a database
+from a password-protected mongod server to the target server.
+This method is deprecated and will be removed in PyMongo 3.0.
+Use the `copyDatabase function in the mongo shell`_ instead.
+
+Until the method is removed from PyMongo, you can copy a database from a
+password-protected server like so::
+
+  >>> client = MongoClient('target.example.com')
+  >>> client.copy_database(from_name='source_db_name',
+                           to_name='target_db_name',
+                           from_host='source.example.com',
+                           username='jesse',
+                           password='pwd',
+                           mechanism='SCRAM-SHA-1')
+
+Provide the username and password of a user who is authorized to read the
+source database on the source host. Again, if the target database is also
+password-protected, authenticate to the "admin" database first.
+
+The mechanism can be "MONGODB-CR" or "SCRAM-SHA-1". Use SCRAM-SHA-1 if the
+target and source hosts are both MongoDB 2.8 or later, otherwise use
+MONGODB-CR.
+
+If no mechanism is specified, PyMongo tries to use MONGODB-CR when
+connected to a pre-2.8 version of MongoDB, and SCRAM-SHA-1 when connected to
+a recent version. However, since PyMongo cannot determine the MongoDB
+version of the **source** host, it is better if you specify a mechanism
+yourself.
+
+.. _copyDatabase function in the mongo shell:
+   http://docs.mongodb.org/manual/reference/method/db.copyDatabase/

doc/examples/index.rst

@@ -18,6 +18,7 @@ MongoDB, you can start it like so:
 
    aggregation
    authentication
+   copydb
    bulk
    custom_type
    geo

doc/examples/requests.rst

@@ -1,11 +1,19 @@
 Requests
 ========
 
-PyMongo supports the idea of a *request*: a series of operations executed with
-a single socket, which are guaranteed to be processed on the server in the same
-order as they ran on the client.
+The ``start_request`` method of :class:`~pymongo.mongo_client.MongoClient`
+and :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` is now
+**deprecated** and will be removed in PyMongo 3.0.
 
-Requests are not usually necessary with PyMongo.
+PyMongo versions previous to 3.0 support the idea of a *request*: a series of
+operations executed with a single socket. This feature intended to make
+read-your-writes consistency more likely, even with unacknowledged writes.
+(That is, operations with write concern ``w=0``.)
+
+However, mongos 2.6 doesn't support socket pinning by default, and `mongos 2.8
+doesn't support it at all`_, so requests provide no benefit with sharding.
+
+In any case, requests are no longer necessary with PyMongo.
 By default, the methods :meth:`~pymongo.collection.Collection.insert`,
 :meth:`~pymongo.collection.Collection.update`,
 :meth:`~pymongo.collection.Collection.save`, and
@@ -14,97 +22,7 @@ acknowledgment from the server, so ordered execution is already guaranteed. You
 can be certain the next :meth:`~pymongo.collection.Collection.find` or
 :meth:`~pymongo.collection.Collection.count`, for example, is executed on the
 server after the writes complete. This is called "read-your-writes
-consistency."
+consistency." If your application requires this consistency, do not override
+the default write concern with ``w=0``.
 
-An example of when a request is necessary is if a series of documents are
-inserted with ``w=0`` for performance reasons, and you want to query those
-documents immediately afterward: With ``w=0`` the writes can queue up at the
-server and might not be immediately visible in query results. Wrapping the
-inserts and queries within
-:meth:`~pymongo.mongo_client.MongoClient.start_request` and
-:meth:`~pymongo.mongo_client.MongoClient.end_request` forces a query to be on
-the same socket as the inserts, so the query won't execute until the inserts
-are complete on the server side.
-
-Example
--------
-
-Let's consider a collection of web-analytics counters. We want to count the
-number of page views our site has served for each combination of browser,
-region, and OS, and then show the user the number of page views from his or her
-region, *including* the user's own visit. We have three ways to do so reliably:
-
-1. Simply update the counters with an acknowledged write (the default), and
-then ``find`` all counters for the visitor's region. This will ensure that the
-``update`` completes before the ``find`` begins, but it comes with a performance
-penalty that may be unacceptable for analytics.
-
-2. Create the :class:`~pymongo.mongo_client.MongoClient` with ``w=0`` and
-``auto_start_request=True`` to do unacknowledged writes and ensure each thread
-gets its own socket.
-
-3. Explicitly call :meth:`~pymongo.mongo_client.MongoClient.start_request`,
-then do the unacknowledged updates and the queries within the request. This
-third method looks like:
-
-.. testsetup::
-
-  from pymongo import MongoClient
-  client = MongoClient()
-  counts = client.requests_example.counts
-  counts.drop()
-  region, browser, os = 'US', 'Firefox', 'Mac OS X'
-
-.. doctest::
-
-  >>> client = MongoClient()
-  >>> counts = client.requests_example.counts
-  >>> region, browser, os = 'US', 'Firefox', 'Mac OS X'
-  >>> request = client.start_request()
-  >>> try:
-  ...   counts.update(
-  ...     {'region': region, 'browser': browser, 'os': os},
-  ...     {'$inc': {'n': 1 }},
-  ...     upsert=True,
-  ...     w=0) # unacknowledged write
-  ...
-  ...   # This always runs after update has completed:
-  ...   count = sum([p['n'] for p in counts.find({'region': region})])
-  ... finally:
-  ...   request.end()
-  >>> print count
-  1
-
-Requests can also be used as context managers, with the `with statement
-<http://docs.python.org/reference/compound_stmts.html#index-15>`_, which makes
-the previous example more terse:
-
-.. doctest::
-
-  >>> client.in_request()
-  False
-  >>> with client.start_request():
-  ...   # MongoClient is now in request
-  ...   counts.update(
-  ...     {'region': region, 'browser': browser, 'os': os},
-  ...     {'$inc': {'n': 1 }},
-  ...     upsert=True,
-  ...     safe=False)
-  ...   print sum([p['n'] for p in counts.find({'region': region})])
-  2
-  >>> client.in_request() # request automatically ended
-  False
-
-Requests And ``max_pool_size``
-------------------------------
-
-A thread in a request retains exclusive access to a socket until its request
-ends or the thread dies; thus, applications in which more than 100 threads are
-in requests at once should disable the ``max_pool_size`` option::
-
-    client = MongoClient(host, port, max_pool_size=None)
-
-Failure to increase or disable ``max_pool_size`` in such an application can
-leave threads forever waiting for sockets.
-
-See :ref:`connection-pooling`
+.. _mongos 2.8 doesn't support it at all: https://jira.mongodb.org/browse/SERVER-12273

doc/faq.rst

@@ -98,6 +98,110 @@ For `Twisted <http://twistedmatrix.com/>`_, see `TxMongo
 <http://github.com/fiorix/mongo-async-python-driver>`_. Compared to PyMongo,
 TxMongo is less stable, lacks features, and is less actively maintained.
 
+Key order in subdocuments -- why does my query work in the shell but not PyMongo?
+---------------------------------------------------------------------------------
+
+.. testsetup:: key-order
+
+  from bson.son import SON
+  from pymongo.mongo_client import MongoClient
+
+  collection = MongoClient().test.collection
+  collection.drop()
+  collection.insert({'_id': 1.0,
+                     'subdocument': SON([('b', 1.0), ('a', 1.0)])})
+
+The key-value pairs in a BSON document can have any order (except that ``_id``
+is always first). The mongo shell preserves key order when reading and writing
+data. Observe that "b" comes before "a" when we create the document and when it
+is displayed:
+
+.. code-block:: javascript
+
+  > // mongo shell.
+  > db.collection.insert( { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } )
+  WriteResult({ "nInserted" : 1 })
+  > db.collection.find()
+  { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } }
+
+PyMongo represents BSON documents as Python dicts by default, and the order
+of keys in dicts is not defined. That is, a dict declared with the "a" key
+first is the same, to Python, as one with "b" first:
+
+.. doctest:: key-order
+
+  >>> print {'a': 1.0, 'b': 1.0}
+  {'a': 1.0, 'b': 1.0}
+  >>> print {'b': 1.0, 'a': 1.0}
+  {'a': 1.0, 'b': 1.0}
+
+Therefore, Python dicts are not guaranteed to show keys in the order they are
+stored in BSON. Here, "a" is shown before "b":
+
+.. doctest:: key-order
+
+  >>> print collection.find_one()
+  {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}}
+
+To preserve order when reading BSON, use the :class:`~bson.son.SON` class,
+which is a dict that remembers its key order. Now, documents and subdocuments
+in query results are represented with :class:`~bson.son.SON` objects:
+
+.. doctest:: key-order
+
+  >>> from bson.son import SON
+  >>> print collection.find_one(as_class=SON)
+  SON([(u'_id', 1.0), (u'subdocument', SON([(u'b', 1.0), (u'a', 1.0)]))])
+
+The subdocument's actual storage layout is now visible: "b" is before "a".
+
+Because a dict's key order is not defined, you cannot predict how it will be
+serialized **to** BSON. But MongoDB considers subdocuments equal only if their
+keys have the same order. So if you use a dict to query on a subdocument it may
+not match:
+
+.. doctest:: key-order
+
+  >>> collection.find_one({'subdocument': {'a': 1.0, 'b': 1.0}}) is None
+  True
+
+Swapping the key order in your query makes no difference:
+
+.. doctest:: key-order
+
+  >>> collection.find_one({'subdocument': {'b': 1.0, 'a': 1.0}}) is None
+  True
+
+... because, as we saw above, Python considers the two dicts the same.
+
+There are two solutions. First, you can match the subdocument field-by-field:
+
+.. doctest:: key-order
+
+  >>> collection.find_one({'subdocument.a': 1.0,
+  ...                      'subdocument.b': 1.0})
+  {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}}
+
+The query matches any subdocument with an "a" of 1.0 and a "b" of 1.0,
+regardless of the order you specify them in Python or the order they are stored
+in BSON. Additionally, this query now matches subdocuments with additional
+keys besides "a" and "b", whereas the previous query required an exact match.
+
+The second solution is to use a :class:`~bson.son.SON` to specify the key order:
+
+.. doctest:: key-order
+
+  >>> query = {'subdocument': SON([('b', 1.0), ('a', 1.0)])}
+  >>> collection.find_one(query)
+  {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}}
+
+The key order you use when you create a :class:`~bson.son.SON` is preserved
+when it is serialized to BSON and used as a query. Thus you can create a
+subdocument that exactly matches the subdocument in the collection.
+
+.. seealso:: `MongoDB Manual entry on subdocument matching
+   <http://docs.mongodb.org/manual/tutorial/query-documents/#embedded-documents>`_.
+
 What does *CursorNotFound* cursor id not valid at server mean?
 --------------------------------------------------------------
 Cursors in MongoDB can timeout on the server if they've been open for

doc/index.rst

@@ -25,6 +25,10 @@ everything you need to know to use **PyMongo**.
 :doc:`python3`
   Frequently asked questions about python 3 support.
 
+:doc:`compatibility-policy`
+  Explanation of deprecations, and how to keep pace with changes in PyMongo's
+  API.
+
 :doc:`api/index`
   The complete API documentation, organized by module.
 
@@ -83,9 +87,9 @@ Indices and tables
    tutorial
    examples/index
    faq
+   compatibility-policy
    api/index
    tools
    contributors
    changelog
    python3
-

doc/installation.rst

@@ -5,6 +5,10 @@ Installing / Upgrading
 **PyMongo** is in the `Python Package Index
 <http://pypi.python.org/pypi/pymongo/>`_.
 
+.. warning:: **Do not install the "bson" package.** PyMongo comes with its own
+   bson package; doing "pip install bson" or "easy_install bson" installs a
+   third-party package that is incompatible with PyMongo.
+
 Microsoft Windows
 -----------------
 
@@ -212,9 +216,9 @@ but can be found on the
 `github tags page <https://github.com/mongodb/mongo-python-driver/tags>`_.
 They can be installed by passing the full URL for the tag to pip::
 
-  $ pip install https://github.com/mongodb/mongo-python-driver/archive/2.7rc1.tar.gz
+  $ pip install https://github.com/mongodb/mongo-python-driver/archive/2.8rc2.tar.gz
 
 or easy_install::
 
-  $ easy_install https://github.com/mongodb/mongo-python-driver/archive/2.7rc1.tar.gz
+  $ easy_install https://github.com/mongodb/mongo-python-driver/archive/2.8rc2.tar.gz
 

doc/tools.rst

@@ -71,7 +71,7 @@ Minimongo
   and provides a number of additional features, including a simple
   document-oriented interface, connection pooling, index management, and
   collection & database naming helpers. The `source is on github
-  <http://github.com/slacy/minimongo>`_.
+  <https://github.com/MiniMongo/minimongo>`_.
 
 Manga
   `Manga <http://pypi.python.org/pypi/manga>`_ aims to be a simpler ORM-like
@@ -107,6 +107,9 @@ various Python frameworks and libraries.
   project to enable using MongoDB as a backend for `beaker's
   <http://beaker.groovie.org/>`_ caching / session system.
   `The source is on github <http://github.com/bwmcadams/mongodb_beaker>`_.
+* `Log4Mongo <https://github.com/log4mongo/log4mongo-python>`_ is a flexible
+  Python logging handler that can store logs in MongoDB using normal and capped
+  collections.
 * `MongoLog <http://github.com/puentesarrin/mongodb-log/>`_ is a Python logging
   handler that stores logs in MongoDB using a capped collection.
 * `c5t <http://bitbucket.org/percious/c5t/>`_ is a content-management system

gridfs/__init__.py

@@ -279,6 +279,33 @@ class GridFS(object):
             name for name in self.__files.distinct("filename")
             if name is not None]
 
+    def find_one(self, spec_or_id=None, *args, **kwargs):
+        """Get a single file from gridfs.
+
+        All arguments to :meth:`find` are also valid arguments for
+        :meth:`find_one`, although any `limit` argument will be
+        ignored. Returns a single :class:`~gridfs.grid_file.GridOut`,
+        or ``None`` if no matching file is found. For example::
+
+            file = fs.find_one({"filename": "lisa.txt"})
+
+        :Parameters:
+          - `spec_or_id` (optional): a dictionary specifying
+            the query to be performing OR any other type to be used  as
+            the value for a query for ``"_id"`` in the file collection.
+          - `*args` (optional): any additional positional arguments are
+            the same as the arguments to :meth:`find`.
+          - `**kwargs` (optional): any additional keyword arguments
+            are the same as the arguments to :meth:`find`.
+        """
+        if spec_or_id is not None and not isinstance(spec_or_id, dict):
+            spec_or_id = {"_id": spec_or_id}
+
+        for f in self.find(spec_or_id, *args, **kwargs):
+            return f
+
+        return None
+
     def find(self, *args, **kwargs):
         """Query GridFS for files.
 

gridfs/grid_file.py

@@ -30,6 +30,7 @@ from pymongo import ASCENDING
 from pymongo.collection import Collection
 from pymongo.cursor import Cursor
 from pymongo.errors import DuplicateKeyError
+from pymongo.read_preferences import ReadPreference
 
 try:
     _SEEK_SET = os.SEEK_SET
@@ -110,7 +111,7 @@ class GridIn(object):
             for the file
 
           - ``"chunkSize"`` or ``"chunk_size"``: size of each of the
-            chunks, in bytes (default: 256 kb)
+            chunks, in bytes (default: 255 kb)
 
           - ``"encoding"``: encoding used for this file. In Python 2,
             any :class:`unicode` that is written to the file will be
@@ -255,10 +256,11 @@ class GridIn(object):
             # connection, can succeed out-of-order due to the writebackListener.
             # We mustn't call "filemd5" until all inserts are complete, which
             # we ensure by calling getLastError (and ignoring the result).
-            db.error()
+            db.command('getlasterror', read_preference=ReadPreference.PRIMARY)
 
             md5 = db.command(
-                "filemd5", self._id, root=self._coll.name)["md5"]
+                "filemd5", self._id, root=self._coll.name,
+                read_preference=ReadPreference.PRIMARY)["md5"]
 
             self._file["md5"] = md5
             self._file["length"] = self._position
@@ -439,19 +441,21 @@ class GridOut(object):
         """Reads a chunk at a time. If the current position is within a
         chunk the remainder of the chunk is returned.
         """
+        self._ensure_file()
         received = len(self.__buffer)
         chunk_data = EMPTY
+        chunk_size = int(self.chunk_size)
 
         if received > 0:
             chunk_data = self.__buffer
         elif self.__position < int(self.length):
-            chunk_number = int((received + self.__position) / self.chunk_size)
+            chunk_number = int((received + self.__position) / chunk_size)
             chunk = self.__chunks.find_one({"files_id": self._id,
                                             "n": chunk_number})
             if not chunk:
                 raise CorruptGridFile("no chunk #%d" % chunk_number)
 
-            chunk_data = chunk["data"][self.__position % self.chunk_size:]
+            chunk_data = chunk["data"][self.__position % chunk_size:]
 
         self.__position += len(chunk_data)
         self.__buffer = EMPTY

pymongo/__init__.py

@@ -27,7 +27,7 @@ GEO2D = "2d"
 
 .. note:: Geo-spatial indexing requires server version **>= 1.3.3**.
 
-.. _geospatial index: http://docs.mongodb.org/manual/core/geospatial-indexes/
+.. _geospatial index: http://docs.mongodb.org/manual/core/2d/
 """
 
 GEOHAYSTACK = "geoHaystack"
@@ -37,7 +37,7 @@ GEOHAYSTACK = "geoHaystack"
 
 .. note:: Geo-spatial indexing requires server version **>= 1.5.6**.
 
-.. _haystack index: http://docs.mongodb.org/manual/core/geospatial-indexes/#haystack-indexes
+.. _haystack index: http://docs.mongodb.org/manual/core/geohaystack/
 """
 
 GEOSPHERE = "2dsphere"
@@ -47,7 +47,7 @@ GEOSPHERE = "2dsphere"
 
 .. note:: 2dsphere indexing requires server version **>= 2.4.0**.
 
-.. _spherical geospatial index: http://docs.mongodb.org/manual/release-notes/2.4/#new-geospatial-indexes-with-geojson-and-improved-spherical-geometry
+.. _spherical geospatial index: http://docs.mongodb.org/manual/core/2dsphere/
 """
 
 HASHED = "hashed"
@@ -57,7 +57,17 @@ HASHED = "hashed"
 
 .. note:: hashed indexing requires server version **>= 2.4.0**.
 
-.. _hashed index: http://docs.mongodb.org/manual/release-notes/2.4/#new-hashed-index-and-sharding-with-a-hashed-shard-key
+.. _hashed index: http://docs.mongodb.org/manual/core/index-hashed/
+"""
+
+TEXT = "text"
+"""Index specifier for a `text index`_.
+
+.. versionadded:: 2.7.1
+
+.. note:: text search requires server version **>= 2.4.0**.
+
+.. _text index: http://docs.mongodb.org/manual/core/index-text/
 """
 
 OFF = 0
@@ -67,7 +77,7 @@ SLOW_ONLY = 1
 ALL = 2
 """Profile all operations."""
 
-version_tuple = (2, 7)
+version_tuple = (2, 8, 1)
 
 def get_version_string():
     if isinstance(version_tuple[-1], basestring):

pymongo/_cmessagemodule.c

@@ -1112,8 +1112,10 @@ _cbson_do_batched_write_command(PyObject* self, PyObject* args) {
              */
             buffer_update_position(buffer, sub_doc_begin);
 
-            if (!buffer_write_bytes(buffer, "\x00\x00", 2))
+            if (!buffer_write_bytes(buffer, "\x00\x00", 2)) {
+                buffer_free(new_buffer);
                 goto cmditerfail;
+            }
 
             result = _send_write_command(client, buffer,
                                          lst_len_loc, cmd_len_loc, &errors);

pymongo/auth.py

@@ -15,13 +15,18 @@
 """Authentication helpers."""
 
 import hmac
+import warnings
 try:
     import hashlib
     _MD5 = hashlib.md5
+    _SHA1 = hashlib.sha1
+    _SHA1MOD = _SHA1
     _DMOD = _MD5
 except ImportError:  # for Python < 2.5
-    import md5
+    import md5, sha
     _MD5 = md5.new
+    _SHA1 = sha.new
+    _SHA1MOD = sha
     _DMOD = md5
 
 HAVE_KERBEROS = True
@@ -30,26 +35,244 @@ try:
 except ImportError:
     HAVE_KERBEROS = False
 
+from base64 import standard_b64decode, standard_b64encode
+from random import SystemRandom
+
 from bson.binary import Binary
-from bson.py3compat import b
+from bson.py3compat import b, PY3
 from bson.son import SON
 from pymongo.errors import ConfigurationError, OperationFailure
 
 
-MECHANISMS = frozenset(['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', 'PLAIN'])
+MECHANISMS = frozenset(
+    ['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', 'PLAIN', 'SCRAM-SHA-1', 'DEFAULT'])
 """The authentication mechanisms supported by PyMongo."""
 
 
 def _build_credentials_tuple(mech, source, user, passwd, extra):
     """Build and return a mechanism specific credentials tuple.
     """
+    user = unicode(user)
     if mech == 'GSSAPI':
-        gsn = extra.get('gssapiservicename', 'mongodb')
+        gsn = 'mongodb'
+        if "gssapiservicename" in extra:
+            gsn = extra.get('gssapiservicename')
+            msg = ('The gssapiServiceName option is deprecated. Use '
+                   '"authMechanismProperties=SERVICE_NAME:%s" instead.' % gsn)
+            warnings.warn(msg, DeprecationWarning, stacklevel=3)
+        # SERVICE_NAME overrides gssapiServiceName.
+        if 'authmechanismproperties' in extra:
+            props = extra['authmechanismproperties']
+            if 'SERVICE_NAME' in props:
+                gsn = props.get('SERVICE_NAME')
         # No password, source is always $external.
         return (mech, '$external', user, gsn)
     elif mech == 'MONGODB-X509':
         return (mech, '$external', user)
-    return (mech, source, user, passwd)
+    else:
+        if passwd is None:
+            raise ConfigurationError("A password is required.")
+        return (mech, source, user, unicode(passwd))
+
+
+if PY3:
+    def _xor(fir, sec):
+        """XOR two byte strings together (python 3.x)."""
+        return _EMPTY.join([bytes([x ^ y]) for x, y in zip(fir, sec)])
+
+    _from_bytes = int.from_bytes
+    _to_bytes = int.to_bytes
+else:
+
+    from binascii import (hexlify as _hexlify,
+                          unhexlify as _unhexlify)
+
+    def _xor(fir, sec):
+        """XOR two byte strings together (python 2.x)."""
+        return _EMPTY.join([chr(ord(x) ^ ord(y)) for x, y in zip(fir, sec)])
+
+    def _from_bytes(value, dummy, int=int, _hexlify=_hexlify):
+        """An implementation of int.from_bytes for python 2.x."""
+        return int(_hexlify(value), 16)
+
+    def _to_bytes(value, dummy0, dummy1, _unhexlify=_unhexlify):
+        """An implementation of int.to_bytes for python 2.x."""
+        return _unhexlify('%040x' % value)
+
+
+_BIGONE = b('\x00\x00\x00\x01')
+
+try:
+    # The fastest option, if it's been compiled to use OpenSSL's HMAC.
+    from backports.pbkdf2 import pbkdf2_hmac
+
+    def _hi(data, salt, iterations):
+        return pbkdf2_hmac('sha1', data, salt, iterations)
+
+except ImportError:
+    try:
+        # Python 2.7.8+, or Python 3.4+.
+        from hashlib import pbkdf2_hmac
+
+        def _hi(data, salt, iterations):
+            return pbkdf2_hmac('sha1', data, salt, iterations)
+
+    except ImportError:
+
+        def _hi(data, salt, iterations):
+            """A simple implementation of PBKDF2."""
+            mac = hmac.HMAC(data, None, _SHA1MOD)
+
+            def _digest(msg, mac=mac):
+                """Get a digest for msg."""
+                _mac = mac.copy()
+                _mac.update(msg)
+                return _mac.digest()
+
+            from_bytes = _from_bytes
+            to_bytes = _to_bytes
+
+            _u1 = _digest(salt + _BIGONE)
+            _ui = from_bytes(_u1, 'big')
+            for _ in range(iterations - 1):
+                _u1 = _digest(_u1)
+                _ui ^= from_bytes(_u1, 'big')
+            return to_bytes(_ui, 20, 'big')
+
+
+_EMPTY = b("")
+_COMMA = b(",")
+_EQUAL = b("=")
+
+def _parse_scram_response(response):
+    """Split a scram response into key, value pairs."""
+    return dict([item.split(_EQUAL, 1) for item in response.split(_COMMA)])
+
+
+def _scram_sha1_conversation(
+        credentials,
+        sock_info,
+        cmd_func,
+        sasl_start,
+        sasl_continue):
+    """Authenticate or copydb using SCRAM-SHA-1.
+
+    sasl_start and sasl_continue are SONs, the base command documents for
+    beginning and continuing the SASL conversation. They may be modified
+    by the callee.
+
+    :Parameters:
+      - `credentials`: A credentials tuple from _build_credentials_tuple.
+      - `sock_info`: A SocketInfo instance.
+      - `cmd_func`: A callback taking args sock_info, database, command doc.
+      - `sasl_start`: A SON.
+      - `sasl_continue`: A SON.
+    """
+    source, username, password = credentials
+
+    # Make local
+    _hmac = hmac.HMAC
+    _sha1 = _SHA1
+    _sha1mod = _SHA1MOD
+
+    user = username.encode("utf-8").replace(
+        _EQUAL, b("=3D")).replace(_COMMA, b("=2C"))
+    nonce = standard_b64encode(
+        (("%s" % (SystemRandom().random(),))[2:]).encode("utf-8"))
+    first_bare = b("n=") + user + b(",r=") + nonce
+
+    sasl_start['payload'] = Binary(b("n,,") + first_bare)
+    res, _ = cmd_func(sock_info, source, sasl_start)
+
+    server_first = res['payload']
+    parsed = _parse_scram_response(server_first)
+    iterations = int(parsed[b('i')])
+    salt = parsed[b('s')]
+    rnonce = parsed[b('r')]
+    if not rnonce.startswith(nonce):
+        raise OperationFailure("Server returned an invalid nonce.")
+
+    without_proof = b("c=biws,r=") + rnonce
+    salted_pass = _hi(_password_digest(username, password).encode("utf-8"),
+                      standard_b64decode(salt),
+                      iterations)
+    client_key = _hmac(salted_pass, b("Client Key"), _sha1mod).digest()
+    stored_key = _sha1(client_key).digest()
+    auth_msg = _COMMA.join((first_bare, server_first, without_proof))
+    client_sig = _hmac(stored_key, auth_msg, _sha1mod).digest()
+    client_proof = b("p=") + standard_b64encode(_xor(client_key, client_sig))
+    client_final = _COMMA.join((without_proof, client_proof))
+
+    server_key = _hmac(salted_pass, b("Server Key"), _sha1mod).digest()
+    server_sig = standard_b64encode(
+        _hmac(server_key, auth_msg, _SHA1MOD).digest())
+
+    cmd = sasl_continue.copy()
+    cmd['conversationId'] = res['conversationId']
+    cmd['payload'] = Binary(client_final)
+    res, _ = cmd_func(sock_info, source, cmd)
+
+    parsed = _parse_scram_response(res['payload'])
+    if parsed[b('v')] != server_sig:
+        raise OperationFailure("Server returned an invalid signature.")
+
+    # Depending on how it's configured, Cyrus SASL (which the server uses)
+    # requires a third empty challenge.
+    if not res['done']:
+        cmd = sasl_continue.copy()
+        cmd['conversationId'] = res['conversationId']
+        cmd['payload'] = Binary(_EMPTY)
+        res, _ = cmd_func(sock_info, source, cmd)
+        if not res['done']:
+            raise OperationFailure('SASL conversation failed to complete.')
+
+
+def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
+    """Authenticate using SCRAM-SHA-1."""
+    # Base commands for starting and continuing SASL authentication.
+    sasl_start = SON([('saslStart', 1),
+                      ('mechanism', 'SCRAM-SHA-1'),
+                      ('autoAuthorize', 1)])
+    sasl_continue = SON([('saslContinue', 1)])
+    _scram_sha1_conversation(credentials, sock_info, cmd_func,
+                             sasl_start, sasl_continue)
+
+
+def _copydb_scram_sha1(
+        credentials,
+        sock_info,
+        cmd_func,
+        fromdb,
+        todb,
+        fromhost):
+    """Copy a database using SCRAM-SHA-1 authentication.
+
+    :Parameters:
+      - `credentials`: A tuple, (mechanism, source, username, password).
+      - `sock_info`: A SocketInfo instance.
+      - `cmd_func`: A callback taking args sock_info, database, command doc.
+      - `fromdb`: Source database.
+      - `todb`: Target database.
+      - `fromhost`: Source host or None.
+    """
+    assert credentials[0] == 'SCRAM-SHA-1'
+
+    sasl_start = SON([('copydbsaslstart', 1),
+                      ('mechanism', 'SCRAM-SHA-1'),
+                      ('autoAuthorize', 1),
+                      ('fromdb', fromdb),
+                      ('fromhost', fromhost)])
+
+    sasl_continue = SON([('copydb', 1),
+                         ('fromdb', fromdb),
+                         ('fromhost', fromhost),
+                         ('todb', todb)])
+
+    _scram_sha1_conversation(credentials[1:],
+                             sock_info,
+                             cmd_func,
+                             sasl_start,
+                             sasl_continue)
 
 
 def _password_digest(username, password):
@@ -75,7 +298,7 @@ def _auth_key(nonce, username, password):
     """
     digest = _password_digest(username, password)
     md5hash = _MD5()
-    data = "%s%s%s" % (nonce, unicode(username), digest)
+    data = "%s%s%s" % (nonce, username, digest)
     md5hash.update(data.encode('utf-8'))
     return unicode(md5hash.hexdigest())
 
@@ -222,12 +445,21 @@ def _authenticate_mongo_cr(credentials, sock_info, cmd_func):
     cmd_func(sock_info, source, query)
 
 
+def _authenticate_default(credentials, sock_info, cmd_func):
+    if sock_info.max_wire_version >= 3:
+        return _authenticate_scram_sha1(credentials, sock_info, cmd_func)
+    else:
+        return _authenticate_mongo_cr(credentials, sock_info, cmd_func)
+
+
 _AUTH_MAP = {
     'CRAM-MD5': _authenticate_cram_md5,
     'GSSAPI': _authenticate_gssapi,
     'MONGODB-CR': _authenticate_mongo_cr,
     'MONGODB-X509': _authenticate_x509,
     'PLAIN': _authenticate_plain,
+    'SCRAM-SHA-1': _authenticate_scram_sha1,
+    'DEFAULT': _authenticate_default,
 }
 
 

pymongo/bulk.py

@@ -82,16 +82,6 @@ def _make_error(index, code, errmsg, operation):
 def _merge_legacy(run, full_result, result, index):
     """Merge a result from a legacy opcode into the full results.
     """
-    # MongoDB 2.6 returns {'ok': 0, 'code': 2, ...} if the j write
-    # concern option is used with --nojournal or w > 1 is used with
-    # a standalone mongod instance. Raise immediately here for
-    # consistency when talking to older servers. Since these are
-    # configuration errors related to write concern the entire batch
-    # will fail.
-    note = result.get("jnote", result.get("wnote"))
-    if note:
-        raise OperationFailure(note, _BAD_VALUE, result)
-
     affected = result.get('n', 0)
 
     errmsg = result.get("errmsg", result.get("err", ""))
@@ -111,13 +101,25 @@ def _merge_legacy(run, full_result, result, index):
 
     if run.op_type == _INSERT:
         full_result['nInserted'] += 1
+
     elif run.op_type == _UPDATE:
         if "upserted" in result:
             doc = {u"index": run.index(index), u"_id": result["upserted"]}
             full_result["upserted"].append(doc)
             full_result['nUpserted'] += affected
+        # Versions of MongoDB before 2.6 don't return the _id for an
+        # upsert if _id is not an ObjectId.
+        elif result.get("updatedExisting") == False and affected == 1:
+            op = run.ops[index]
+            # If _id is in both the update document *and* the query spec
+            # the update document _id takes precedence.
+            _id = op['u'].get('_id', op['q'].get('_id'))
+            doc = {u"index": run.index(index), u"_id": _id}
+            full_result["upserted"].append(doc)
+            full_result['nUpserted'] += affected
         else:
             full_result['nMatched'] += affected
+
     elif run.op_type == _DELETE:
         full_result['nRemoved'] += affected
 

pymongo/collection.py

@@ -28,6 +28,7 @@ from pymongo.cursor import Cursor
 from pymongo.errors import InvalidName, OperationFailure
 from pymongo.helpers import _check_write_command_response
 from pymongo.message import _INSERT, _UPDATE, _DELETE
+from pymongo.read_preferences import ReadPreference
 
 
 try:
@@ -125,9 +126,12 @@ class Collection(common.BaseObject):
         if options:
             if "size" in options:
                 options["size"] = float(options["size"])
-            self.__database.command("create", self.__name, **options)
+            self.__database.command("create", self.__name,
+                                    read_preference=ReadPreference.PRIMARY,
+                                    **options)
         else:
-            self.__database.command("create", self.__name)
+            self.__database.command("create", self.__name,
+                                    read_preference=ReadPreference.PRIMARY)
 
     def __getattr__(self, name):
         """Get a sub-collection of this collection by name.
@@ -352,7 +356,7 @@ class Collection(common.BaseObject):
            Support for passing `getLastError` options as keyword
            arguments.
         .. versionchanged:: 1.1
-           Bulk insert works with any iterable
+           Bulk insert works with an iterable sequence of documents.
 
         .. mongodoc:: insert
         """
@@ -374,11 +378,14 @@ class Collection(common.BaseObject):
             def gen():
                 db = self.__database
                 for doc in docs:
+                    # Apply user-configured SON manipulators. This order of
+                    # operations is required for backwards compatibility,
+                    # see PYTHON-709.
+                    doc = db._apply_incoming_manipulators(doc, self)
                     if '_id' not in doc:
                         doc['_id'] = ObjectId()
 
-                    # Apply user-configured SON manipulators.
-                    doc = db._fix_incoming(doc, self)
+                    doc = db._apply_incoming_copying_manipulators(doc, self)
                     ids.append(doc['_id'])
                     yield doc
         else:
@@ -550,6 +557,10 @@ class Collection(common.BaseObject):
                 result['updatedExisting'] = True
             else:
                 result['updatedExisting'] = False
+                # MongoDB >= 2.6.0 returns the upsert _id in an array
+                # element. Break it out for backward compatibility.
+                if isinstance(result.get('upserted'), list):
+                    result['upserted'] = result['upserted'][0]['_id']
 
             return result
 
@@ -942,32 +953,37 @@ class Collection(common.BaseObject):
 
         Takes either a single key or a list of (key, direction) pairs.
         The key(s) must be an instance of :class:`basestring`
-        (:class:`str` in python 3), and the direction(s) must be one of
+        (:class:`str` in python 3), and the direction(s) should be one of
         (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`,
         :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`,
-        :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`).
+        :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`,
+        :data:`~pymongo.TEXT`).
 
-        To create a single key index on the key ``'mike'`` we just use
-        a string argument:
+        To create a simple ascending index on the key ``'mike'`` we just
+        use a string argument::
 
-        >>> my_collection.create_index("mike")
+          >>> my_collection.create_index("mike")
 
         For a compound index on ``'mike'`` descending and ``'eliot'``
-        ascending we need to use a list of tuples:
+        ascending we need to use a list of tuples::
 
-        >>> my_collection.create_index([("mike", pymongo.DESCENDING),
-        ...                             ("eliot", pymongo.ASCENDING)])
+          >>> my_collection.create_index([("mike", pymongo.DESCENDING),
+          ...                             ("eliot", pymongo.ASCENDING)])
 
         All optional index creation parameters should be passed as
-        keyword arguments to this method. Valid options include:
+        keyword arguments to this method. For example::
+
+          >>> my_collection.create_index([("mike", pymongo.DESCENDING)],
+          ...                            background=True)
+
+        Valid options include, but are not limited to:
 
           - `name`: custom name to use for this index - if none is
             given, a name will be generated
-          - `unique`: should this index guarantee uniqueness?
-          - `dropDups` or `drop_dups`: should we drop duplicates
-          - `background`: if this index should be created in the
+          - `unique`: if ``True`` creates a unique constraint on the index
+          - `background`: if ``True`` this index should be created in the
             background
-          - `sparse`: if True, omit from the index any documents that lack
+          - `sparse`: if ``True``, omit from the index any documents that lack
             the indexed field
           - `bucketSize` or `bucket_size`: for use with geoHaystack indexes.
             Number of documents to group together within a certain proximity
@@ -980,6 +996,17 @@ class Collection(common.BaseObject):
             collection. MongoDB will automatically delete documents from
             this collection after <int> seconds. The indexed field must
             be a UTC datetime or the data will not expire.
+          - `dropDups` or `drop_dups` (**deprecated**): if ``True`` duplicate
+            values are dropped during index creation when creating a unique
+            index
+
+        See the MongoDB documentation for a full list of supported options by
+        server version.
+
+        .. warning:: `dropDups` / `drop_dups` is no longer supported by
+          MongoDB starting with server version 2.7.5. The option is silently
+          ignored by the server and unique index builds using the option will
+          fail if a duplicate value is detected.
 
         .. note:: `expireAfterSeconds` requires server version **>= 2.1.2**
 
@@ -1037,9 +1064,11 @@ class Collection(common.BaseObject):
         index.update(kwargs)
 
         try:
-            self.__database.command('createIndexes', self.name, indexes=[index])
+            self.__database.command('createIndexes', self.name,
+                                    read_preference=ReadPreference.PRIMARY,
+                                    indexes=[index])
         except OperationFailure, exc:
-            if exc.code in (59, None):
+            if exc.code in common.COMMAND_NOT_FOUND_CODES:
                 index["ns"] = self.__full_name
                 self.__database.system.indexes.insert(index, manipulate=False,
                                                       check_keys=False,
@@ -1057,11 +1086,13 @@ class Collection(common.BaseObject):
 
         Takes either a single key or a list of (key, direction) pairs.
         The key(s) must be an instance of :class:`basestring`
-        (:class:`str` in python 3), and the direction(s) must be one of
+        (:class:`str` in python 3), and the direction(s) should be one of
         (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`,
         :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`,
-        :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`).
-        See :meth:`create_index` for a detailed example.
+        :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`,
+        :data:`pymongo.TEXT`).
+
+        See :meth:`create_index` for detailed examples.
 
         Unlike :meth:`create_index`, which attempts to create an index
         unconditionally, :meth:`ensure_index` takes advantage of some
@@ -1083,16 +1114,15 @@ class Collection(common.BaseObject):
         ``None`` if the index is already cached.
 
         All optional index creation parameters should be passed as
-        keyword arguments to this method. Valid options include:
+        keyword arguments to this method. Valid options include, but are not
+        limited to:
 
           - `name`: custom name to use for this index - if none is
             given, a name will be generated
-          - `unique`: should this index guarantee uniqueness?
-          - `dropDups` or `drop_dups`: should we drop duplicates
-            during index creation when creating a unique index?
-          - `background`: if this index should be created in the
+          - `unique`: if ``True`` creates a unique constraint on the index
+          - `background`: if ``True`` this index should be created in the
             background
-          - `sparse`: if True, omit from the index any documents that lack
+          - `sparse`: if ``True``, omit from the index any documents that lack
             the indexed field
           - `bucketSize` or `bucket_size`: for use with geoHaystack indexes.
             Number of documents to group together within a certain proximity
@@ -1105,6 +1135,17 @@ class Collection(common.BaseObject):
             collection. MongoDB will automatically delete documents from
             this collection after <int> seconds. The indexed field must
             be a UTC datetime or the data will not expire.
+          - `dropDups` or `drop_dups` (**deprecated**): if ``True`` duplicate
+            values are dropped during index creation when creating a unique
+            index
+
+        See the MongoDB documentation for a full list of supported options by
+        server version.
+
+        .. warning:: `dropDups` / `drop_dups` is no longer supported by
+          MongoDB starting with server version 2.7.5. The option is silently
+          ignored by the server and unique index builds using the option will
+          fail if a duplicate value is detected.
 
         .. note:: `expireAfterSeconds` requires server version **>= 2.1.2**
 
@@ -1159,7 +1200,8 @@ class Collection(common.BaseObject):
         """Drops the specified index on this collection.
 
         Can be used on non-existant collections or collections with no
-        indexes.  Raises OperationFailure on an error. `index_or_name`
+        indexes.  Raises OperationFailure on an error (e.g. trying to
+        drop an index that does not exist). `index_or_name`
         can be either an index name (as returned by `create_index`),
         or an index specifier (as passed to `create_index`). An index
         specifier should be a list of (key, direction) pairs. Raises
@@ -1183,7 +1225,9 @@ class Collection(common.BaseObject):
 
         self.__database.connection._purge_index(self.__database.name,
                                                 self.__name, name)
-        self.__database.command("dropIndexes", self.__name, index=name,
+        self.__database.command("dropIndexes", self.__name,
+                                read_preference=ReadPreference.PRIMARY,
+                                index=name,
                                 allowable_errors=["ns not found"])
 
     def reindex(self):
@@ -1195,7 +1239,8 @@ class Collection(common.BaseObject):
 
         .. versionadded:: 1.11+
         """
-        return self.__database.command("reIndex", self.__name)
+        return self.__database.command("reIndex", self.__name,
+                                       read_preference=ReadPreference.PRIMARY)
 
     def index_information(self):
         """Get information on this collection's indexes.
@@ -1206,7 +1251,7 @@ class Collection(common.BaseObject):
         guaranteed to contain at least a single key, ``"key"`` which
         is a list of (key, direction) pairs specifying the index (as
         passed to create_index()). It will also contain any other
-        information in `system.indexes`, except for the ``"ns"`` and
+        metadata about the indexes, except for the ``"ns"`` and
         ``"name"`` keys, which are cleaned. Example output might look
         like this:
 
@@ -1222,8 +1267,26 @@ class Collection(common.BaseObject):
            themselves, whose ``"key"`` item contains the list that was
            the value in previous versions of PyMongo.
         """
-        raw = self.__database.system.indexes.find({"ns": self.__full_name},
-                                                  {"ns": 0}, as_class=SON)
+        client = self.database.connection
+        client._ensure_connected(True)
+
+        slave_okay = not client._rs_client and not client.is_mongos
+        if client.max_wire_version > 2:
+            res, addr = self.__database._command(
+                "listIndexes", self.__name, as_class=SON,
+                cursor={}, slave_okay=slave_okay,
+                read_preference=ReadPreference.PRIMARY)
+            # MongoDB 2.8rc2
+            if "indexes" in res:
+                raw = res["indexes"]
+            # >= MongoDB 2.8rc3
+            else:
+                raw = CommandCursor(self, res["cursor"], addr)
+        else:
+            raw = self.__database.system.indexes.find({"ns": self.__full_name},
+                                                      {"ns": 0}, as_class=SON,
+                                                      slave_okay=slave_okay,
+                                                      _must_use_master=True)
         info = {}
         for index in raw:
             index["key"] = index["key"].items()
@@ -1239,8 +1302,32 @@ class Collection(common.BaseObject):
         information on the possible options. Returns an empty
         dictionary if the collection has not been created yet.
         """
-        result = self.__database.system.namespaces.find_one(
-            {"name": self.__full_name})
+        client = self.database.connection
+        client._ensure_connected(True)
+
+        result = None
+        slave_okay = not client._rs_client and not client.is_mongos
+        if client.max_wire_version > 2:
+            res, addr = self.__database._command(
+                "listCollections",
+                cursor={},
+                filter={"name": self.__name},
+                read_preference=ReadPreference.PRIMARY,
+                slave_okay=slave_okay)
+            # MongoDB 2.8rc2
+            if "collections" in res:
+                results = res["collections"]
+            # >= MongoDB 2.8rc3
+            else:
+                results = CommandCursor(self, res["cursor"], addr)
+            for doc in results:
+                result = doc
+                break
+        else:
+            result = self.__database.system.namespaces.find_one(
+                {"name": self.__full_name},
+                slave_okay=slave_okay,
+                _must_use_master=True)
 
         if not result:
             return {}
@@ -1416,9 +1503,10 @@ class Collection(common.BaseObject):
             raise InvalidName("collection names must not contain '$'")
 
         new_name = "%s.%s" % (self.__database.name, new_name)
-        self.__database.connection.admin.command("renameCollection",
-                                                 self.__full_name,
-                                                 to=new_name, **kwargs)
+        client = self.__database.connection
+        client.admin.command("renameCollection", self.__full_name,
+                             read_preference=ReadPreference.PRIMARY,
+                             to=new_name, **kwargs)
 
     def distinct(self, key):
         """Get a list of distinct values for `key` among all documents
@@ -1561,7 +1649,8 @@ class Collection(common.BaseObject):
             return res.get("results")
 
     def find_and_modify(self, query={}, update=None,
-                        upsert=False, sort=None, full_response=False, **kwargs):
+                        upsert=False, sort=None, full_response=False,
+                        manipulate=False, **kwargs):
         """Update and return an object.
 
         This is a thin wrapper around the findAndModify_ command. The
@@ -1593,6 +1682,9 @@ class Collection(common.BaseObject):
             - `new`: return updated rather than original object
               (default ``False``)
             - `fields`: see second argument to :meth:`find` (default all)
+            - `manipulate`: (optional): If ``True``, apply any outgoing SON
+              manipulators before returning. Ignored when `full_response`
+              is set to True. Defaults to ``False``.
             - `**kwargs`: any other options the findAndModify_ command
               supports can be passed here.
 
@@ -1603,6 +1695,9 @@ class Collection(common.BaseObject):
 
         .. note:: Requires server version **>= 1.3.0**
 
+        .. versionchanged:: 2.8
+           Added the optional manipulate parameter
+
         .. versionchanged:: 2.5
            Added the optional full_response parameter
 
@@ -1645,6 +1740,7 @@ class Collection(common.BaseObject):
 
         out = self.__database.command("findAndModify", self.__name,
                                       allowable_errors=[no_obj_error],
+                                      read_preference=ReadPreference.PRIMARY,
                                       uuid_subtype=self.uuid_subtype,
                                       **kwargs)
 
@@ -1658,7 +1754,10 @@ class Collection(common.BaseObject):
         if full_response:
             return out
         else:
-            return out.get('value')
+            document = out.get('value')
+            if manipulate:
+                document = self.__database._fix_outgoing(document, self)
+            return document
 
     def __iter__(self):
         return self

pymongo/command_cursor.py

@@ -40,7 +40,12 @@ class CommandCursor(object):
         )
         self.__retrieved = retrieved
         self.__batch_size = 0
-        self.__killed = False
+        self.__killed = (self.__id == 0)
+
+        if "ns" in cursor_info:
+            self.__ns = cursor_info["ns"]
+        else:
+            self.__ns = collection.full_name
 
     def __del__(self):
         if self.__id and not self.__killed:
@@ -118,6 +123,8 @@ class CommandCursor(object):
             client.disconnect()
             raise
         self.__id = response["cursor_id"]
+        if self.__id == 0:
+            self.__killed = True
 
         assert response["starting_from"] == self.__retrieved, (
             "Result batch started from %s, expected %s" % (
@@ -138,7 +145,7 @@ class CommandCursor(object):
 
         if self.__id:  # Get More
             self.__send_message(
-                message.get_more(self.__collection.full_name,
+                message.get_more(self.__ns,
                                  self.__batch_size, self.__id))
 
         else:  # Cursor id is zero nothing else to return
@@ -148,7 +155,19 @@ class CommandCursor(object):
 
     @property
     def alive(self):
-        """Does this cursor have the potential to return more data?"""
+        """Does this cursor have the potential to return more data?
+
+        Even if :attr:`alive` is ``True``, :meth:`next` can raise
+        :exc:`StopIteration`. Best to use a for loop::
+
+            for doc in collection.aggregate(pipeline):
+                print(doc)
+
+        .. note:: :attr:`alive` can be True while iterating a cursor from
+          a failed server. In this case :attr:`alive` will return False after
+          :meth:`next` fails to retrieve the next batch of results from the
+          server.
+        """
         return bool(len(self.__data) or (not self.__killed))
 
     @property
@@ -160,11 +179,10 @@ class CommandCursor(object):
         return self
 
     def next(self):
-        """Advance the cursor.
-        """
+        """Advance the cursor."""
         if len(self.__data) or self._refresh():
             coll = self.__collection
-            return coll.database._fix_incoming(self.__data.popleft(), coll)
+            return coll.database._fix_outgoing(self.__data.popleft(), coll)
         else:
             raise StopIteration
 

pymongo/common.py

@@ -45,7 +45,15 @@ MAX_WRITE_BATCH_SIZE = 1000
 
 # What this version of PyMongo supports.
 MIN_SUPPORTED_WIRE_VERSION = 0
-MAX_SUPPORTED_WIRE_VERSION = 2
+MAX_SUPPORTED_WIRE_VERSION = 3
+
+# mongod/s 2.6 and above return code 59 when a
+# command doesn't exist. mongod versions previous
+# to 2.6 and mongos 2.4.x return no error code
+# when a command does exist. mongos versions previous
+# to 2.4.0 return code 13390 when a command does not
+# exist.
+COMMAND_NOT_FOUND_CODES = (59, 13390, None)
 
 
 def raise_config_error(key, dummy):
@@ -101,6 +109,8 @@ def validate_positive_integer(option, value):
 def validate_readable(option, value):
     """Validates that 'value' is file-like and readable.
     """
+    if value is None:
+        return value
     # First make sure its a string py3.3 open(True, 'r') succeeds
     # Used in ssl cert checking due to poor ssl module error reporting
     value = validate_basestring(option, value)
@@ -114,6 +124,8 @@ def validate_cert_reqs(option, value):
     if value is None:
         return value
     if HAS_SSL:
+        if isinstance(value, basestring) and hasattr(ssl, value):
+            value = getattr(ssl, value)
         if value in (ssl.CERT_NONE, ssl.CERT_OPTIONAL, ssl.CERT_REQUIRED):
             return value
         raise ConfigurationError("The value of %s must be one of: "
@@ -142,6 +154,14 @@ def validate_basestring(option, value):
                     "instance of %s" % (option, basestring.__name__))
 
 
+def validate_basestring_or_none(option, value):
+    """Validates that 'value' is an instance of `basestring` or `None`.
+    """
+    if value is None:
+        return value
+    return validate_basestring(option, value)
+
+
 def validate_int_or_basestring(option, value):
     """Validates that 'value' is an integer or string.
     """
@@ -182,6 +202,15 @@ def validate_timeout_or_none(option, value):
     return validate_positive_float(option, value) / 1000.0
 
 
+def validate_positive_float_or_zero(option, value):
+    """Validates that 'value' is 0 or a positive float or can be converted to
+    0 or a positive float.
+    """
+    if value == 0 or value == "0":
+        return 0
+    return validate_positive_float(option, value)
+
+
 def validate_read_preference(dummy, value):
     """Validate read preference for a ReplicaSetConnection.
     """
@@ -247,11 +276,33 @@ def validate_uuid_subtype(dummy, value):
     return value
 
 
+_MECHANISM_PROPS = frozenset(['SERVICE_NAME'])
+
+
+def validate_auth_mechanism_properties(option, value):
+    """Validate authMechanismProperties."""
+    value = validate_basestring(option, value)
+    props = {}
+    for opt in value.split(','):
+        try:
+            key, val = opt.split(':')
+            if key not in _MECHANISM_PROPS:
+                raise ConfigurationError("%s is not a supported auth "
+                                         "mechanism property. Must be one of "
+                                         "%s." % (key, tuple(_MECHANISM_PROPS)))
+            props[key] = val
+        except ValueError:
+            raise ConfigurationError("auth mechanism properties must be "
+                                     "key:value pairs like SERVICE_NAME:"
+                                     "mongodb, not %s." % (opt,))
+    return props
+
+
 # jounal is an alias for j,
 # wtimeoutms is an alias for wtimeout,
 # readpreferencetags is an alias for tag_sets.
 VALIDATORS = {
-    'replicaset': validate_basestring,
+    'replicaset': validate_basestring_or_none,
     'slaveok': validate_boolean,
     'slave_okay': validate_boolean,
     'safe': validate_boolean,
@@ -274,18 +325,20 @@ VALIDATORS = {
     'read_preference': validate_read_preference,
     'readpreferencetags': validate_tag_sets,
     'tag_sets': validate_tag_sets,
-    'secondaryacceptablelatencyms': validate_positive_float,
-    'secondary_acceptable_latency_ms': validate_positive_float,
+    'secondaryacceptablelatencyms': validate_positive_float_or_zero,
+    'secondary_acceptable_latency_ms': validate_positive_float_or_zero,
     'auto_start_request': validate_boolean,
     'use_greenlets': validate_boolean,
     'authmechanism': validate_auth_mechanism,
     'authsource': validate_basestring,
     'gssapiservicename': validate_basestring,
+    'authmechanismproperties': validate_auth_mechanism_properties,
     'uuidrepresentation': validate_uuid_representation,
+    'socketkeepalive': validate_boolean
 }
 
 
-_AUTH_OPTIONS = frozenset(['gssapiservicename'])
+_AUTH_OPTIONS = frozenset(['gssapiservicename', 'authmechanismproperties'])
 
 
 def validate_auth_option(option, value):
@@ -393,12 +446,10 @@ class BaseObject(object):
                 self.__tag_sets = validate_tag_sets(option, value)
             elif option == 'uuidrepresentation':
                 self.__uuid_subtype = validate_uuid_subtype(option, value)
-            elif option in (
-                'secondaryacceptablelatencyms',
-                'secondary_acceptable_latency_ms'
-            ):
-                self.__secondary_acceptable_latency_ms = \
-                    validate_positive_float(option, value)
+            elif option in ('secondaryacceptablelatencyms',
+                            'secondary_acceptable_latency_ms'):
+                self.__secondary_acceptable_latency_ms = (
+                    validate_positive_float_or_zero(option, value))
             elif option in SAFE_OPTIONS:
                 if option == 'journal':
                     self.__set_safe_option('j', value)
@@ -535,8 +586,9 @@ class BaseObject(object):
 
     def __set_acceptable_latency(self, value):
         """Property setter for secondary_acceptable_latency_ms"""
-        self.__secondary_acceptable_latency_ms = (validate_positive_float(
-            'secondary_acceptable_latency_ms', value))
+        self.__secondary_acceptable_latency_ms = (
+            validate_positive_float_or_zero(
+                'secondary_acceptable_latency_ms', value))
 
     secondary_acceptable_latency_ms = property(
         __get_acceptable_latency, __set_acceptable_latency)
@@ -680,12 +732,6 @@ class BaseObject(object):
 
         .. versionadded:: 2.3
         """
-        # Don't ever send w=1 to the server.
-        def pop1(dct):
-            if dct.get('w') == 1:
-                dct.pop('w')
-            return dct
-
         if safe is not None:
             warnings.warn("The safe parameter is deprecated. Please use "
                           "write concern options instead.", DeprecationWarning,
@@ -706,7 +752,7 @@ class BaseObject(object):
                     if options.get('w') == 0:
                         return True, {}
                 # Passing w=0 overrides passing safe=True.
-                return options.get('w') != 0, pop1(options)
+                return options.get('w') != 0, options
             return False, {}
 
         # Fall back to collection level defaults.
@@ -714,6 +760,6 @@ class BaseObject(object):
         if self.__write_concern.get('w') == 0:
             return False, {}
         elif self.safe or self.__write_concern.get('w', 0) != 0:
-            return True, pop1(self.__write_concern.copy())
+            return True, self.__write_concern.copy()
 
         return False, {}

pymongo/connection.py

@@ -99,26 +99,27 @@ class Connection(MongoClient):
 
           | **Other optional parameters can be passed as keyword arguments:**
 
-          - `socketTimeoutMS`: (integer) How long (in milliseconds) a send or
-            receive on a socket can take before timing out. Defaults to ``None``
-            (no timeout).
-          - `connectTimeoutMS`: (integer) How long (in milliseconds) a
+          - `socketTimeoutMS`: (integer or None) How long (in milliseconds) a
+            send or receive on a socket can take before timing out. Defaults to
+            ``None`` (no timeout).
+          - `connectTimeoutMS`: (integer or None) How long (in milliseconds) a
             connection can take to be opened before timing out. Defaults to
             ``20000``.
-          - `waitQueueTimeoutMS`: (integer) How long (in milliseconds) a
-            thread will wait for a socket from the pool if the pool has no
+          - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds)
+            a thread will wait for a socket from the pool if the pool has no
             free sockets. Defaults to ``None`` (no timeout).
-          - `waitQueueMultiple`: (integer) Multiplied by max_pool_size to give
-            the number of threads allowed to wait for a socket at one time.
-            Defaults to ``None`` (no waiters).
-
+          - `waitQueueMultiple`: (integer or None) Multiplied by max_pool_size
+            to give the number of threads allowed to wait for a socket at one
+            time. Defaults to ``None`` (no waiters).
+          - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
           - `auto_start_request`: If ``True`` (the default), each thread that
             accesses this Connection has a socket allocated to it for the
-            thread's lifetime.  This ensures consistent reads, even if you read
-            after an unsafe write.
+            thread's lifetime, or until :meth:`end_request` is called.
           - `use_greenlets`: if ``True``, :meth:`start_request()` will ensure
             that the current greenlet uses the same socket for all operations
-            until :meth:`end_request()`
+            until :meth:`end_request()`. Defaults to ``False``.
 
           | **Write Concern options:**
 
@@ -155,27 +156,31 @@ class Connection(MongoClient):
             The driver will verify that the replica-set it connects to matches
             this name. Implies that the hosts specified are a seed list and the
             driver should attempt to find all members of the set. *Ignored by
-            mongos*.
+            mongos*. Defaults to ``None``.
           - `read_preference`: The read preference for this client. If
             connecting to a secondary then a read preference mode *other* than
             PRIMARY is required - otherwise all queries will throw a
             :class:`~pymongo.errors.AutoReconnect` "not master" error.
             See :class:`~pymongo.read_preferences.ReadPreference` for all
-            available read preference options.
+            available read preference options. Defaults to ``PRIMARY``.
           - `tag_sets`: Ignored unless connecting to a replica-set via mongos.
             Specify a priority-order for tag sets, provide a list of
             tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
             set, ``{}``, means "read from any member that matches the mode,
-            ignoring tags.
+            ignoring tags. Defaults to ``[{}]``, meaning "ignore members'
+            tags."
 
           | **SSL configuration:**
 
           - `ssl`: If ``True``, create the connection to the server using SSL.
+            Defaults to ``False``.
           - `ssl_keyfile`: The private keyfile used to identify the local
             connection against mongod.  If included with the ``certfile` then
             only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
+            Defaults to ``None``.
           - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``ssl=True``.
+            connection against mongod. Implies ``ssl=True``. Defaults to
+            ``None``.
           - `ssl_cert_reqs`: The parameter cert_reqs specifies whether a
             certificate is required from the other side of the connection,
             and whether it will be validated if provided. It must be one of the
@@ -184,11 +189,11 @@ class Connection(MongoClient):
             ``ssl.CERT_REQUIRED`` (required and validated). If the value of
             this parameter is not ``ssl.CERT_NONE``, then the ``ssl_ca_certs``
             parameter must point to a file of CA certificates.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``ssl.CERT_NONE``.
           - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``None``.
 
         .. seealso:: :meth:`end_request`
         .. versionchanged:: 2.5

pymongo/cursor.py

@@ -22,8 +22,8 @@ from bson.son import SON
 from pymongo import helpers, message, read_preferences
 from pymongo.read_preferences import ReadPreference, secondary_ok_commands
 from pymongo.errors import (AutoReconnect,
-                            CursorNotFound,
-                            InvalidOperation)
+                            InvalidOperation,
+                            OperationFailure)
 
 _QUERY_OPTIONS = {
     "tailable_cursor": 2,
@@ -56,6 +56,15 @@ class _SocketManager:
             self.pool.maybe_return_socket(self.sock)
             self.sock, self.pool = None, None
 
+    def error(self):
+        """Clean up after an error on the managed socket.
+        """
+        if self.sock:
+            self.sock.close()
+
+        # Return the closed socket to avoid a semaphore leak in the pool.
+        self.close()
+
 
 # TODO might be cool to be able to do find().include("foo") or
 # find().exclude(["bar", "baz"]) or find().slice("a", 1, 2) as an
@@ -691,6 +700,12 @@ class Cursor(object):
         `with_limit_and_skip` to ``True`` if that is the desired behavior.
         Raises :class:`~pymongo.errors.OperationFailure` on a database error.
 
+        When used with MongoDB >= 2.6, :meth:`~count` uses any :meth:`~hint`
+        applied to the query. In the following example the hint is passed to
+        the count command:
+
+          collection.find({'field': 'value'}).hint('field_1').count()
+
         With :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`
         or :class:`~pymongo.master_slave_connection.MasterSlaveConnection`,
         if `read_preference` is not
@@ -712,6 +727,9 @@ class Cursor(object):
 
             collection.find({}, network_timeout=1).count()
 
+        .. versionchanged:: 2.8
+           The :meth:`~count` method now supports :meth:`~hint`.
+
         .. versionadded:: 1.1.1
            The `with_limit_and_skip` parameter.
            :meth:`~pymongo.cursor.Cursor.__len__` was deprecated in favor of
@@ -733,6 +751,9 @@ class Cursor(object):
         if self.__comment:
             command['$comment'] = self.__comment
 
+        if self.__hint is not None:
+            command['hint'] = self.__hint
+
         if with_limit_and_skip:
             if self.__limit:
                 command["limit"] = self.__limit
@@ -825,20 +846,26 @@ class Cursor(object):
 
         `index` should be an index as passed to
         :meth:`~pymongo.collection.Collection.create_index`
-        (e.g. ``[('field', ASCENDING)]``). If `index`
-        is ``None`` any existing hints for this query are cleared. The
-        last hint applied to this cursor takes precedence over all
-        others.
+        (e.g. ``[('field', ASCENDING)]``) or the name of the index.
+        If `index` is ``None`` any existing hint for this query is
+        cleared. The last hint applied to this cursor takes precedence
+        over all others.
 
         :Parameters:
           - `index`: index to hint on (as an index specifier)
+
+        .. versionchanged:: 2.8
+           The :meth:`~hint` method accepts the name of the index.
         """
         self.__check_okay_to_chain()
         if index is None:
             self.__hint = None
             return self
 
-        self.__hint = helpers._index_document(index)
+        if isinstance(index, basestring):
+            self.__hint = index
+        else:
+            self.__hint = helpers._index_document(index)
         return self
 
     def comment(self, comment):
@@ -914,8 +941,14 @@ class Cursor(object):
                 # due to a socket timeout.
                 self.__killed = True
                 raise
-        else: # exhaust cursor - no getMore message
-            response = client._exhaust_next(self.__exhaust_mgr.sock)
+        else:
+            # Exhaust cursor - no getMore message.
+            try:
+                response = client._exhaust_next(self.__exhaust_mgr.sock)
+            except AutoReconnect:
+                self.__killed = True
+                self.__exhaust_mgr.error()
+                raise
 
         try:
             response = helpers._unpack_response(response, self.__id,
@@ -923,8 +956,10 @@ class Cursor(object):
                                                 self.__tz_aware,
                                                 self.__uuid_subtype,
                                                 self.__compile_re)
-        except CursorNotFound:
+        except OperationFailure:
             self.__killed = True
+            # Make sure exhaust socket is returned immediately, if necessary.
+            self.__die()
             # If this is a tailable cursor the error is likely
             # due to capped collection roll over. Setting
             # self.__killed to True ensures Cursor.alive will be
@@ -936,9 +971,14 @@ class Cursor(object):
             # Don't send kill cursors to another server after a "not master"
             # error. It's completely pointless.
             self.__killed = True
+            # Make sure exhaust socket is returned immediately, if necessary.
+            self.__die()
             client.disconnect()
             raise
+
         self.__id = response["cursor_id"]
+        if self.__id == 0:
+            self.__killed = True
 
         # starting from doesn't get set on getmore's for tailable cursors
         if not (self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]):
@@ -1012,6 +1052,17 @@ class Cursor(object):
         since they will stop iterating even though they *may* return more
         results in the future.
 
+        With regular cursors, simply use a for loop instead of :attr:`alive`::
+
+            for doc in collection.find():
+                print(doc)
+
+        .. note:: Even if :attr:`alive` is True, :meth:`next` can raise
+          :exc:`StopIteration`. :attr:`alive` can also be True while iterating
+          a cursor from a failed server. In this case :attr:`alive` will
+          return False after :meth:`next` fails to retrieve the next batch
+          of results from the server.
+
         .. versionadded:: 1.5
         """
         return bool(len(self.__data) or (not self.__killed))
@@ -1032,6 +1083,7 @@ class Cursor(object):
         return self
 
     def next(self):
+        """Advance the cursor."""
         if self.__empty:
             raise StopIteration
         db = self.__collection.database

pymongo/database.py

@@ -22,23 +22,14 @@ from bson.dbref import DBRef
 from bson.son import SON
 from pymongo import auth, common, helpers
 from pymongo.collection import Collection
+from pymongo.command_cursor import CommandCursor
 from pymongo.errors import (CollectionInvalid,
                             ConfigurationError,
-                            InvalidName,
                             OperationFailure)
-from pymongo import read_preferences as rp
-
-
-def _check_name(name):
-    """Check if a database name is valid.
-    """
-    if not name:
-        raise InvalidName("database name cannot be the empty string")
-
-    for invalid_char in [" ", ".", "$", "/", "\\", "\x00"]:
-        if invalid_char in name:
-            raise InvalidName("database names cannot contain the "
-                              "character %r" % invalid_char)
+from pymongo.read_preferences import (modes,
+                                      secondary_ok_commands,
+                                      ReadPreference)
+from pymongo.son_manipulator import SONManipulator
 
 
 class Database(common.BaseObject):
@@ -74,7 +65,7 @@ class Database(common.BaseObject):
                             "of %s" % (basestring.__name__,))
 
         if name != '$external':
-            _check_name(name)
+            helpers._check_database_name(name)
 
         self.__name = unicode(name)
         self.__connection = connection
@@ -92,9 +83,10 @@ class Database(common.BaseObject):
         :Parameters:
           - `manipulator`: the manipulator to add
         """
+        base = SONManipulator()
         def method_overwritten(instance, method):
-            return getattr(instance, method) != \
-                getattr(super(instance.__class__, instance), method)
+            return (getattr(
+                instance, method).im_func != getattr(base, method).im_func)
 
         if manipulator.will_copy():
             if method_overwritten(manipulator, "transform_incoming"):
@@ -217,8 +209,8 @@ class Database(common.BaseObject):
         creation. :class:`~pymongo.errors.CollectionInvalid` will be
         raised if the collection already exists.
 
-        Options should be passed as keyword arguments to this
-        method. Any of the following options are valid:
+        Options should be passed as keyword arguments to this method. Supported
+        options vary with MongoDB release. Some examples include:
 
           - "size": desired initial size for the collection (in
             bytes). For capped collections this size is the max
@@ -226,6 +218,9 @@ class Database(common.BaseObject):
           - "capped": if True, this is a capped collection
           - "max": maximum number of objects if capped (optional)
 
+        See the MongoDB documentation for a full list of supported options by
+        server version.
+
         :Parameters:
           - `name`: the name of the collection to create
           - `**kwargs` (optional): additional keyword arguments will
@@ -245,6 +240,16 @@ class Database(common.BaseObject):
 
         return Collection(self, name, **opts)
 
+    def _apply_incoming_manipulators(self, son, collection):
+        for manipulator in self.__incoming_manipulators:
+            son = manipulator.transform_incoming(son, collection)
+        return son
+
+    def _apply_incoming_copying_manipulators(self, son, collection):
+        for manipulator in self.__incoming_copying_manipulators:
+            son = manipulator.transform_incoming(son, collection)
+        return son
+
     def _fix_incoming(self, son, collection):
         """Apply manipulators to an incoming SON object before it gets stored.
 
@@ -252,10 +257,8 @@ class Database(common.BaseObject):
           - `son`: the son object going into the database
           - `collection`: the collection the son object is being saved in
         """
-        for manipulator in self.__incoming_manipulators:
-            son = manipulator.transform_incoming(son, collection)
-        for manipulator in self.__incoming_copying_manipulators:
-            son = manipulator.transform_incoming(son, collection)
+        son = self._apply_incoming_manipulators(son, collection)
+        son = self._apply_incoming_copying_manipulators(son, collection)
         return son
 
     def _fix_outgoing(self, son, collection):
@@ -282,7 +285,7 @@ class Database(common.BaseObject):
 
         command_name = command.keys()[0].lower()
         must_use_master = kwargs.pop('_use_master', False)
-        if command_name not in rp.secondary_ok_commands:
+        if command_name not in secondary_ok_commands:
             must_use_master = True
 
         # Special-case: mapreduce can go to secondaries only if inline
@@ -323,20 +326,21 @@ class Database(common.BaseObject):
         command.update(kwargs)
 
         # Warn if must_use_master will override read_preference.
-        if (extra_opts['read_preference'] != rp.ReadPreference.PRIMARY and
-                extra_opts['_must_use_master']):
+        if (extra_opts['read_preference'] != ReadPreference.PRIMARY and
+                extra_opts['_must_use_master'] and self.connection._rs_client):
             warnings.warn("%s does not support %s read preference "
                           "and will be routed to the primary instead." %
                           (command_name,
-                           rp.modes[extra_opts['read_preference']]),
-                          UserWarning)
+                           modes[extra_opts['read_preference']]),
+                          UserWarning, stacklevel=3)
 
         cursor = self["$cmd"].find(command, **extra_opts).limit(-1)
         for doc in cursor:
             result = doc
 
         if check:
-            msg = "command %s failed: %%s" % repr(command).replace("%", "%%")
+            msg = "command %s on namespace %s failed: %%s" % (
+                repr(command).replace("%", "%%"), self.name + '.$cmd')
             helpers._check_command_response(result, self.connection.disconnect,
                                             msg, allowable_errors)
 
@@ -441,10 +445,30 @@ class Database(common.BaseObject):
           - `include_system_collections` (optional): if ``False`` list
             will not include system collections (e.g ``system.indexes``)
         """
-        results = self["system.namespaces"].find(_must_use_master=True)
-        names = [r["name"] for r in results]
-        names = [n[len(self.__name) + 1:] for n in names
-                 if n.startswith(self.__name + ".") and "$" not in n]
+        client = self.connection
+        client._ensure_connected(True)
+
+        slave_okay = not client._rs_client and not client.is_mongos
+        if client.max_wire_version > 2:
+            res, addr = self._command("listCollections",
+                                      cursor={},
+                                      read_preference=ReadPreference.PRIMARY,
+                                      slave_okay=slave_okay)
+            # MongoDB 2.8rc2
+            if "collections" in res:
+                results = res["collections"]
+            # >= MongoDB 2.8rc3
+            else:
+                results = CommandCursor(self["$cmd"], res["cursor"], addr)
+            names = [result["name"] for result in results]
+        else:
+            names = [result["name"] for result
+                     in self["system.namespaces"].find(
+                         slave_okay=slave_okay,
+                         _must_use_master=True)]
+            names = [n[len(self.__name) + 1:] for n in names
+                     if n.startswith(self.__name + ".") and "$" not in n]
+
         if not include_system_collections:
             names = [n for n in names if not n.startswith("system.")]
         return names
@@ -466,7 +490,8 @@ class Database(common.BaseObject):
 
         self.__connection._purge_index(self.__name, name)
 
-        self.command("drop", unicode(name), allowable_errors=["ns not found"])
+        self.command("drop", unicode(name), allowable_errors=["ns not found"],
+                     read_preference=ReadPreference.PRIMARY)
 
     def validate_collection(self, name_or_collection,
                             scandata=False, full=False):
@@ -504,7 +529,8 @@ class Database(common.BaseObject):
                             "%s or Collection" % (basestring.__name__,))
 
         result = self.command("validate", unicode(name),
-                              scandata=scandata, full=full)
+                              scandata=scandata, full=full,
+                              read_preference=ReadPreference.PRIMARY)
 
         valid = True
         # Pre 1.9 results
@@ -553,7 +579,8 @@ class Database(common.BaseObject):
 
         .. mongodoc:: profiling
         """
-        result = self.command("profile", -1)
+        result = self.command("profile", -1,
+                              read_preference=ReadPreference.PRIMARY)
 
         assert result["was"] >= 0 and result["was"] <= 2
         return result["was"]
@@ -593,9 +620,11 @@ class Database(common.BaseObject):
             raise TypeError("slow_ms must be an integer")
 
         if slow_ms is not None:
-            self.command("profile", level, slowms=slow_ms)
+            self.command("profile", level, slowms=slow_ms,
+                         read_preference=ReadPreference.PRIMARY)
         else:
-            self.command("profile", level)
+            self.command("profile", level,
+                         read_preference=ReadPreference.PRIMARY)
 
     def profiling_info(self):
         """Returns a list containing current profiling information.
@@ -605,12 +634,28 @@ class Database(common.BaseObject):
         return list(self["system.profile"].find())
 
     def error(self):
-        """Get a database error if one occured on the last operation.
+        """**DEPRECATED**: Get the error if one occurred on the last operation.
+
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0.
 
         Return None if the last operation was error-free. Otherwise return the
         error that occurred.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        error = self.command("getlasterror")
+        warnings.warn("Database.error() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        error = self.command("getlasterror",
+                             read_preference=ReadPreference.PRIMARY)
         error_msg = error.get("err", "")
         if error_msg is None:
             return None
@@ -619,31 +664,81 @@ class Database(common.BaseObject):
         return error
 
     def last_status(self):
-        """Get status information from the last operation.
+        """**DEPRECATED**: Get status information from the last operation.
+
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0.
 
         Returns a SON object with status information.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        return self.command("getlasterror")
+        warnings.warn("last_status() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        return self.command("getlasterror",
+                            read_preference=ReadPreference.PRIMARY)
 
     def previous_error(self):
-        """Get the most recent error to have occurred on this database.
+        """**DEPRECATED**: Get the most recent error on this database.
+
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0. Furthermore, the underlying database command
+        ``getpreverror`` will be removed in a future MongoDB release.
 
         Only returns errors that have occurred since the last call to
-        `Database.reset_error_history`. Returns None if no such errors have
+        :meth:`reset_error_history`. Returns None if no such errors have
         occurred.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        error = self.command("getpreverror")
+        warnings.warn("previous_error() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        error = self.command("getpreverror",
+                             read_preference=ReadPreference.PRIMARY)
         if error.get("err", 0) is None:
             return None
         return error
 
     def reset_error_history(self):
-        """Reset the error history of this database.
+        """**DEPRECATED**: Reset the error history of this database.
 
-        Calls to `Database.previous_error` will only return errors that have
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0. Furthermore, the underlying database command
+        ``reseterror`` will be removed in a future MongoDB release.
+
+        Calls to :meth:`previous_error` will only return errors that have
         occurred since the most recent call to this method.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        self.command("reseterror")
+        warnings.warn("reset_error_history() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        self.command("reseterror",
+                     read_preference=ReadPreference.PRIMARY)
 
     def __iter__(self):
         return self
@@ -689,7 +784,7 @@ class Database(common.BaseObject):
             opts["pwd"] = auth._password_digest(name, password)
             opts["digestPassword"] = False
 
-        opts["writeConcern"] = self._get_wc_override()
+        opts["writeConcern"] = self._get_wc_override() or self.write_concern
         opts.update(kwargs)
 
         if create:
@@ -697,7 +792,8 @@ class Database(common.BaseObject):
         else:
             command_name = "updateUser"
 
-        self.command(command_name, name, **opts)
+        self.command(command_name, name,
+                     read_preference=ReadPreference.PRIMARY, **opts)
 
     def _legacy_add_user(self, name, password, read_only, **kwargs):
         """Uses v1 system to add users, i.e. saving to system.users.
@@ -716,6 +812,11 @@ class Database(common.BaseObject):
             # See SERVER-4225 for more information.
             if 'login' in str(exc):
                 pass
+            # First admin user add fails gle from mongos 2.0.x
+            # and 2.2.x.
+            elif (exc.details and
+                    'getlasterror' in exc.details.get('note', '')):
+                pass
             else:
                 raise
 
@@ -763,21 +864,22 @@ class Database(common.BaseObject):
                                          "read_only and roles together")
 
         try:
-            uinfo = self.command("usersInfo", name)
+            uinfo = self.command("usersInfo", name,
+                                 read_preference=ReadPreference.PRIMARY)
+            self._create_or_update_user(
+                (not uinfo["users"]), name, password, read_only, **kwargs)
         except OperationFailure, exc:
             # MongoDB >= 2.5.3 requires the use of commands to manage
-            # users. "No such command" error didn't return an error
-            # code (59) before MongoDB 2.4.7 so we assume that an error
-            # code of None means the userInfo command doesn't exist and
-            # we should fall back to the legacy add user code.
-            if exc.code in (59, None):
+            # users.
+            if exc.code in common.COMMAND_NOT_FOUND_CODES:
                 self._legacy_add_user(name, password, read_only, **kwargs)
-                return
-            raise
-
-        # Create the user if not found in uinfo, otherwise update one.
-        self._create_or_update_user(
-            (not uinfo["users"]), name, password, read_only, **kwargs)
+            # Unauthorized. MongoDB >= 2.7.1 has a narrow localhost exception,
+            # and we must add a user before sending commands.
+            elif exc.code == 13:
+                self._create_or_update_user(
+                    True, name, password, read_only, **kwargs)
+            else:
+                raise
 
     def remove_user(self, name):
         """Remove user `name` from this :class:`Database`.
@@ -792,18 +894,20 @@ class Database(common.BaseObject):
         """
 
         try:
+            write_concern = self._get_wc_override() or self.write_concern
             self.command("dropUser", name,
-                         writeConcern=self._get_wc_override())
+                         read_preference=ReadPreference.PRIMARY,
+                         writeConcern=write_concern)
         except OperationFailure, exc:
             # See comment in add_user try / except above.
-            if exc.code in (59, None):
+            if exc.code in common.COMMAND_NOT_FOUND_CODES:
                 self.system.users.remove({"user": name},
                                          **self._get_wc_override())
                 return
             raise
 
     def authenticate(self, name, password=None,
-                     source=None, mechanism='MONGODB-CR', **kwargs):
+                     source=None, mechanism='DEFAULT', **kwargs):
         """Authenticate to use this database.
 
         Authentication lasts for the life of the underlying client
@@ -839,11 +943,15 @@ class Database(common.BaseObject):
             specified the current database is used.
           - `mechanism` (optional): See
             :data:`~pymongo.auth.MECHANISMS` for options.
-            Defaults to MONGODB-CR (MongoDB Challenge Response protocol)
+            By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
+            MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
           - `gssapiServiceName` (optional): Used with the GSSAPI mechanism
             to specify the service name portion of the service principal name.
             Defaults to 'mongodb'.
 
+        .. versionadded:: 2.8
+           Use SCRAM-SHA-1 with MongoDB 3.0 and later.
+
         .. versionchanged:: 2.5
            Added the `source` and `mechanism` parameters. :meth:`authenticate`
            now raises a subclass of :class:`~pymongo.errors.PyMongoError` if
@@ -869,9 +977,8 @@ class Database(common.BaseObject):
             validated_options[normalized] = val
 
         credentials = auth._build_credentials_tuple(mechanism,
-                                source or self.name, unicode(name),
-                                password and unicode(password) or None,
-                                validated_options)
+                                source or self.name, name,
+                                password, validated_options)
         self.connection._cache_credentials(self.name, credentials)
         return True
 
@@ -886,7 +993,7 @@ class Database(common.BaseObject):
         # Sockets will be deauthenticated as they are used.
         self.connection._purge_credentials(self.name)
 
-    def dereference(self, dbref):
+    def dereference(self, dbref, **kwargs):
         """Dereference a :class:`~bson.dbref.DBRef`, getting the
         document it points to.
 
@@ -898,6 +1005,9 @@ class Database(common.BaseObject):
 
         :Parameters:
           - `dbref`: the reference
+          - `**kwargs` (optional): any additional keyword arguments
+            are the same as the arguments to
+            :meth:`~pymongo.collection.Collection.find`.
         """
         if not isinstance(dbref, DBRef):
             raise TypeError("cannot dereference a %s" % type(dbref))
@@ -905,7 +1015,7 @@ class Database(common.BaseObject):
             raise ValueError("trying to dereference a DBRef that points to "
                              "another database (%r not %r)" % (dbref.database,
                                                                self.__name))
-        return self[dbref.collection].find_one({"_id": dbref.id})
+        return self[dbref.collection].find_one({"_id": dbref.id}, **kwargs)
 
     def eval(self, code, *args):
         """Evaluate a JavaScript expression in MongoDB.
@@ -930,7 +1040,9 @@ class Database(common.BaseObject):
         if not isinstance(code, Code):
             code = Code(code)
 
-        result = self.command("$eval", code, args=args)
+        result = self.command("$eval", code,
+                              read_preference=ReadPreference.PRIMARY,
+                              args=args)
         return result.get("retval", None)
 
     def __call__(self, *args, **kwargs):

pymongo/helpers.py

@@ -16,15 +16,19 @@
 
 import random
 import struct
+import warnings
 
 import bson
 import pymongo
 
 from bson.binary import OLD_UUID_SUBTYPE
 from bson.son import SON
+from pymongo import auth
 from pymongo.errors import (AutoReconnect,
                             CursorNotFound,
                             DuplicateKeyError,
+                            InvalidName,
+                            InvalidOperation,
                             OperationFailure,
                             ExecutionTimeout,
                             WTimeoutError)
@@ -143,8 +147,8 @@ def _check_command_response(response, reset, msg=None, allowable_errors=None):
         # for some errors.
         if "raw" in response:
             for shard in response["raw"].itervalues():
-                if not shard.get("ok"):
-                    # Just grab the first error...
+                # Grab the first non-empty raw error from a shard.
+                if shard.get("errmsg") and not shard.get("ok"):
                     details = shard
                     break
 
@@ -223,6 +227,121 @@ def _fields_list_to_dict(fields):
     return as_dict
 
 
+def _check_database_name(name):
+    """Check if a database name is valid."""
+    if not name:
+        raise InvalidName("database name cannot be the empty string")
+
+    for invalid_char in [" ", ".", "$", "/", "\\", "\x00"]:
+        if invalid_char in name:
+            raise InvalidName("database names cannot contain the "
+                              "character %r" % invalid_char)
+
+
+def _copy_database(
+        fromdb,
+        todb,
+        fromhost,
+        mechanism,
+        username,
+        password,
+        sock_info,
+        cmd_func):
+    """Copy a database, perhaps from a remote host.
+
+    :Parameters:
+      - `fromdb`: Source database.
+      - `todb`: Target database.
+      - `fromhost`: Source host like 'foo.com', 'foo.com:27017', or None.
+      - `mechanism`: An authentication mechanism.
+      - `username`: A str or unicode, or None.
+      - `password`: A str or unicode, or None.
+      - `sock_info`: A SocketInfo instance.
+      - `cmd_func`: A callback taking args sock_info, database, command doc.
+    """
+    if not isinstance(fromdb, basestring):
+        raise TypeError('from_name must be an instance '
+                        'of %s' % (basestring.__name__,))
+    if not isinstance(todb, basestring):
+        raise TypeError('to_name must be an instance '
+                        'of %s' % (basestring.__name__,))
+
+    _check_database_name(todb)
+
+    warnings.warn("copy_database is deprecated. Use the raw 'copydb' command"
+                  " or db.copyDatabase() in the mongo shell. See"
+                  " doc/examples/copydb.",
+                  DeprecationWarning, stacklevel=2)
+
+    # It would be better if the user told us what mechanism to use, but for
+    # backwards compatibility with earlier PyMongos we don't require the
+    # mechanism. Hope 'fromhost' runs the same version as the target.
+    if mechanism == 'DEFAULT':
+        if sock_info.max_wire_version >= 3:
+            mechanism = 'SCRAM-SHA-1'
+        else:
+            mechanism = 'MONGODB-CR'
+
+    if username is not None:
+        if mechanism == 'SCRAM-SHA-1':
+            credentials = auth._build_credentials_tuple(mech=mechanism,
+                                                        source='admin',
+                                                        user=username,
+                                                        passwd=password,
+                                                        extra=None)
+
+            try:
+                auth._copydb_scram_sha1(credentials=credentials,
+                                        sock_info=sock_info,
+                                        cmd_func=cmd_func,
+                                        fromdb=fromdb,
+                                        todb=todb,
+                                        fromhost=fromhost)
+            except OperationFailure, exc:
+                errmsg = exc.details and exc.details.get('errmsg') or ''
+                if 'no such cmd: saslStart' in errmsg:
+                    explanation = (
+                        "%s doesn't support SCRAM-SHA-1, pass"
+                        " mechanism='MONGODB-CR' to copy_database" % fromhost)
+
+                    raise OperationFailure(explanation,
+                                           exc.code,
+                                           exc.details)
+                else:
+                    raise
+
+        elif mechanism == 'MONGODB-CR':
+            get_nonce_cmd = SON([('copydbgetnonce', 1),
+                                 ('fromhost', fromhost)])
+
+            get_nonce_response, _ = cmd_func(sock_info, 'admin', get_nonce_cmd)
+            nonce = get_nonce_response['nonce']
+            copydb_cmd = SON([('copydb', 1),
+                              ('fromdb', fromdb),
+                              ('todb', todb)])
+
+            copydb_cmd['username'] = username
+            copydb_cmd['nonce'] = nonce
+            copydb_cmd['key'] = auth._auth_key(nonce, username, password)
+            if fromhost is not None:
+                copydb_cmd['fromhost'] = fromhost
+
+            cmd_func(sock_info, 'admin', copydb_cmd)
+        else:
+            raise InvalidOperation('Authentication mechanism %r not supported'
+                                   ' for copy_database' % mechanism)
+    else:
+        # No username.
+        copydb_cmd = SON([('copydb', 1),
+                          ('fromdb', fromdb),
+                          ('todb', todb)])
+
+        if fromhost:
+            copydb_cmd['fromhost'] = fromhost
+
+        cmd_func(sock_info, 'admin', copydb_cmd)
+
+
 def shuffled(sequence):
     """Returns a copy of the sequence (as a :class:`list`) which has been
     shuffled by :func:`random.shuffle`.

pymongo/master_slave_connection.py

@@ -12,13 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Master-Slave connection to Mongo.
+"""**DEPRECATED**: Master-Slave connection to Mongo.
 
 Performs all writes to Master instance and distributes reads among all
 slaves. Reads are tried on each slave in turn until the read succeeds
 or all slaves failed.
+
+MasterSlaveConnection is deprecated and will be removed in PyMongo 3.0.
+Deploy your MongoDB servers as a replica set instead of a master-slave set,
+and use a :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`.
+If you cannot replace your master-slave set with a replica set, connect
+directly to the master and each slave with instances of
+:class:`~pymongo.mongo_client.MongoClient`.
+
+.. seealso:: `replica set documentation <http://dochub.mongodb.org/core/rs>`_.
+
+.. versionchanged:: 2.8
+   Deprecated.
 """
 
+import warnings
+
 from pymongo import helpers, thread_util
 from pymongo import ReadPreference
 from pymongo.common import BaseObject
@@ -28,8 +42,8 @@ from pymongo.errors import AutoReconnect
 
 
 class MasterSlaveConnection(BaseObject):
-    """A master-slave connection to Mongo.
-    """
+
+    _rs_client = True
 
     def __init__(self, master, slaves=[], document_class=dict, tz_aware=False):
         """Create a new Master-Slave connection.
@@ -67,6 +81,10 @@ class MasterSlaveConnection(BaseObject):
                 raise TypeError("slave %r is not an instance of MongoClient" %
                                 slave)
 
+        warnings.warn("MasterSlaveConnection is deprecated, and will be"
+                      " removed in PyMongo 3.0.",
+                      DeprecationWarning, stacklevel=2)
+
         super(MasterSlaveConnection,
               self).__init__(read_preference=ReadPreference.SECONDARY,
                              safe=master.safe,
@@ -179,6 +197,14 @@ class MasterSlaveConnection(BaseObject):
         for slave in self.__slaves:
             slave.disconnect()
 
+    def close(self):
+        """Alias for :meth:`disconnect`
+
+        .. seealso:: :meth:`end_request`
+        .. versionadded:: 2.8
+        """
+        self.disconnect()
+
     def set_cursor_manager(self, manager_class):
         """Set the cursor manager for this connection.
 
@@ -194,12 +220,9 @@ class MasterSlaveConnection(BaseObject):
         """
         self.__master._ensure_connected(sync)
 
-    # _connection_to_use is a hack that we need to include to make sure
-    # that killcursor operations can be sent to the same instance on which
-    # the cursor actually resides...
     def _send_message(self, message,
                       with_last_error=False,
-                      command=False, _connection_to_use=None):
+                      command=False):
         """Say something to Mongo.
 
         Sends a message on the Master connection. This is used for inserts,
@@ -213,11 +236,8 @@ class MasterSlaveConnection(BaseObject):
           - `data`: data to send
           - `safe`: perform a getLastError after sending the message
         """
-        if _connection_to_use is None or _connection_to_use == -1:
-            return self.__master._send_message(message,
-                                               with_last_error, command)
-        return self.__slaves[_connection_to_use]._send_message(
-            message, with_last_error, command, check_primary=False)
+        return self.__master._send_message(message,
+                                           with_last_error, command)
 
     # _connection_to_use is a hack that we need to include to make sure
     # that getmore operations can be sent to the same instance on which
@@ -366,3 +386,26 @@ class MasterSlaveConnection(BaseObject):
         return self.__master._purge_index(database_name,
                                           collection_name,
                                           index_name)
+
+    def server_info(self):
+        """Get information about the MongoDB master we're connected to.
+
+        .. versionadded:: 2.8
+        """
+        return self.__master.admin.command("buildinfo")
+
+    def _cache_credentials(self, source, credentials, connect=True):
+        self.__master._cache_credentials(source, credentials, connect)
+        for slave in self.__slaves:
+            # Use connect=False here so that credentials are cached
+            # on the slaves no matter what. Since auth succeeded on the
+            # master we know the credentials are correct and the slaves
+            # will authenticate as needed. This avoids issues with slave
+            # auth problems due to problems other than bad credentials
+            # (e.g. network errors).
+            slave._cache_credentials(source, credentials, connect=False)
+
+    def _purge_credentials(self, source):
+        self.__master._purge_credentials(source)
+        for slave in self.__slaves:
+            slave._purge_credentials(source)

pymongo/member.py

@@ -145,6 +145,31 @@ class Member(object):
 
         return False
 
+    def get_socket(self, force=False):
+        sock_info = self.pool.get_socket(force)
+        sock_info.set_wire_version_range(self.min_wire_version,
+                                         self.max_wire_version)
+
+        return sock_info
+
+    def maybe_return_socket(self, sock_info):
+        self.pool.maybe_return_socket(sock_info)
+
+    def discard_socket(self, sock_info):
+        self.pool.discard_socket(sock_info)
+
+    def start_request(self):
+        self.pool.start_request()
+
+    def in_request(self):
+        return self.pool.in_request()
+
+    def end_request(self):
+        self.pool.end_request()
+
+    def reset(self):
+        self.pool.reset()
+
     def __str__(self):
         return '<Member "%s:%s" primary=%r>' % (
             self.host[0], self.host[1], self.is_primary)

pymongo/mongo_client.py

@@ -61,6 +61,9 @@ from pymongo.errors import (AutoReconnect,
                             InvalidURI,
                             OperationFailure)
 from pymongo.member import Member
+from pymongo.read_preferences import ReadPreference
+
+
 EMPTY = b("")
 
 
@@ -84,6 +87,7 @@ class MongoClient(common.BaseObject):
 
     HOST = "localhost"
     PORT = 27017
+    _rs_client = False
 
     def __init__(self, host=None, port=None, max_pool_size=100,
                  document_class=dict, tz_aware=False, _connect=True,
@@ -134,27 +138,28 @@ class MongoClient(common.BaseObject):
 
           | **Other optional parameters can be passed as keyword arguments:**
 
-          - `socketTimeoutMS`: (integer) How long (in milliseconds) a send or
-            receive on a socket can take before timing out. Defaults to ``None``
-            (no timeout).
-          - `connectTimeoutMS`: (integer) How long (in milliseconds) a
+          - `socketTimeoutMS`: (integer or None) How long (in milliseconds) a
+            send or receive on a socket can take before timing out. Defaults to
+            ``None`` (no timeout).
+          - `connectTimeoutMS`: (integer or None) How long (in milliseconds) a
             connection can take to be opened before timing out. Defaults to
             ``20000``.
-          - `waitQueueTimeoutMS`: (integer) How long (in milliseconds) a
-            thread will wait for a socket from the pool if the pool has no
+          - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds)
+            a thread will wait for a socket from the pool if the pool has no
             free sockets. Defaults to ``None`` (no timeout).
-          - `waitQueueMultiple`: (integer) Multiplied by max_pool_size to give
-            the number of threads allowed to wait for a socket at one time.
-            Defaults to ``None`` (no waiters).
-          - `auto_start_request`: If ``True``, each thread that accesses
-            this :class:`MongoClient` has a socket allocated to it for the
-            thread's lifetime.  This ensures consistent reads, even if you
-            read after an unacknowledged write. Defaults to ``False``
+          - `waitQueueMultiple`: (integer or None) Multiplied by max_pool_size
+            to give the number of threads allowed to wait for a socket at one
+            time. Defaults to ``None`` (no waiters).
+          - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
+          - `auto_start_request`: Deprecated.
           - `use_greenlets`: If ``True``, :meth:`start_request()` will ensure
             that the current greenlet uses the same socket for all
-            operations until :meth:`end_request()`
+            operations until :meth:`end_request()`. Defaults to ``False``.
 
           | **Write Concern options:**
+          | (Only set if passed. No default values.)
 
           - `w`: (integer or string) If this is a replica set, write operations
             will block until they have been replicated to the specified number
@@ -182,31 +187,35 @@ class MongoClient(common.BaseObject):
             - either directly or via a mongos:**
           | (ignored by standalone mongod instances)
 
-          - `replicaSet`: (string) The name of the replica set to connect to.
-            The driver will verify that the replica set it connects to matches
-            this name. Implies that the hosts specified are a seed list and the
-            driver should attempt to find all members of the set. *Ignored by
-            mongos*.
+          - `replicaSet`: (string or None) The name of the replica set to
+            connect to. The driver will verify that the replica set it connects
+            to matches this name. Implies that the hosts specified are a seed
+            list and the driver should attempt to find all members of the set.
+            *Ignored by mongos*. Defaults to ``None``.
           - `read_preference`: The read preference for this client. If
             connecting to a secondary then a read preference mode *other* than
             PRIMARY is required - otherwise all queries will throw
             :class:`~pymongo.errors.AutoReconnect` "not master".
             See :class:`~pymongo.read_preferences.ReadPreference` for all
-            available read preference options.
+            available read preference options. Defaults to ``PRIMARY``.
           - `tag_sets`: Ignored unless connecting to a replica set via mongos.
             Specify a priority-order for tag sets, provide a list of
             tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
             set, ``{}``, means "read from any member that matches the mode,
-            ignoring tags.
+            ignoring tags. Defaults to ``[{}]``, meaning "ignore members'
+            tags."
 
           | **SSL configuration:**
 
           - `ssl`: If ``True``, create the connection to the server using SSL.
+            Defaults to ``False``.
           - `ssl_keyfile`: The private keyfile used to identify the local
             connection against mongod.  If included with the ``certfile`` then
             only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
+            Defaults to ``None``.
           - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``ssl=True``.
+            connection against mongod. Implies ``ssl=True``. Defaults to
+            ``None``.
           - `ssl_cert_reqs`: Specifies whether a certificate is required from
             the other side of the connection, and whether it will be validated
             if provided. It must be one of the three values ``ssl.CERT_NONE``
@@ -214,11 +223,12 @@ class MongoClient(common.BaseObject):
             (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
             (required and validated). If the value of this parameter is not
             ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
-            to a file of CA certificates. Implies ``ssl=True``.
+            to a file of CA certificates. Implies ``ssl=True``. Defaults to
+            ``ssl.CERT_NONE``.
           - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``None``.
 
         .. seealso:: :meth:`end_request`
 
@@ -291,9 +301,10 @@ class MongoClient(common.BaseObject):
         self.__direct = len(seeds) == 1 and not self.__repl
 
         self.__net_timeout = options.get('sockettimeoutms')
-        self.__conn_timeout = options.get('connecttimeoutms')
+        self.__conn_timeout = options.get('connecttimeoutms', 20.0)
         self.__wait_queue_timeout = options.get('waitqueuetimeoutms')
         self.__wait_queue_multiple = options.get('waitqueuemultiple')
+        self.__socket_keepalive = options.get('socketkeepalive', False)
 
         self.__use_ssl = options.get('ssl', None)
         self.__ssl_keyfile = options.get('ssl_keyfile', None)
@@ -301,7 +312,8 @@ class MongoClient(common.BaseObject):
         self.__ssl_cert_reqs = options.get('ssl_cert_reqs', None)
         self.__ssl_ca_certs = options.get('ssl_ca_certs', None)
 
-        ssl_kwarg_keys = [k for k in kwargs.keys() if k.startswith('ssl_')]
+        ssl_kwarg_keys = [k for k in kwargs.keys()
+                          if k.startswith('ssl_') and kwargs[k]]
         if self.__use_ssl == False and ssl_kwarg_keys:
             raise ConfigurationError("ssl has not been enabled but the "
                                      "following ssl parameters have been set: "
@@ -366,7 +378,7 @@ class MongoClient(common.BaseObject):
                 raise ConnectionFailure(str(e))
 
         if username:
-            mechanism = options.get('authmechanism', 'MONGODB-CR')
+            mechanism = options.get('authmechanism', 'DEFAULT')
             source = (
                 options.get('authsource')
                 or self.__default_database_name
@@ -374,8 +386,8 @@ class MongoClient(common.BaseObject):
 
             credentials = auth._build_credentials_tuple(mechanism,
                                                         source,
-                                                        unicode(username),
-                                                        unicode(password),
+                                                        username,
+                                                        password,
                                                         options)
             try:
                 self._cache_credentials(source, credentials, _connect)
@@ -456,7 +468,7 @@ class MongoClient(common.BaseObject):
                 auth.authenticate(credentials, sock_info, self.__simple_command)
                 sock_info.authset.add(credentials)
             finally:
-                member.pool.maybe_return_socket(sock_info)
+                member.maybe_return_socket(sock_info)
 
         self.__auth_credentials[source] = credentials
 
@@ -479,7 +491,8 @@ class MongoClient(common.BaseObject):
             ssl_cert_reqs=self.__ssl_cert_reqs,
             ssl_ca_certs=self.__ssl_ca_certs,
             wait_queue_timeout=self.__wait_queue_timeout,
-            wait_queue_multiple=self.__wait_queue_multiple)
+            wait_queue_multiple=self.__wait_queue_multiple,
+            socket_keepalive=self.__socket_keepalive)
 
     def __check_auth(self, sock_info):
         """Authenticate using cached database credentials.
@@ -589,8 +602,7 @@ class MongoClient(common.BaseObject):
 
     @property
     def auto_start_request(self):
-        """Is auto_start_request enabled?
-        """
+        """**DEPRECATED** Is auto_start_request enabled?"""
         return self.__auto_start_request
 
     def get_document_class(self):
@@ -670,20 +682,25 @@ class MongoClient(common.BaseObject):
             'max_write_batch_size', common.MAX_WRITE_BATCH_SIZE)
 
     def __simple_command(self, sock_info, dbname, spec):
-        """Send a command to the server.
+        """Send a command to the server. May raise AutoReconnect.
         """
-        rqst_id, msg, _ = message.query(0, dbname + '.$cmd', 0, -1, spec)
+        ns = dbname + '.$cmd'
+        rqst_id, msg, _ = message.query(0, ns, 0, -1, spec)
         start = time.time()
         try:
             sock_info.sock.sendall(msg)
             response = self.__receive_message_on_socket(1, rqst_id, sock_info)
+        except socket.error, e:
+            sock_info.close()
+            raise AutoReconnect(e)
         except:
             sock_info.close()
             raise
 
         end = time.time()
         response = helpers._unpack_response(response)['data'][0]
-        msg = "command %r failed: %%s" % spec
+        msg = "command %s on namespace %s failed: %%s" % (
+            repr(spec).replace("%", "%%"), ns)
         helpers._check_command_response(response, None, msg)
         return response, end - start
 
@@ -754,7 +771,7 @@ class MongoClient(common.BaseObject):
 
         near_candidates = [
             member for member in candidates
-            if member.ping_time - fastest < latency / 1000.0]
+            if member.ping_time - fastest <= latency / 1000.0]
 
         return random.choice(near_candidates)
 
@@ -894,12 +911,11 @@ class MongoClient(common.BaseObject):
 
         Calls disconnect() on error.
         """
-        connection_pool = member.pool
         try:
-            if self.auto_start_request and not connection_pool.in_request():
-                connection_pool.start_request()
+            if self.auto_start_request and not member.in_request():
+                member.start_request()
 
-            sock_info = connection_pool.get_socket()
+            sock_info = member.get_socket()
         except socket.error, why:
             self.disconnect()
 
@@ -913,8 +929,8 @@ class MongoClient(common.BaseObject):
                                 "%s %s" % (host_details, str(why)))
         try:
             self.__check_auth(sock_info)
-        except OperationFailure:
-            connection_pool.maybe_return_socket(sock_info)
+        except:
+            member.maybe_return_socket(sock_info)
             raise
         return sock_info
 
@@ -941,7 +957,7 @@ class MongoClient(common.BaseObject):
 
         # Close sockets promptly.
         if member:
-            member.pool.reset()
+            member.reset()
 
     def close(self):
         """Alias for :meth:`disconnect`
@@ -986,12 +1002,12 @@ class MongoClient(common.BaseObject):
             sock_info = None
             try:
                 try:
-                    sock_info = member.pool.get_socket()
+                    sock_info = member.get_socket()
                     return not pool._closed(sock_info.sock)
                 except (socket.error, ConnectionFailure):
                     return False
             finally:
-                member.pool.maybe_return_socket(sock_info)
+                member.maybe_return_socket(sock_info)
 
     def set_cursor_manager(self, manager_class):
         """Set this client's cursor manager.
@@ -1050,7 +1066,7 @@ class MongoClient(common.BaseObject):
         # for some errors.
         if "errObjects" in result:
             for errobj in result["errObjects"]:
-                if errobj["err"] == error_msg:
+                if errobj.get("err") == error_msg:
                     details = errobj
                     break
 
@@ -1081,7 +1097,7 @@ class MongoClient(common.BaseObject):
             return message
 
     def _send_message(self, message,
-                      with_last_error=False, command=False, check_primary=True):
+                      with_last_error=False, command=False):
         """Say something to Mongo.
 
         Raises ConnectionFailure if the message cannot be sent. Raises
@@ -1094,11 +1110,9 @@ class MongoClient(common.BaseObject):
           - `message`: message to send
           - `with_last_error`: check getLastError status after sending the
             message
-          - `check_primary`: don't try to write to a non-primary; see
-            kill_cursors for an exception to this rule
         """
         member = self.__ensure_member()
-        if check_primary and not with_last_error and not self.is_primary:
+        if not with_last_error and not self.is_primary:
             # The write won't succeed, bail as if we'd done a getLastError
             raise AutoReconnect("not master")
 
@@ -1127,7 +1141,7 @@ class MongoClient(common.BaseObject):
                 sock_info.close()
                 raise
         finally:
-            member.pool.maybe_return_socket(sock_info)
+            member.maybe_return_socket(sock_info)
 
     def __receive_data_on_socket(self, length, sock_info):
         """Lowest level receive operation.
@@ -1186,49 +1200,48 @@ class MongoClient(common.BaseObject):
         sock_info = self.__socket(member)
         exhaust = kwargs.get('exhaust')
         try:
-            try:
-                if not exhaust and "network_timeout" in kwargs:
-                    sock_info.sock.settimeout(kwargs["network_timeout"])
-                response = self.__send_and_receive(message, sock_info)
+            if not exhaust and "network_timeout" in kwargs:
+                sock_info.sock.settimeout(kwargs["network_timeout"])
 
-                if not exhaust:
-                    if "network_timeout" in kwargs:
-                        sock_info.sock.settimeout(self.__net_timeout)
+            response = self.__send_and_receive(message, sock_info)
 
-                return (None, (response, sock_info, member.pool))
-            except (ConnectionFailure, socket.error), e:
-                self.disconnect()
-                raise AutoReconnect(str(e))
-        finally:
             if not exhaust:
-                member.pool.maybe_return_socket(sock_info)
+                if "network_timeout" in kwargs:
+                    sock_info.sock.settimeout(self.__net_timeout)
+
+                member.maybe_return_socket(sock_info)
+
+            return (None, (response, sock_info, member.pool))
+        except (ConnectionFailure, socket.error), e:
+            self.disconnect()
+            member.maybe_return_socket(sock_info)
+            raise AutoReconnect(str(e))
+        except:
+            member.maybe_return_socket(sock_info)
+            raise
 
     def _exhaust_next(self, sock_info):
         """Used with exhaust cursors to get the next batch off the socket.
+
+        Can raise AutoReconnect.
         """
-        return self.__receive_message_on_socket(1, None, sock_info)
+        try:
+            return self.__receive_message_on_socket(1, None, sock_info)
+        except socket.error, e:
+            raise AutoReconnect(str(e))
 
     def start_request(self):
-        """Ensure the current thread or greenlet always uses the same socket
-        until it calls :meth:`end_request`. This ensures consistent reads,
-        even if you read after an unacknowledged write.
+        """**DEPRECATED**: start_request will be removed in PyMongo 3.0.
 
-        In Python 2.6 and above, or in Python 2.5 with
-        "from __future__ import with_statement", :meth:`start_request` can be
-        used as a context manager:
+        When doing w=0 writes to MongoDB 2.4 or earlier, :meth:`start_request`
+        was sometimes useful to ensure the current thread always used the same
+        socket until it called :meth:`end_request`. This made consistent reads
+        more likely after an unacknowledged write. Requests are no longer
+        useful in modern MongoDB applications, see
+        `PYTHON-785 <https://jira.mongodb.org/browse/PYTHON-785>`_.
 
-        >>> client = pymongo.MongoClient(auto_start_request=False)
-        >>> db = client.test
-        >>> _id = db.test_collection.insert({})
-        >>> with client.start_request():
-        ...     for i in range(100):
-        ...         db.test_collection.update({'_id': _id}, {'$set': {'i':i}})
-        ...
-        ...     # Definitely read the document after the final update completes
-        ...     print db.test_collection.find({'_id': _id})
-
-        If a thread or greenlet calls start_request multiple times, an equal
-        number of calls to :meth:`end_request` is required to end the request.
+        .. versionchanged:: 2.8
+           Deprecated.
 
         .. versionchanged:: 2.4
            Now counts the number of calls to start_request and doesn't end
@@ -1239,21 +1252,21 @@ class MongoClient(common.BaseObject):
            :meth:`start_request` previously returned None
         """
         member = self.__ensure_member()
-        member.pool.start_request()
+        member.start_request()
         return pool.Request(self)
 
     def in_request(self):
-        """True if this thread is in a request, meaning it has a socket
-        reserved for its exclusive use.
+        """**DEPRECATED**: True if this thread is in a request, meaning it has
+        a socket reserved for its exclusive use.
         """
         member = self.__member  # Don't try to connect if disconnected.
-        return member and member.pool.in_request()
+        return member and member.in_request()
 
     def end_request(self):
-        """Undo :meth:`start_request`. If :meth:`end_request` is called as many
-        times as :meth:`start_request`, the request is over and this thread's
-        connection returns to the pool. Extra calls to :meth:`end_request` have
-        no effect.
+        """**DEPRECATED**: Undo :meth:`start_request`. If :meth:`end_request`
+        is called as many times as :meth:`start_request`, the request is over
+        and this thread's connection returns to the pool. Extra calls to
+        :meth:`end_request` have no effect.
 
         Ending a request allows the :class:`~socket.socket` that has
         been reserved for this thread by :meth:`start_request` to be returned to
@@ -1267,7 +1280,7 @@ class MongoClient(common.BaseObject):
         """
         member = self.__member  # Don't try to connect if disconnected.
         if member:
-            member.pool.end_request()
+            member.end_request()
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
@@ -1331,19 +1344,43 @@ class MongoClient(common.BaseObject):
         """
         if not isinstance(cursor_ids, list):
             raise TypeError("cursor_ids must be a list")
-        return self._send_message(
-            message.kill_cursors(cursor_ids), check_primary=False)
+
+        member = self.__member
+
+        # We can't risk taking the lock to reconnect if we're being called
+        # from Cursor.__del__, see PYTHON-799.
+        if not member:
+            warnings.warn("not connected, couldn't close cursor")
+            return
+
+        _, kill_cursors_msg = message.kill_cursors(cursor_ids)
+        sock_info = self.__socket(member)
+        try:
+            try:
+                sock_info.sock.sendall(kill_cursors_msg)
+            except:
+                sock_info.close()
+                raise
+        finally:
+            member.maybe_return_socket(sock_info)
 
     def server_info(self):
         """Get information about the MongoDB server we're connected to.
         """
-        return self.admin.command("buildinfo")
+        return self.admin.command("buildinfo",
+                                  read_preference=ReadPreference.PRIMARY)
 
     def database_names(self):
         """Get a list of the names of all databases on the connected server.
         """
+        # SERVER-15994 changed listDatabases to require slaveOk when run
+        # against a secondary / slave. Passing slave_okay=True makes things
+        # consistent across server versions.
         return [db["name"] for db in
-                self.admin.command("listDatabases")["databases"]]
+                self.admin.command(
+                    "listDatabases",
+                    read_preference=ReadPreference.PRIMARY,
+                    slave_okay=not self.is_mongos)["databases"]]
 
     def drop_database(self, name_or_database):
         """Drop a database.
@@ -1365,11 +1402,16 @@ class MongoClient(common.BaseObject):
                             "%s or Database" % (basestring.__name__,))
 
         self._purge_index(name)
-        self[name].command("dropDatabase")
+        self[name].command("dropDatabase",
+                           read_preference=ReadPreference.PRIMARY)
 
     def copy_database(self, from_name, to_name,
-                      from_host=None, username=None, password=None):
-        """Copy a database, potentially from another host.
+                      from_host=None, username=None, password=None,
+                      mechanism='DEFAULT'):
+        """**DEPRECATED**: Copy a database, potentially from another host.
+
+        :meth:`copy_database` will be removed in PyMongo 3.0. See the
+        :doc:`copy_database examples </examples/copydb>` for alternatives.
 
         Raises :class:`TypeError` if `from_name` or `to_name` is not
         an instance of :class:`basestring` (:class:`str` in python 3).
@@ -1380,7 +1422,13 @@ class MongoClient(common.BaseObject):
         source. Otherwise the database is copied from `from_host`.
 
         If the source database requires authentication, `username` and
-        `password` must be specified.
+        `password` must be specified. By default, use SCRAM-SHA-1 with
+        MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response
+        protocol) for older servers.
+
+        .. note:: mongos does not support copying a database from a server
+           with authentication, see
+           `SERVER-6427 <https://jira.mongodb.org/browse/SERVER-6427>`_.
 
         :Parameters:
           - `from_name`: the name of the source database
@@ -1388,39 +1436,25 @@ class MongoClient(common.BaseObject):
           - `from_host` (optional): host name to copy from
           - `username` (optional): username for source database
           - `password` (optional): password for source database
+          - `mechanism` (optional): auth method, 'MONGODB-CR' or 'SCRAM-SHA-1'
 
-        .. note:: Specifying `username` and `password` requires server
-           version **>= 1.3.3+**.
-
-        .. versionadded:: 1.5
+        .. versionchanged:: 2.8
+           Deprecated copy_database, and added SCRAM-SHA-1 support.
         """
-        if not isinstance(from_name, basestring):
-            raise TypeError("from_name must be an instance "
-                            "of %s" % (basestring.__name__,))
-        if not isinstance(to_name, basestring):
-            raise TypeError("to_name must be an instance "
-                            "of %s" % (basestring.__name__,))
-
-        database._check_name(to_name)
-
-        command = {"fromdb": from_name, "todb": to_name}
-
-        if from_host is not None:
-            command["fromhost"] = from_host
-
+        member = self.__ensure_member()
+        sock_info = self.__socket(member)
         try:
-            self.start_request()
-
-            if username is not None:
-                nonce = self.admin.command("copydbgetnonce",
-                                           fromhost=from_host)["nonce"]
-                command["username"] = username
-                command["nonce"] = nonce
-                command["key"] = auth._auth_key(nonce, username, password)
-
-            return self.admin.command("copydb", **command)
+            helpers._copy_database(
+                fromdb=from_name,
+                todb=to_name,
+                fromhost=from_host,
+                mechanism=mechanism,
+                username=username,
+                password=password,
+                sock_info=sock_info,
+                cmd_func=self.__simple_command)
         finally:
-            self.end_request()
+            member.pool.maybe_return_socket(sock_info)
 
     def get_default_database(self):
         """Get the database named in the MongoDB connection URI.
@@ -1467,7 +1501,8 @@ class MongoClient(common.BaseObject):
 
         .. versionadded:: 2.0
         """
-        self.admin.command("fsync", **kwargs)
+        self.admin.command("fsync",
+                           read_preference=ReadPreference.PRIMARY, **kwargs)
 
     def unlock(self):
         """Unlock a previously locked server.

pymongo/mongo_replica_set_client.py

@@ -59,6 +59,7 @@ from pymongo.errors import (AutoReconnect,
                             DuplicateKeyError,
                             OperationFailure,
                             InvalidOperation)
+from pymongo.read_preferences import ReadPreference
 from pymongo.thread_util import DummyLock
 
 EMPTY = b("")
@@ -86,7 +87,7 @@ def shutdown_monitors():
         monitor = ref()
         if monitor:
             monitor.shutdown()
-            monitor.join()
+            monitor.join(10)
 atexit.register(shutdown_monitors)
 
 def _partition_node(node):
@@ -355,7 +356,7 @@ class Monitor(object):
                     self.rsc.refresh()
                 finally:
                     self.refreshed.set()
-            except AutoReconnect:
+            except (AutoReconnect, OperationFailure):
                 pass
 
             # RSC has been collected or there
@@ -430,6 +431,7 @@ class MongoReplicaSetClient(common.BaseObject):
 
     # For tests.
     _refresh_timeout_sec = 5
+    _rs_client = True
 
     def __init__(self, hosts_or_uri=None, max_pool_size=100,
                  document_class=dict, tz_aware=False, _connect=True, **kwargs):
@@ -487,33 +489,31 @@ class MongoReplicaSetClient(common.BaseObject):
             precedence.
           - `port`: For compatibility with :class:`~mongo_client.MongoClient`.
             The default port number to use for hosts.
-          - `socketTimeoutMS`: (integer) How long (in milliseconds) a send or
-            receive on a socket can take before timing out. Defaults to ``None``
-            (no timeout).
-          - `connectTimeoutMS`: (integer) How long (in milliseconds) a
+          - `socketTimeoutMS`: (integer or None) How long (in milliseconds) a
+            send or receive on a socket can take before timing out. Defaults to
+            ``None`` (no timeout).
+          - `connectTimeoutMS`: (integer or None) How long (in milliseconds) a
             connection can take to be opened before timing out. Defaults to
             ``20000``.
-          - `waitQueueTimeoutMS`: (integer) How long (in milliseconds) a
-            thread will wait for a socket from the pool if the pool has no
+          - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds)
+            a thread will wait for a socket from the pool if the pool has no
             free sockets. Defaults to ``None`` (no timeout).
-          - `waitQueueMultiple`: (integer) Multiplied by max_pool_size to give
-            the number of threads allowed to wait for a socket at one time.
-            Defaults to ``None`` (no waiters).
-          - `auto_start_request`: If ``True``, each thread that accesses
-            this :class:`MongoReplicaSetClient` has a socket allocated to it
-            for the thread's lifetime, for each member of the set. For
-            :class:`~pymongo.read_preferences.ReadPreference` PRIMARY,
-            auto_start_request=True ensures consistent reads, even if you read
-            after an unacknowledged write. For read preferences other than
-            PRIMARY, there are no consistency guarantees. Default to ``False``.
+          - `waitQueueMultiple`: (integer or None) Multiplied by max_pool_size
+            to give the number of threads allowed to wait for a socket at one
+            time. Defaults to ``None`` (no waiters).
+          - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
+          - `auto_start_request`: Deprecated.
           - `use_greenlets`: If ``True``, use a background Greenlet instead of
-            a background thread to monitor state of replica set. Additionally,
-            :meth:`start_request()` assigns a greenlet-local, rather than
-            thread-local, socket.
+            a background thread to monitor the state of the replica set.
+            Additionally, :meth:`start_request` assigns a greenlet-local,
+            rather than thread-local, socket. Defaults to ``False``.
             `use_greenlets` with :class:`MongoReplicaSetClient` requires
             `Gevent <http://gevent.org/>`_ to be installed.
 
           | **Write Concern options:**
+          | (Only set if passed. No default values.)
 
           - `w`: (integer or string) Write operations will block until they have
             been replicated to the specified number or tagged set of servers.
@@ -541,14 +541,14 @@ class MongoReplicaSetClient(common.BaseObject):
 
           - `read_preference`: The read preference for this client.
             See :class:`~pymongo.read_preferences.ReadPreference` for available
-            options.
+            options. Defaults to ``PRIMARY``.
           - `tag_sets`: Read from replica-set members with these tags.
             To specify a priority-order for tag sets, provide a list of
             tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
             set, ``{}``, means "read from any member that matches the mode,
             ignoring tags." :class:`MongoReplicaSetClient` tries each set of
             tags in turn until it finds a set of tags with at least one matching
-            member.
+            member. Defaults to ``[{}]``, meaning "ignore members' tags."
           - `secondary_acceptable_latency_ms`: (integer) Any replica-set member
             whose ping time is within secondary_acceptable_latency_ms of the
             nearest member may accept reads. Default 15 milliseconds.
@@ -558,11 +558,14 @@ class MongoReplicaSetClient(common.BaseObject):
           | **SSL configuration:**
 
           - `ssl`: If ``True``, create the connection to the servers using SSL.
+            Defaults to ``False``.
           - `ssl_keyfile`: The private keyfile used to identify the local
             connection against mongod.  If included with the ``certfile`` then
             only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
+            Defaults to ``None``.
           - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``ssl=True``.
+            connection against mongod. Implies ``ssl=True``. Defaults to
+            ``None``.
           - `ssl_cert_reqs`: Specifies whether a certificate is required from
             the other side of the connection, and whether it will be validated
             if provided. It must be one of the three values ``ssl.CERT_NONE``
@@ -570,11 +573,12 @@ class MongoReplicaSetClient(common.BaseObject):
             (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
             (required and validated). If the value of this parameter is not
             ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
-            to a file of CA certificates. Implies ``ssl=True``.
+            to a file of CA certificates. Implies ``ssl=True``. Defaults to
+            ``ssl.CERT_NONE``.
           - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``None``.
 
         .. versionchanged:: 2.5
            Added additional ssl options
@@ -647,16 +651,18 @@ class MongoReplicaSetClient(common.BaseObject):
                                      "keyword parameter is required.")
 
         self.__net_timeout = self.__opts.get('sockettimeoutms')
-        self.__conn_timeout = self.__opts.get('connecttimeoutms')
+        self.__conn_timeout = self.__opts.get('connecttimeoutms', 20.0)
         self.__wait_queue_timeout = self.__opts.get('waitqueuetimeoutms')
         self.__wait_queue_multiple = self.__opts.get('waitqueuemultiple')
+        self.__socket_keepalive = self.__opts.get('socketkeepalive', False)
         self.__use_ssl = self.__opts.get('ssl', None)
         self.__ssl_keyfile = self.__opts.get('ssl_keyfile', None)
         self.__ssl_certfile = self.__opts.get('ssl_certfile', None)
         self.__ssl_cert_reqs = self.__opts.get('ssl_cert_reqs', None)
         self.__ssl_ca_certs = self.__opts.get('ssl_ca_certs', None)
 
-        ssl_kwarg_keys = [k for k in kwargs.keys() if k.startswith('ssl_')]
+        ssl_kwarg_keys = [k for k in kwargs.keys()
+                          if k.startswith('ssl_') and kwargs[k]]
         if self.__use_ssl is False and ssl_kwarg_keys:
             raise ConfigurationError("ssl has not been enabled but the "
                                      "following ssl parameters have been set: "
@@ -693,7 +699,7 @@ class MongoReplicaSetClient(common.BaseObject):
                 raise ConnectionFailure(str(e))
 
         if username:
-            mechanism = options.get('authmechanism', 'MONGODB-CR')
+            mechanism = options.get('authmechanism', 'DEFAULT')
             source = (
                 options.get('authsource')
                 or self.__default_database_name
@@ -701,8 +707,8 @@ class MongoReplicaSetClient(common.BaseObject):
 
             credentials = auth._build_credentials_tuple(mechanism,
                                                         source,
-                                                        unicode(username),
-                                                        unicode(password),
+                                                        username,
+                                                        password,
                                                         options)
             try:
                 self._cache_credentials(source, credentials, _connect)
@@ -814,7 +820,7 @@ class MongoReplicaSetClient(common.BaseObject):
                 auth.authenticate(credentials, sock_info, self.__simple_command)
                 sock_info.authset.add(credentials)
             finally:
-                member.pool.maybe_return_socket(sock_info)
+                member.maybe_return_socket(sock_info)
 
         self.__auth_credentials[source] = credentials
 
@@ -1002,15 +1008,15 @@ class MongoReplicaSetClient(common.BaseObject):
 
     @property
     def auto_start_request(self):
-        """Is auto_start_request enabled?
-        """
+        """**DEPRECATED** Is auto_start_request enabled?"""
         return self.__auto_start_request
 
     def __simple_command(self, sock_info, dbname, spec):
         """Send a command to the server.
            Returns (response, ping_time in seconds).
         """
-        rqst_id, msg, _ = message.query(0, dbname + '.$cmd', 0, -1, spec)
+        ns = dbname + '.$cmd'
+        rqst_id, msg, _ = message.query(0, ns, 0, -1, spec)
         start = time.time()
         try:
             sock_info.sock.sendall(msg)
@@ -1021,7 +1027,8 @@ class MongoReplicaSetClient(common.BaseObject):
 
         end = time.time()
         response = helpers._unpack_response(response)['data'][0]
-        msg = "command %r failed: %%s" % spec
+        msg = "command %s on namespace %s failed: %%s" % (
+            repr(spec).replace("%", "%%"), ns)
         helpers._check_command_response(response, None, msg)
         return response, end - start
 
@@ -1037,6 +1044,7 @@ class MongoReplicaSetClient(common.BaseObject):
             self.__use_ssl,
             wait_queue_timeout=self.__wait_queue_timeout,
             wait_queue_multiple=self.__wait_queue_multiple,
+            socket_keepalive=self.__socket_keepalive,
             use_greenlets=self.__use_greenlets,
             ssl_keyfile=self.__ssl_keyfile,
             ssl_certfile=self.__ssl_certfile,
@@ -1137,7 +1145,7 @@ class MongoReplicaSetClient(common.BaseObject):
                     sock_info = self.__socket(member, force=True)
                     response, ping_time = self.__simple_command(
                         sock_info, 'admin', {'ismaster': 1})
-                    member.pool.maybe_return_socket(sock_info)
+                    member.maybe_return_socket(sock_info)
                     new_member = member.clone_with(response, ping_time)
                 else:
                     response, pool, ping_time = self.__is_master(node)
@@ -1173,9 +1181,11 @@ class MongoReplicaSetClient(common.BaseObject):
                     if response['ismaster']:
                         writer = node
 
-            except (ConnectionFailure, socket.error), why:
+            except (ConnectionFailure, socket.error, OperationFailure), why:
+                # Member unreachable, or transient auth failure while member
+                # is resyncing credentials.
                 if member:
-                    member.pool.discard_socket(sock_info)
+                    member.discard_socket(sock_info)
                 errors.append("%s:%d: %s" % (node[0], node[1], str(why)))
             if hosts:
                 break
@@ -1203,7 +1213,7 @@ class MongoReplicaSetClient(common.BaseObject):
                         # Not a member of this set.
                         continue
 
-                    member.pool.maybe_return_socket(sock_info)
+                    member.maybe_return_socket(sock_info)
                     new_member = member.clone_with(res, ping_time)
                 else:
                     res, connection_pool, ping_time = self.__is_master(host)
@@ -1216,9 +1226,11 @@ class MongoReplicaSetClient(common.BaseObject):
 
                 members[host] = new_member
 
-            except (ConnectionFailure, socket.error):
+            except (ConnectionFailure, socket.error, OperationFailure):
+                # Member unreachable, or transient auth failure while member
+                # is resyncing credentials.
                 if member:
-                    member.pool.discard_socket(sock_info)
+                    member.discard_socket(sock_info)
                 continue
 
             if res['ismaster']:
@@ -1295,12 +1307,14 @@ class MongoReplicaSetClient(common.BaseObject):
         if self.auto_start_request and not self.in_request():
             self.start_request()
 
-        sock_info = member.pool.get_socket(force=force)
+        sock_info = member.get_socket(force=force)
 
         try:
             self.__check_auth(sock_info)
-        except OperationFailure:
-            member.pool.maybe_return_socket(sock_info)
+        except:
+            # No matter whether an auth failure or network error, increment
+            # the pool's semaphore by returning the socket.
+            member.maybe_return_socket(sock_info)
             raise
         return sock_info
 
@@ -1321,7 +1335,7 @@ class MongoReplicaSetClient(common.BaseObject):
         """
         rs_state = self.__rs_state
         if rs_state.primary_member:
-            rs_state.primary_member.pool.reset()
+            rs_state.primary_member.reset()
 
         threadlocal = self.__make_threadlocal()
         self.__rs_state = rs_state.clone_without_writer(threadlocal)
@@ -1357,10 +1371,9 @@ class MongoReplicaSetClient(common.BaseObject):
         primary, else ``True``.
 
         This method attempts to check the status of the primary with minimal
-        I/O. The current thread / greenlet retrieves a socket (its request
-        socket if it's in a request, or a random idle socket if it's not in a
-        request) from the primary's connection pool and checks whether calling
-        select_ on it raises an error. If there are currently no idle sockets,
+        I/O. The current thread / greenlet retrieves a socket from the
+        primary's connection pool and checks whether calling select_ on it
+        raises an error. If there are currently no idle sockets,
         :meth:`alive` attempts to connect a new socket.
 
         A more certain way to determine primary availability is to ping it::
@@ -1386,7 +1399,7 @@ class MongoReplicaSetClient(common.BaseObject):
                 return False
         finally:
             if primary:
-                primary.pool.maybe_return_socket(sock_info)
+                primary.maybe_return_socket(sock_info)
 
     def __check_response_to_last_error(self, response, is_command):
         """Check a response to a lastError message for errors.
@@ -1513,7 +1526,7 @@ class MongoReplicaSetClient(common.BaseObject):
             except OperationFailure:
                 raise
             except(ConnectionFailure, socket.error), why:
-                member.pool.discard_socket(sock_info)
+                member.discard_socket(sock_info)
                 if _connection_to_use in (None, -1):
                     self.disconnect()
                 raise AutoReconnect(str(why))
@@ -1521,7 +1534,7 @@ class MongoReplicaSetClient(common.BaseObject):
                 sock_info.close()
                 raise
         finally:
-            member.pool.maybe_return_socket(sock_info)
+            member.maybe_return_socket(sock_info)
 
     def __send_and_receive(self, member, msg, **kwargs):
         """Send a message on the given socket and return the response data.
@@ -1543,13 +1556,13 @@ class MongoReplicaSetClient(common.BaseObject):
             if not exhaust:
                 if "network_timeout" in kwargs:
                     sock_info.sock.settimeout(self.__net_timeout)
-                member.pool.maybe_return_socket(sock_info)
+                member.maybe_return_socket(sock_info)
 
             return response, sock_info, member.pool
         except:
             if sock_info is not None:
                 sock_info.close()
-                member.pool.maybe_return_socket(sock_info)
+                member.maybe_return_socket(sock_info)
             raise
 
     def __try_read(self, member, msg, **kwargs):
@@ -1710,30 +1723,26 @@ class MongoReplicaSetClient(common.BaseObject):
 
     def _exhaust_next(self, sock_info):
         """Used with exhaust cursors to get the next batch off the socket.
+
+        Can raise AutoReconnect.
         """
-        return self.__recv_msg(1, None, sock_info)
+        try:
+            return self.__recv_msg(1, None, sock_info)
+        except socket.error, e:
+            raise AutoReconnect(str(e))
 
     def start_request(self):
-        """Ensure the current thread or greenlet always uses the same socket
-        until it calls :meth:`end_request`. For
-        :class:`~pymongo.read_preferences.ReadPreference` PRIMARY,
-        auto_start_request=True ensures consistent reads, even if you read
-        after an unacknowledged write. For read preferences other than PRIMARY,
-        there are no consistency guarantees.
+        """**DEPRECATED**: start_request will be removed in PyMongo 3.0.
 
-        In Python 2.6 and above, or in Python 2.5 with
-        "from __future__ import with_statement", :meth:`start_request` can be
-        used as a context manager:
+        When doing w=0 writes to MongoDB 2.4 or earlier, :meth:`start_request`
+        was sometimes useful to ensure the current thread always used the same
+        socket until it called :meth:`end_request`. This made consistent reads
+        more likely after an unacknowledged write. Requests are no longer
+        useful in modern MongoDB applications, see
+        `PYTHON-785 <https://jira.mongodb.org/browse/PYTHON-785>`_.
 
-        >>> client = pymongo.MongoReplicaSetClient()
-        >>> db = client.test
-        >>> _id = db.test_collection.insert({})
-        >>> with client.start_request():
-        ...     for i in range(100):
-        ...         db.test_collection.update({'_id': _id}, {'$set': {'i':i}})
-        ...
-        ...     # Definitely read the document after the final update completes
-        ...     print db.test_collection.find({'_id': _id})
+        .. versionchanged:: 2.8
+           Deprecated.
 
         .. versionadded:: 2.2
            The :class:`~pymongo.pool.Request` return value.
@@ -1747,36 +1756,24 @@ class MongoReplicaSetClient(common.BaseObject):
         # within a request.
         if 1 == self.__request_counter.inc():
             for member in self.__rs_state.members:
-                member.pool.start_request()
+                member.start_request()
 
         return pool.Request(self)
 
     def in_request(self):
-        """True if :meth:`start_request` has been called, but not
-        :meth:`end_request`, or if `auto_start_request` is True and
+        """**DEPRECATED**: True if :meth:`start_request` has been called, but
+        not :meth:`end_request`, or if `auto_start_request` is True and
         :meth:`end_request` has not been called in this thread or greenlet.
         """
         return bool(self.__request_counter.get())
 
     def end_request(self):
-        """Undo :meth:`start_request` and allow this thread's connections to
-        replica set members to return to the pool.
-
-        Calling :meth:`end_request` allows the :class:`~socket.socket` that has
-        been reserved for this thread by :meth:`start_request` to be returned
-        to the pool. Other threads will then be able to re-use that
-        :class:`~socket.socket`. If your application uses many threads, or has
-        long-running threads that infrequently perform MongoDB operations, then
-        judicious use of this method can lead to performance gains. Care should
-        be taken, however, to make sure that :meth:`end_request` is not called
-        in the middle of a sequence of operations in which ordering is
-        important. This could lead to unexpected results.
-        """
+        """**DEPRECATED**: Undo :meth:`start_request`."""
         rs_state = self.__rs_state
         if 0 == self.__request_counter.dec():
             for member in rs_state.members:
                 # No effect if not in a request
-                member.pool.end_request()
+                member.end_request()
 
             rs_state.unpin_host()
 
@@ -1817,8 +1814,7 @@ class MongoReplicaSetClient(common.BaseObject):
         """Close a single database cursor.
 
         Raises :class:`TypeError` if `cursor_id` is not an instance of
-        ``(int, long)``. What closing the cursor actually means
-        depends on this client's cursor manager.
+        ``(int, long)``.
 
         :Parameters:
           - `cursor_id`: id of cursor to close
@@ -1826,19 +1822,38 @@ class MongoReplicaSetClient(common.BaseObject):
         if not isinstance(cursor_id, (int, long)):
             raise TypeError("cursor_id must be an instance of (int, long)")
 
-        self._send_message(message.kill_cursors([cursor_id]),
-                           _connection_to_use=_conn_id)
+        member = self.__get_rs_state().get(_conn_id)
+
+        # We can't risk taking the lock to reconnect if we're being called
+        # from Cursor.__del__, see PYTHON-799.
+        if not member:
+            warnings.warn("not connected, couldn't close cursor",
+                          stacklevel=2)
+            return
+
+        _, kill_cursors_msg = message.kill_cursors([cursor_id])
+        sock_info = self.__socket(member)
+        try:
+            try:
+                sock_info.sock.sendall(kill_cursors_msg)
+            except:
+                sock_info.close()
+                raise
+        finally:
+            member.maybe_return_socket(sock_info)
 
     def server_info(self):
         """Get information about the MongoDB primary we're connected to.
         """
-        return self.admin.command("buildinfo")
+        return self.admin.command("buildinfo",
+                                  read_preference=ReadPreference.PRIMARY)
 
     def database_names(self):
         """Get a list of the names of all databases on the connected server.
         """
         return [db["name"] for db in
-                self.admin.command("listDatabases")["databases"]]
+                self.admin.command("listDatabases",
+                    read_preference=ReadPreference.PRIMARY)["databases"]]
 
     def drop_database(self, name_or_database):
         """Drop a database.
@@ -1860,11 +1875,16 @@ class MongoReplicaSetClient(common.BaseObject):
                             "%s or Database" % (basestring.__name__,))
 
         self._purge_index(name)
-        self[name].command("dropDatabase")
+        self[name].command("dropDatabase",
+                           read_preference=ReadPreference.PRIMARY)
 
     def copy_database(self, from_name, to_name,
-                      from_host=None, username=None, password=None):
-        """Copy a database, potentially from another host.
+                      from_host=None, username=None, password=None,
+                      mechanism='DEFAULT'):
+        """**DEPRECATED**: Copy a database, potentially from another host.
+
+        :meth:`copy_database` will be removed in PyMongo 3.0. See the
+        :doc:`copy_database examples </examples/copydb>` for alternatives.
 
         Raises :class:`TypeError` if `from_name` or `to_name` is not
         an instance of :class:`basestring` (:class:`str` in python 3).
@@ -1875,7 +1895,9 @@ class MongoReplicaSetClient(common.BaseObject):
         source. Otherwise the database is copied from `from_host`.
 
         If the source database requires authentication, `username` and
-        `password` must be specified.
+        `password` must be specified. By default, use SCRAM-SHA-1 with
+        MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response
+        protocol) for older servers.
 
         :Parameters:
           - `from_name`: the name of the source database
@@ -1883,37 +1905,27 @@ class MongoReplicaSetClient(common.BaseObject):
           - `from_host` (optional): host name to copy from
           - `username` (optional): username for source database
           - `password` (optional): password for source database
+          - `mechanism` (optional): auth method, 'MONGODB-CR' or 'SCRAM-SHA-1'
 
-        .. note:: Specifying `username` and `password` requires server
-           version **>= 1.3.3+**.
+        .. seealso:: The :doc:`copy_database examples </examples/copydb>`.
+
+        .. versionchanged:: 2.8
+           Deprecated copy_database, and added SCRAM-SHA-1 support.
         """
-        if not isinstance(from_name, basestring):
-            raise TypeError("from_name must be an instance "
-                            "of %s" % (basestring.__name__,))
-        if not isinstance(to_name, basestring):
-            raise TypeError("to_name must be an instance "
-                            "of %s" % (basestring.__name__,))
-
-        database._check_name(to_name)
-
-        command = {"fromdb": from_name, "todb": to_name}
-
-        if from_host is not None:
-            command["fromhost"] = from_host
-
+        member = self.__find_primary()
+        sock_info = self.__socket(member)
         try:
-            self.start_request()
-
-            if username is not None:
-                nonce = self.admin.command("copydbgetnonce",
-                                           fromhost=from_host)["nonce"]
-                command["username"] = username
-                command["nonce"] = nonce
-                command["key"] = auth._auth_key(nonce, username, password)
-
-            return self.admin.command("copydb", **command)
+            helpers._copy_database(
+                fromdb=from_name,
+                todb=to_name,
+                fromhost=from_host,
+                mechanism=mechanism,
+                username=username,
+                password=password,
+                sock_info=sock_info,
+                cmd_func=self.__simple_command)
         finally:
-            self.end_request()
+            member.pool.maybe_return_socket(sock_info)
 
     def get_default_database(self):
         """Get the database named in the MongoDB connection URI.

pymongo/pool.py

@@ -63,6 +63,9 @@ class SocketInfo(object):
         self.last_checkout = time.time()
         self.forced = False
 
+        self._min_wire_version = None
+        self._max_wire_version = None
+
         # The pool's pool_id changes with each reset() so we can close sockets
         # created before the last reset.
         self.pool_id = pool_id
@@ -74,6 +77,20 @@ class SocketInfo(object):
             self.sock.close()
         except:
             pass
+        
+    def set_wire_version_range(self, min_wire_version, max_wire_version):
+        self._min_wire_version = min_wire_version
+        self._max_wire_version = max_wire_version
+        
+    @property
+    def min_wire_version(self):
+        assert self._min_wire_version is not None
+        return self._min_wire_version
+        
+    @property
+    def max_wire_version(self):
+        assert self._max_wire_version is not None
+        return self._max_wire_version
 
     def __eq__(self, other):
         # Need to check if other is NO_REQUEST or NO_SOCKET_YET, and then check
@@ -100,7 +117,8 @@ class Pool:
     def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl,
                  use_greenlets, ssl_keyfile=None, ssl_certfile=None,
                  ssl_cert_reqs=None, ssl_ca_certs=None,
-                 wait_queue_timeout=None, wait_queue_multiple=None):
+                 wait_queue_timeout=None, wait_queue_multiple=None,
+                 socket_keepalive=False):
         """
         :Parameters:
           - `pair`: a (hostname, port) tuple
@@ -136,6 +154,9 @@ class Pool:
             free sockets.
           - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
             the number of threads allowed to wait for a socket at one time.
+          - `socket_keepalive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
         """
         # Only check a socket's health with _closed() every once in a while.
         # Can override for testing: 0 to always check, None to never check.
@@ -154,6 +175,7 @@ class Pool:
         self.conn_timeout = conn_timeout
         self.wait_queue_timeout = wait_queue_timeout
         self.wait_queue_multiple = wait_queue_multiple
+        self.socket_keepalive = socket_keepalive
         self.use_ssl = use_ssl
         self.ssl_keyfile = ssl_keyfile
         self.ssl_certfile = ssl_certfile
@@ -191,13 +213,12 @@ class Pool:
         self.pool_id += 1
         self.pid = os.getpid()
 
-        sockets = None
+        # Allocate outside the lock. Triggering a GC while holding the lock
+        # could run Cursor.__del__ and deadlock. See PYTHON-799.
+        new_sockets = set()
+        self.lock.acquire()
         try:
-            # Swapping variables is not atomic. We need to ensure no other
-            # thread is modifying self.sockets, or replacing it, in this
-            # critical section.
-            self.lock.acquire()
-            sockets, self.sockets = self.sockets, set()
+            sockets, self.sockets = self.sockets, new_sockets
         finally:
             self.lock.release()
 
@@ -240,7 +261,9 @@ class Pool:
             try:
                 sock = socket.socket(af, socktype, proto)
                 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-                sock.settimeout(self.conn_timeout or 20.0)
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE,
+                                self.socket_keepalive)
+                sock.settimeout(self.conn_timeout)
                 sock.connect(sa)
                 return sock
             except socket.error, e:
@@ -528,8 +551,11 @@ class Pool:
         for sock_info in self.sockets:
             sock_info.close()
 
-        for request_sock in self._tid_to_sock.values():
-            if request_sock not in (NO_REQUEST, NO_SOCKET_YET):
+        # Don't use self._tid_to_sock.values(): 2to3 would translate to
+        # list(self._tid_to_sock.values()), but during interpreter shutdown
+        # list() may already be set to None.
+        for request_sock in self._tid_to_sock.itervalues():
+            if request_sock not in (None, -1):
                 request_sock.close()
 
 

pymongo/read_preferences.py

@@ -113,7 +113,7 @@ def select_member_with_tags(members, tags, secondary_only, latency):
     fastest = min([candidate.get_avg_ping_time() for candidate in candidates])
     near_candidates = [
         candidate for candidate in candidates
-        if candidate.get_avg_ping_time() - fastest < latency / 1000.]
+        if candidate.get_avg_ping_time() - fastest <= latency / 1000.]
 
     return random.choice(near_candidates)
 

pymongo/replica_set_connection.py

@@ -106,29 +106,30 @@ class ReplicaSetConnection(MongoReplicaSetClient):
             is no timeout. If both `network_timeout` and `socketTimeoutMS` are
             specified `network_timeout` takes precedence, matching
             connection.Connection.
-          - `socketTimeoutMS`: (integer) How long (in milliseconds) a send or
-            receive on a socket can take before timing out. Defaults to ``None``
-            (no timeout).
-          - `connectTimeoutMS`: (integer) How long (in milliseconds) a
+          - `socketTimeoutMS`: (integer or None) How long (in milliseconds) a
+            send or receive on a socket can take before timing out. Defaults
+            to ``None`` (no timeout).
+          - `connectTimeoutMS`: (integer or None) How long (in milliseconds) a
             connection can take to be opened before timing out. Defaults to
             ``20000``.
-          - `waitQueueTimeoutMS`: (integer) How long (in milliseconds) a
-            thread will wait for a socket from the pool if the pool has no
+          - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds)
+            a thread will wait for a socket from the pool if the pool has no
             free sockets. Defaults to ``None`` (no timeout).
-          - `waitQueueMultiple`: (integer) Multiplied by max_pool_size to give
-            the number of threads allowed to wait for a socket at one time.
-            Defaults to ``None`` (no waiters).
+          - `waitQueueMultiple`: (integer or None) Multiplied by max_pool_size
+            to give the number of threads allowed to wait for a socket at one
+            time. Defaults to ``None`` (no waiters).
+          - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
           - `auto_start_request`: If ``True`` (the default), each thread that
             accesses this :class:`ReplicaSetConnection` has a socket allocated
-            to it for the thread's lifetime, for each member of the set. For
-            :class:`~pymongo.read_preferences.ReadPreference` PRIMARY,
-            auto_start_request=True ensures consistent reads, even if you read
-            after an unsafe write. For read preferences other than PRIMARY,
-            there are no consistency guarantees.
+            to it for each member of the set until the thread calls
+            :meth:`end_request` or terminates.
           - `use_greenlets`: if ``True``, use a background Greenlet instead of
             a background thread to monitor state of replica set. Additionally,
             :meth:`start_request()` will ensure that the current greenlet uses
             the same socket for all operations until :meth:`end_request()`.
+            Defaults to ``False``.
             `use_greenlets` with ReplicaSetConnection requires `Gevent
             <http://gevent.org/>`_ to be installed.
 
@@ -164,13 +165,14 @@ class ReplicaSetConnection(MongoReplicaSetClient):
             instead.
           - `read_preference`: The read preference for this connection.
             See :class:`~pymongo.read_preferences.ReadPreference` for available
+            options. Defaults to ``PRIMARY``.
           - `tag_sets`: Read from replica-set members with these tags.
             To specify a priority-order for tag sets, provide a list of
             tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
             set, ``{}``, means "read from any member that matches the mode,
             ignoring tags." :class:`MongoReplicaSetClient` tries each set of
             tags in turn until it finds a set of tags with at least one matching
-            member.
+            member. Defaults to ``[{}]``, meaning "ignore members' tags."
           - `secondary_acceptable_latency_ms`: (integer) Any replica-set member
             whose ping time is within secondary_acceptable_latency_ms of the
             nearest member may accept reads. Default 15 milliseconds.
@@ -180,11 +182,14 @@ class ReplicaSetConnection(MongoReplicaSetClient):
           | **SSL configuration:**
 
           - `ssl`: If ``True``, create the connection to the servers using SSL.
+            Defaults to ``False``.
           - `ssl_keyfile`: The private keyfile used to identify the local
             connection against mongod.  If included with the ``certfile` then
             only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
+            Defaults to ``None``.
           - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``ssl=True``.
+            connection against mongod. Implies ``ssl=True``. Defaults to
+            ``None``.
           - `ssl_cert_reqs`: Specifies whether a certificate is required from
             the other side of the connection, and whether it will be validated
             if provided. It must be one of the three values ``ssl.CERT_NONE``
@@ -192,11 +197,12 @@ class ReplicaSetConnection(MongoReplicaSetClient):
             (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
             (required and validated). If the value of this parameter is not
             ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
-            to a file of CA certificates. Implies ``ssl=True``.
+            to a file of CA certificates. Implies ``ssl=True``. Defaults to
+            ``ssl.CERT_NONE``.
           - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``None``.
 
         .. versionchanged:: 2.5
            Added additional ssl options

setup.py

@@ -33,7 +33,7 @@ from distutils.errors import CCompilerError
 from distutils.errors import DistutilsPlatformError, DistutilsExecError
 from distutils.core import Extension
 
-version = "2.7"
+version = "2.8.1"
 
 f = open("README.rst")
 try:

test/__init__.py

@@ -19,7 +19,8 @@ import os
 import warnings
 
 import pymongo
-from pymongo.errors import ConnectionFailure
+from nose.plugins.skip import SkipTest
+from pymongo.errors import ConnectionFailure, OperationFailure
 
 # hostnames retrieved by MongoReplicaSetClient from isMaster will be of unicode
 # type in Python 2, so ensure these hostnames are unicodes, too. It makes tests
@@ -34,6 +35,83 @@ port2 = int(os.environ.get("DB_PORT2", 27018))
 host3 = unicode(os.environ.get("DB_IP3", 'localhost'))
 port3 = int(os.environ.get("DB_PORT3", 27019))
 
+db_user = unicode(os.environ.get("DB_USER", "administrator"))
+db_pwd = unicode(os.environ.get("DB_PASSWORD", "password"))
+
+
+class AuthContext(object):
+
+    def __init__(self):
+        self.auth_enabled = False
+        self.restricted_localhost = False
+        try:
+            self.client = pymongo.MongoClient(host, port)
+        except ConnectionFailure:
+            self.client = None
+        else:
+            try:
+                command_line = self.client.admin.command('getCmdLineOpts')
+                if self._server_started_with_auth(command_line):
+                    self.auth_enabled = True
+            except OperationFailure, e:
+                msg = e.details.get('errmsg', '')
+                if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
+                    self.auth_enabled = True
+                    self.restricted_localhost = True
+                else:
+                    raise
+            # See if the user has already been set up.
+            try:
+                self.client.admin.authenticate(db_user, db_pwd)
+                self.user_provided = True
+            except OperationFailure, e:
+                msg = e.details.get('errmsg', '')
+                if e.code == 18 or 'auth fails' in msg:
+                    self.user_provided = False
+                else:
+                    raise
+
+    def _server_started_with_auth(self, command_line):
+        # MongoDB >= 2.0
+        if 'parsed' in command_line:
+            parsed = command_line['parsed']
+            # MongoDB >= 2.6
+            if 'security' in parsed:
+                security = parsed['security']
+                if 'authorization' in security:
+                    return security['authorization'] == 'enabled'
+                return security.get('auth', bool(security.get('keyFile')))
+            return parsed.get('auth', bool(parsed.get('keyFile')))
+        # Legacy
+        argv = command_line['argv']
+        return '--auth' in argv or '--keyFile' in argv
+
+    def add_user_and_log_in(self):
+        if not self.user_provided:
+            self.client.admin.add_user(db_user, db_pwd,
+                                       roles=('userAdminAnyDatabase',
+                                              'readWriteAnyDatabase',
+                                              'dbAdminAnyDatabase',
+                                              'clusterAdmin'))
+        self.client.admin.authenticate(db_user, db_pwd)
+
+    def remove_user_and_log_out(self):
+        if not self.user_provided:
+            self.client.admin.remove_user(db_user)
+        self.client.admin.logout()
+        self.client.disconnect()
+
+
+auth_context = AuthContext()
+
+
+def skip_restricted_localhost():
+    """Skip tests when the localhost exception is restricted (SERVER-12621)."""
+    if auth_context.restricted_localhost:
+        raise SkipTest("Cannot test with restricted localhost exception "
+                       "(SERVER-12621).")
+
+
 # Make sure warnings are always raised, regardless of
 # python version.
 def setup():
@@ -42,16 +120,16 @@ def setup():
 
 
 def teardown():
-    try:
-        c = pymongo.MongoClient(host, port)
-    except ConnectionFailure:
-        # Tests where ssl=True can cause connection failures here.
-        # Ignore and continue.
-        return
+    client = auth_context.client
+    if auth_context.auth_enabled:
+        auth_context.add_user_and_log_in()
 
-    c.drop_database("pymongo-pooling-tests")
-    c.drop_database("pymongo_test")
-    c.drop_database("pymongo_test1")
-    c.drop_database("pymongo_test2")
-    c.drop_database("pymongo_test_mike")
-    c.drop_database("pymongo_test_bernie")
+    client.drop_database("pymongo-pooling-tests")
+    client.drop_database("pymongo_test")
+    client.drop_database("pymongo_test1")
+    client.drop_database("pymongo_test2")
+    client.drop_database("pymongo_test_mike")
+    client.drop_database("pymongo_test_bernie")
+
+    if auth_context.auth_enabled:
+        auth_context.remove_user_and_log_out()

test/certificates/crl.pem

@@ -0,0 +1,10 @@
+-----BEGIN X509 CRL-----
+MIIBazCB1QIBATANBgkqhkiG9w0BAQUFADCBkjELMAkGA1UEBhMCVVMxETAPBgNV
+BAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYDVQQKDAUx
+MEdlbjEPMA0GA1UECwwGS2VybmVsMRowGAYDVQQDDBFNeSBDZXJ0IEF1dGhvcml0
+eTEbMBkGCSqGSIb3DQEJARYMcm9vdEBsYXphcnVzFw0xMjEyMTIxODQ3NDFaFw00
+MDA0MjgxODQ3NDFaoA4wDDAKBgNVHRQEAwIBCzANBgkqhkiG9w0BAQUFAAOBgQAu
+PlPDGei2q6kdkoHe8vmDuts7Hm/o9LFbBmn0XUcfHisCJCPsJTyGCsgnfIiBcXJY
+1LMKsQFnYGv28rE2ZPpFg2qNxL+6qUEzCvqaHLX9q1V0F+f8hHDxucNYu52oo/h0
+uNZxB1KPFI2PReG5d3oUYqJ2+EctKkrGtxSPzbN0gg==
+-----END X509 CRL-----

test/certificates/server.pem

@@ -0,0 +1,34 @@
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAK53miP9GczBWXnq
+NxHwQkgVqsDuesjwJbWilMK4gf3fjnf2PN3qDpnGbZbPD0ij8975pIKtSPoDycFm
+A8Mogip0yU2Lv2lL56CWthSBftOFDL2CWIsmuuURFXZPiVLtLytfI9oLASZFlywW
+Cs83qEDTvdW8VoVhVsxV1JFDnpXLAgMBAAECgYBoGBgxrMt97UazhNkCrPT/CV5t
+6lv8E7yMGMrlOyzkCkR4ssQyK3o2qbutJTGbR6czvIM5LKbD9Qqlh3ZrNHokWmTR
+VQQpJxt8HwP5boQvwRHg9+KSGr4JvRko1qxFs9C7Bzjt4r9VxdjhwZPdy0McGI/z
+yPXyQHjqBayrHV1EwQJBANorfCKeIxLhH3LAeUZuRS8ACldJ2N1kL6Ov43/v+0S/
+OprQeBTODuTds3sv7FCT1aYDTOe6JLNOwN2i4YVOMBsCQQDMuCozrwqftD17D06P
+9+lRXUekY5kFBs5j28Xnl8t8jnuxsXtQUTru660LD0QrmDNSauhpEmlpJknicnGt
+hmwRAkEA12MI6bBPlir0/jgxQqxI1w7mJqj8Vg27zpEuO7dzzLoyJHddpcSNBbwu
+npaAakiZK42klj26T9+XHvjYRuAbMwJBAJ5WnwWEkGH/pUHGEAyYQdSVojDKe/MA
+Vae0tzguFswK5C8GyArSGRPsItYYA7D4MlG/sGx8Oh2C6MiFndkJzBECQDcP1y4r
+Qsek151t1zArLKH4gG5dQAeZ0Lc2VeC4nLMUqVwrHcZDdd1RzLlSaH3j1MekFVfT
+6v6rrcNLEVbeuk4=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIC7jCCAlegAwIBAgIBCjANBgkqhkiG9w0BAQUFADCBkjELMAkGA1UEBhMCVVMx
+ETAPBgNVBAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYD
+VQQKDAUxMEdlbjEPMA0GA1UECwwGS2VybmVsMRowGAYDVQQDDBFNeSBDZXJ0IEF1
+dGhvcml0eTEbMBkGCSqGSIb3DQEJARYMcm9vdEBsYXphcnVzMB4XDTEzMTIwNTEz
+MjU0MFoXDTQxMDQyMTEzMjU0MFowajELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE5l
+dyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYDVQQKDAUxMEdlbjEP
+MA0GA1UECwwGS2VybmVsMQ8wDQYDVQQDDAZzZXJ2ZXIwgZ8wDQYJKoZIhvcNAQEB
+BQADgY0AMIGJAoGBAK53miP9GczBWXnqNxHwQkgVqsDuesjwJbWilMK4gf3fjnf2
+PN3qDpnGbZbPD0ij8975pIKtSPoDycFmA8Mogip0yU2Lv2lL56CWthSBftOFDL2C
+WIsmuuURFXZPiVLtLytfI9oLASZFlywWCs83qEDTvdW8VoVhVsxV1JFDnpXLAgMB
+AAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJh
+dGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQgCkKiZhUV9/Zo7RwYYwm2cNK6tzAf
+BgNVHSMEGDAWgBQHQRk6n37FtyJOt7zV3+T8CbhkFjANBgkqhkiG9w0BAQUFAAOB
+gQCbsfr+Q4pty4Fy38lSxoCgnbB4pX6+Ex3xyw5zxDYR3xUlb/uHBiNZ1dBrXBxU
+ekU8dEvf+hx4iRDSW/C5N6BGnBBhCHcrPabo2bEEWKVsbUC3xchTB5rNGkvnMt9t
+G9ol7vanuzjL3S8/2PB33OshkBH570CxqqPflQbdjwt9dg==
+-----END CERTIFICATE-----

test/mod_wsgi_test/README.rst

@@ -0,0 +1,106 @@
+Testing PyMongo with mod_wsgi
+=============================
+
+These tests verify that PyMongo works with Apache and mod_wsgi. They are
+primarily intended to prevent regression of
+`PYTHON-353 <https://jira.mongodb.org/browse/PYTHON-353>`_, a connection leak
+when PyMongo 2.2 was used with Python 2.6 and mod_wsgi 2.8. However, the test
+may also catch concurrency bugs, or incompatibilities between PyMongo's C
+extensions and the way mod_wsgi manages Python sub interpreters. It is
+generally useful to test PyMongo in the unconventional environment that
+mod_wsgi creates.
+
+Test Matrix
+-----------
+
+PyMongo should be tested with several versions of mod_wsgi and a selection
+of Python versions. Each combination of mod_wsgi and Python version should
+be tested with a standalone and a replica set. ``mod_wsgi_test.wsgi``
+detects if the deployment is a replica set and connects to the whole set.
+
+Setup
+-----
+
+Compile Python
+..............
+
+We need a Python interpreter built as a shared library. Download the
+source tarball for each Python version tested, untar it, and run::
+
+    ./configure --prefix=/some/directory --enable-shared
+    make
+    make install
+
+This results in an executable named "python" and a shared
+library named something like "libpython2.7.so.1.0".
+It may be necessary to add /some/directory/lib to your system's
+LD_LIBRARY_PATH, or to make a symlink from your system's default library
+directory to the shared library. For example, on Ubuntu::
+
+    ln -s /some/directory/lib/libpython2.7.so.1.0 /usr/lib64/
+
+Compile mod_wsgi
+................
+
+Compile mod_wsgi for each combination for Python and mod_wsgi version in the
+test matrix. For example, to compile mod_wsgi 3.4 for Python 2.6 on a
+RedHat-like Linux::
+
+    sudo yum install -y httpd httpd-devel
+    wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz
+    tar xzf mod_wsgi-3.4.tar.gz
+    cd mod_wsgi-3.4
+    ./configure --with-python=/some/directory/python
+    make
+    make install
+
+To ease testing of several matrix combinations, copy the resulting
+``mod_wsgi.so`` to a safe place.
+
+Start mongod
+............
+
+Start a standalone listening on port 27017, or a replica set with a member
+listening on port 27017.
+
+Configure Apache
+................
+
+Copy the appropriate version of ``mod_wsgi.so`` into Apache's modules
+directory. Start Apache with the ``mod_wsgi_test.conf`` in this directory.
+
+Run the test
+------------
+
+Run the included ``test_client.py`` script::
+
+    python test/mod_wsgi_test/test_client.py -n 2500 -t 100 parallel \
+         http://localhost/${WORKSPACE}
+
+...where the "n" argument is the total number of requests to make to Apache,
+and "t" specifies the number of threads. ``WORKSPACE`` is the location of
+the PyMongo checkout.
+
+Run this script again with different arguments to make serial requests::
+
+    python test/mod_wsgi_test/test_client.py -n 25000 serial \
+        http://localhost/${WORKSPACE}
+
+The ``test_client.py`` script merely makes HTTP requests to Apache. Its
+exit code is non-zero if any of its requests fails, for example with an
+HTTP 500.
+
+The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`.
+This script inserts some documents into MongoDB at startup, then queries
+documents for each HTTP request.
+
+If PyMongo is leaking connections and "n" is much greater than the ulimit,
+the test will fail when PyMongo exhausts its file descriptors.
+
+Automation
+----------
+
+At MongoDB, Inc. we use a Jenkins job that tests each combination in the
+matrix. The job copies the appropriate version of ``mod_wsgi.so`` into
+place, sets up Apache, starts a single server or replica set,
+and runs ``test_client.py`` with the proper arguments.

test/mod_wsgi_test/mod_wsgi_test.conf

@@ -28,12 +28,9 @@ WSGISocketPrefix /tmp/
     WSGIProcessGroup mod_wsgi_test
 
     # For the convienience of unittests, rather than hard-code the location of
-    # mod_wsgi_test_single_server.wsgi and mod_wsgi_test_replica_set.wsgi,
-    # include it in the URL, so
-    # http://localhost/single_server/location-of-pymongo-checkout will work:
+    # mod_wsgi_test.wsgi, include it in the URL, so
+    # http://localhost/location-of-pymongo-checkout will work:
 
-    WSGIScriptAliasMatch ^/single_server(.+) $1/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi
-
-    WSGIScriptAliasMatch ^/replica_set(.+) $1/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi
+    WSGIScriptAliasMatch ^/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.wsgi
 
 </VirtualHost>

test/mod_wsgi_test/mod_wsgi_test.wsgi

@@ -26,9 +26,18 @@ sys.path.insert(0, repository_path)
 
 import pymongo
 from pymongo.mongo_client import MongoClient
+from pymongo.mongo_replica_set_client import MongoReplicaSetClient
 
 # auto_start_request is part of the PYTHON-353 pathology
 client = MongoClient(auto_start_request=True)
+
+# If the deployment is a replica set, connect to the whole set.
+replica_set_name = client.admin.command('ismaster').get('setName')
+if replica_set_name:
+    client = MongoReplicaSetClient(
+        auto_start_request=True,
+        replicaSet=replica_set_name)
+
 collection = client.test.test
 
 ndocs = 20

test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi

@@ -1,54 +0,0 @@
-# Copyright 2012-2014 MongoDB, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Minimal test of PyMongo in a WSGI application with MongoReplicaSetClient,
-   see bug PYTHON-353.
-"""
-
-import os
-import sys
-
-this_path = os.path.dirname(os.path.join(os.getcwd(), __file__))
-
-# Location of PyMongo checkout
-repository_path = os.path.normpath(os.path.join(this_path, '..', '..'))
-sys.path.insert(0, repository_path)
-
-import pymongo
-from pymongo.mongo_replica_set_client import MongoReplicaSetClient
-
-# auto_start_request is part of the PYTHON-353 pathology
-client = MongoReplicaSetClient(replicaSet='repl0', auto_start_request=True)
-collection = client.test.test
-
-ndocs = 20
-
-collection.drop()
-collection.insert([{'i': i} for i in range(ndocs)])
-client.disconnect()  # Discard main thread's request socket.
-
-try:
-    from mod_wsgi import version as mod_wsgi_version
-except:
-    mod_wsgi_version = None
-
-
-def application(environ, start_response):
-    results = list(collection.find().batch_size(10))
-    assert len(results) == ndocs
-    output = 'python %s, mod_wsgi %s, pymongo %s' % (
-        sys.version, mod_wsgi_version, pymongo.version)
-    response_headers = [('Content-Length', str(len(output)))]
-    start_response('200 OK', response_headers)
-    return [output]

test/test_auth.py

@@ -15,9 +15,11 @@
 """Authentication Tests."""
 
 import os
+import socket
 import sys
 import threading
 import unittest
+import warnings
 
 from urllib import quote_plus
 
@@ -25,12 +27,29 @@ sys.path[0:0] = [""]
 
 from nose.plugins.skip import SkipTest
 
-from pymongo import MongoClient, MongoReplicaSetClient
+from pymongo import (MongoClient,
+                     MongoReplicaSetClient,
+                     auth)
 from pymongo.auth import HAVE_KERBEROS
-from pymongo.errors import OperationFailure, ConfigurationError
+from pymongo.errors import (OperationFailure,
+                            ConfigurationError,
+                            ConnectionFailure,
+                            AutoReconnect)
 from pymongo.read_preferences import ReadPreference
-from test import version, host, port
-from test.utils import is_mongos, server_started_with_auth
+from test import version, host, port, pair, auth_context, db_user, db_pwd
+from test.test_bulk import BulkTestBase
+from test.test_client import get_client
+from test.test_pooling_base import get_pool
+from test.test_replica_set_client import TestReplicaSetClientBase
+from test.test_threads import AutoAuthenticateThreads
+from test.utils import (is_mongos,
+                        remove_all_users,
+                        assertRaisesExactly,
+                        one,
+                        catch_warnings,
+                        TestRequestMixin,
+                        joinall,
+                        get_command_line)
 
 # YOU MUST RUN KINIT BEFORE RUNNING GSSAPI TESTS.
 GSSAPI_HOST = os.environ.get('GSSAPI_HOST')
@@ -44,6 +63,20 @@ SASL_PASS = os.environ.get('SASL_PASS')
 SASL_DB   = os.environ.get('SASL_DB', '$external')
 
 
+def setUpModule():
+    if not auth_context.auth_enabled:
+        raise SkipTest("Server not started with --auth.")
+    if (is_mongos(auth_context.client) and
+            not version.at_least(auth_context.client, (2, 0, 0))):
+        raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
+    auth_context.add_user_and_log_in()
+
+
+def tearDownModule():
+    if auth_context.auth_enabled:
+        auth_context.remove_user_and_log_out()
+
+
 class AutoAuthenticateThread(threading.Thread):
     """Used in testing threaded authentication.
     """
@@ -74,22 +107,27 @@ class TestGSSAPI(unittest.TestCase):
         # Without gssapiServiceName
         self.assertTrue(client.test.authenticate(PRINCIPAL,
                                                  mechanism='GSSAPI'))
-        self.assertTrue(client.database_names())
+        client.database_names()
         uri = ('mongodb://%s@%s:%d/?authMechanism='
                'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
         client = MongoClient(uri)
-        self.assertTrue(client.database_names())
+        client.database_names()
 
         # With gssapiServiceName
         self.assertTrue(client.test.authenticate(PRINCIPAL,
                                                  mechanism='GSSAPI',
                                                  gssapiServiceName='mongodb'))
-        self.assertTrue(client.database_names())
+        client.database_names()
         uri = ('mongodb://%s@%s:%d/?authMechanism='
                'GSSAPI;gssapiServiceName=mongodb' % (quote_plus(PRINCIPAL),
                                                      GSSAPI_HOST, GSSAPI_PORT))
         client = MongoClient(uri)
-        self.assertTrue(client.database_names())
+        client.database_names()
+        uri = ('mongodb://%s@%s:%d/?authMechanism='
+               'GSSAPI;authMechanismProperties=SERVICE_NAME:mongodb' % (
+               quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
+        client = MongoClient(uri)
+        client.database_names()
 
         set_name = client.admin.command('ismaster').get('setName')
         if set_name:
@@ -99,25 +137,31 @@ class TestGSSAPI(unittest.TestCase):
             # Without gssapiServiceName
             self.assertTrue(client.test.authenticate(PRINCIPAL,
                                                      mechanism='GSSAPI'))
-            self.assertTrue(client.database_names())
+            client.database_names()
             uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
                    '=%s' % (quote_plus(PRINCIPAL),
                             GSSAPI_HOST, GSSAPI_PORT, str(set_name)))
             client = MongoReplicaSetClient(uri)
-            self.assertTrue(client.database_names())
+            client.database_names()
 
             # With gssapiServiceName
             self.assertTrue(client.test.authenticate(PRINCIPAL,
                                                      mechanism='GSSAPI',
                                                      gssapiServiceName='mongodb'))
-            self.assertTrue(client.database_names())
+            client.database_names()
             uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
                    '=%s;gssapiServiceName=mongodb' % (quote_plus(PRINCIPAL),
                                                       GSSAPI_HOST,
                                                       GSSAPI_PORT,
                                                       str(set_name)))
             client = MongoReplicaSetClient(uri)
-            self.assertTrue(client.database_names())
+            client.database_names()
+            uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet=%s;'
+                   'authMechanismProperties=SERVICE_NAME:mongodb' % (
+                   quote_plus(PRINCIPAL),
+                   GSSAPI_HOST, GSSAPI_PORT, str(set_name)))
+            client = MongoReplicaSetClient(uri)
+            client.database_names()
 
     def test_gssapi_threaded(self):
 
@@ -156,7 +200,7 @@ class TestGSSAPI(unittest.TestCase):
                 self.assertTrue(thread.success)
 
 
-class TestSASL(unittest.TestCase):
+class TestSASLPlain(unittest.TestCase):
 
     def setUp(self):
         if not SASL_HOST or not SASL_USER or not SASL_PASS:
@@ -226,21 +270,72 @@ class TestSASL(unittest.TestCase):
                           auth_string(SASL_USER, 'not-pwd'))
 
 
+class TestSCRAMSHA1(unittest.TestCase):
+
+    def setUp(self):
+        client = auth_context.client
+        if not version.at_least(client, (2, 7, 2)):
+            raise SkipTest("SCRAM-SHA-1 requires MongoDB >= 2.7.2")
+        ismaster = client.admin.command('ismaster')
+        self.is_mongos = ismaster.get('msg') == 'isdbgrid'
+        self.set_name = ismaster.get('setName')
+
+        # SCRAM-SHA-1 is always enabled beginning in 2.7.8.
+        if not version.at_least(client, (2, 7, 8)):
+            cmd_line = get_command_line(client)
+            if 'SCRAM-SHA-1' not in cmd_line.get(
+                'parsed', {}).get('setParameter',
+                                  {}).get('authenticationMechanisms', ''):
+                raise SkipTest('SCRAM-SHA-1 mechanism not enabled')
+
+        if self.set_name:
+            client.pymongo_test.add_user('user', 'pass',
+                roles=['userAdmin', 'readWrite'],
+                writeConcern={'w': len(ismaster['hosts'])})
+        else:
+            client.pymongo_test.add_user(
+                'user', 'pass', roles=['userAdmin', 'readWrite'])
+
+    def test_scram_sha1(self):
+        client = MongoClient(host, port)
+        self.assertTrue(client.pymongo_test.authenticate(
+            'user', 'pass', mechanism='SCRAM-SHA-1'))
+        client.pymongo_test.command('dbstats')
+
+        client = MongoClient('mongodb://user:pass@%s:%d/pymongo_test'
+                             '?authMechanism=SCRAM-SHA-1' % (host, port))
+        client.pymongo_test.command('dbstats')
+
+        if self.set_name:
+            client = MongoReplicaSetClient(
+                'mongodb://localhost:%d/?replicaSet=%s' % (port, self.set_name))
+            self.assertTrue(client.pymongo_test.authenticate(
+                'user', 'pass', mechanism='SCRAM-SHA-1'))
+            client.pymongo_test.command('dbstats')
+
+            uri = ('mongodb://user:pass'
+                   '@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1'
+                   '&replicaSet=%s' % (host, port, self.set_name))
+            client = MongoReplicaSetClient(uri)
+            client.pymongo_test.command('dbstats')
+            client.read_preference = ReadPreference.SECONDARY
+            client.pymongo_test.command('dbstats')
+
+    def tearDown(self):
+        auth_context.client.pymongo_test.remove_user('user')
+
+
 class TestAuthURIOptions(unittest.TestCase):
 
     def setUp(self):
         client = MongoClient(host, port)
-        # Sharded auth not supported before MongoDB 2.0
-        if is_mongos(client) and not version.at_least(client, (2, 0, 0)):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-        if not server_started_with_auth(client):
-            raise SkipTest('Authentication is not enabled on server')
         response = client.admin.command('ismaster')
         self.set_name = str(response.get('setName', ''))
-        client.admin.add_user('admin', 'pass', roles=['userAdminAnyDatabase',
-                                                      'dbAdminAnyDatabase',
-                                                      'readWriteAnyDatabase',
-                                                      'clusterAdmin'])
+        auth_context.client.admin.add_user('admin', 'pass',
+                                           roles=['userAdminAnyDatabase',
+                                                  'dbAdminAnyDatabase',
+                                                  'readWriteAnyDatabase',
+                                                  'clusterAdmin'])
         client.admin.authenticate('admin', 'pass')
         client.pymongo_test.add_user('user', 'pass',
                                      roles=['userAdmin', 'readWrite'])
@@ -314,19 +409,18 @@ class TestDelegatedAuth(unittest.TestCase):
 
     def setUp(self):
         self.client = MongoClient(host, port)
-        if not version.at_least(self.client, (2, 4, 0)):
+        authed_client = auth_context.client
+        if not version.at_least(authed_client, (2, 4, 0)):
             raise SkipTest('Delegated authentication requires MongoDB >= 2.4.0')
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-        if version.at_least(self.client, (2, 5, 3, -1)):
+        if version.at_least(authed_client, (2, 5, 3, -1)):
             raise SkipTest('Delegated auth does not exist in MongoDB >= 2.5.3')
         # Give admin all privileges.
-        self.client.admin.add_user('admin', 'pass',
-                                   roles=['readAnyDatabase',
-                                          'readWriteAnyDatabase',
-                                          'userAdminAnyDatabase',
-                                          'dbAdminAnyDatabase',
-                                          'clusterAdmin'])
+        authed_client.admin.add_user('admin', 'pass',
+                                     roles=['readAnyDatabase',
+                                            'readWriteAnyDatabase',
+                                            'userAdminAnyDatabase',
+                                            'dbAdminAnyDatabase',
+                                            'clusterAdmin'])
 
     def tearDown(self):
         self.client.admin.authenticate('admin', 'pass')
@@ -370,5 +464,736 @@ class TestDelegatedAuth(unittest.TestCase):
                           self.client.pymongo_test2.foo.find_one)
 
 
+class TestClientAuth(unittest.TestCase):
+
+    def test_copy_db(self):
+        authed_client = auth_context.client
+        if version.at_least(authed_client, (2, 7, 2)):
+            raise SkipTest("SERVER-17034")
+        if is_mongos(authed_client):
+            raise SkipTest("SERVER-6427")
+
+        c = MongoClient(host, port)
+
+        authed_client.admin.add_user("admin", "password")
+        c.admin.authenticate("admin", "password")
+        c.drop_database("pymongo_test")
+        c.drop_database("pymongo_test1")
+        c.pymongo_test.test.insert({"foo": "bar"})
+
+        try:
+            c.pymongo_test.add_user("mike", "password")
+
+            self.assertRaises(OperationFailure, c.copy_database,
+                              "pymongo_test", "pymongo_test1",
+                              username="foo", password="bar")
+            self.assertFalse("pymongo_test1" in c.database_names())
+
+            self.assertRaises(OperationFailure, c.copy_database,
+                              "pymongo_test", "pymongo_test1",
+                              username="mike", password="bar")
+            self.assertFalse("pymongo_test1" in c.database_names())
+
+            c.copy_database("pymongo_test", "pymongo_test1",
+                            username="mike", password="password")
+            self.assertTrue("pymongo_test1" in c.database_names())
+            self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
+        finally:
+            # Cleanup
+            remove_all_users(c.pymongo_test)
+            c.admin.remove_user("admin")
+            c.disconnect()
+
+    def test_auth_from_uri(self):
+        c = MongoClient(host, port)
+        auth_context.client.admin.add_user("admin", "pass")
+        c.admin.authenticate("admin", "pass")
+        try:
+            c.pymongo_test.add_user("user", "pass",
+                                    roles=['userAdmin', 'readWrite'])
+
+            self.assertRaises(ConfigurationError, MongoClient,
+                              "mongodb://foo:bar@%s:%d" % (host, port))
+            self.assertRaises(ConfigurationError, MongoClient,
+                              "mongodb://admin:bar@%s:%d" % (host, port))
+            self.assertRaises(ConfigurationError, MongoClient,
+                              "mongodb://user:pass@%s:%d" % (host, port))
+            MongoClient("mongodb://admin:pass@%s:%d" % (host, port))
+
+            self.assertRaises(ConfigurationError, MongoClient,
+                              "mongodb://admin:pass@%s:%d/pymongo_test" %
+                              (host, port))
+            self.assertRaises(ConfigurationError, MongoClient,
+                              "mongodb://user:foo@%s:%d/pymongo_test" %
+                              (host, port))
+            MongoClient("mongodb://user:pass@%s:%d/pymongo_test" %
+                       (host, port))
+
+            # Auth with lazy connection.
+            MongoClient(
+                "mongodb://user:pass@%s:%d/pymongo_test" % (host, port),
+                _connect=False).pymongo_test.test.find_one()
+
+            # Wrong password.
+            bad_client = MongoClient(
+                "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port),
+                _connect=False)
+
+            self.assertRaises(OperationFailure,
+                              bad_client.pymongo_test.test.find_one)
+
+        finally:
+            # Clean up.
+            remove_all_users(c.pymongo_test)
+            c.admin.remove_user('admin')
+
+    def test_lazy_auth_raises_operation_failure(self):
+        lazy_client = MongoClient(
+            "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port),
+            _connect=False)
+
+        assertRaisesExactly(
+            OperationFailure, lazy_client.test.collection.find_one)
+
+    def test_unix_socket(self):
+        authed_client = auth_context.client
+        if not hasattr(socket, "AF_UNIX"):
+            raise SkipTest("UNIX-sockets are not supported on this system")
+        if (sys.platform == 'darwin' and
+                not version.at_least(authed_client, (2, 7, 1))):
+            raise SkipTest("SERVER-8492")
+
+        mongodb_socket = '/tmp/mongodb-27017.sock'
+        if not os.access(mongodb_socket, os.R_OK):
+            raise SkipTest("Socket file is not accessable")
+
+        self.assertTrue(MongoClient("mongodb://%s" % mongodb_socket))
+
+        authed_client.admin.add_user('admin', 'pass')
+
+        try:
+            client = MongoClient("mongodb://%s" % mongodb_socket)
+            client.admin.authenticate('admin', 'pass')
+            client.pymongo_test.test.save({"dummy": "object"})
+
+            # Confirm we can read via the socket
+            dbs = client.database_names()
+            self.assertTrue("pymongo_test" in dbs)
+
+            # Confirm it fails with a missing socket
+            self.assertRaises(ConnectionFailure, MongoClient,
+                              "mongodb:///tmp/none-existent.sock")
+        finally:
+            authed_client.admin.remove_user('admin')
+
+    def test_auth_network_error(self):
+        # Make sure there's no semaphore leak if we get a network error
+        # when authenticating a new socket with cached credentials.
+        auth_client = get_client()
+
+        auth_context.client.admin.add_user('admin', 'password')
+        auth_client.admin.authenticate('admin', 'password')
+        try:
+            # Get a client with one socket so we detect if it's leaked.
+            c = get_client(max_pool_size=1, waitQueueTimeoutMS=1)
+
+            # Simulate an authenticate() call on a different socket.
+            credentials = auth._build_credentials_tuple(
+                'DEFAULT', 'admin',
+                unicode('admin'), unicode('password'),
+                {})
+
+            c._cache_credentials('test', credentials, connect=False)
+
+            # Cause a network error on the actual socket.
+            pool = get_pool(c)
+            socket_info = one(pool.sockets)
+            socket_info.sock.close()
+
+            # In __check_auth, the client authenticates its socket with the
+            # new credential, but gets a socket.error. Should be reraised as
+            # AutoReconnect.
+            self.assertRaises(AutoReconnect, c.test.collection.find_one)
+
+            # No semaphore leak, the pool is allowed to make a new socket.
+            c.test.collection.find_one()
+        finally:
+            auth_client.admin.remove_user('admin')
+
+
+class TestDatabaseAuth(unittest.TestCase):
+    def setUp(self):
+        self.client = MongoClient(host, port)
+
+    def test_authenticate_add_remove_user(self):
+        authed_client = auth_context.client
+        db = authed_client.pymongo_test
+
+        # Configuration errors
+        self.assertRaises(ValueError, db.add_user, "user", '')
+        self.assertRaises(TypeError, db.add_user, "user", 'password', 15)
+        self.assertRaises(ConfigurationError, db.add_user,
+                          "user", 'password', 'True')
+        self.assertRaises(ConfigurationError, db.add_user,
+                          "user", 'password', True, roles=['read'])
+
+        if version.at_least(authed_client, (2, 5, 3, -1)):
+            ctx = catch_warnings()
+            try:
+                warnings.simplefilter("error", DeprecationWarning)
+                self.assertRaises(DeprecationWarning, db.add_user,
+                                  "user", "password")
+                self.assertRaises(DeprecationWarning, db.add_user,
+                                  "user", "password", True)
+            finally:
+                ctx.exit()
+
+            self.assertRaises(ConfigurationError, db.add_user,
+                              "user", "password", digestPassword=True)
+
+        authed_client.admin.add_user("admin", "password")
+        self.client.admin.authenticate("admin", "password")
+        db = self.client.pymongo_test
+
+        try:
+            # Add / authenticate / remove
+            db.add_user("mike", "password")
+            self.assertRaises(TypeError, db.authenticate, 5, "password")
+            self.assertRaises(TypeError, db.authenticate, "mike", 5)
+            self.assertRaises(OperationFailure,
+                              db.authenticate, "mike", "not a real password")
+            self.assertRaises(OperationFailure,
+                              db.authenticate, "faker", "password")
+            self.assertTrue(db.authenticate("mike", "password"))
+            db.logout()
+            self.assertTrue(db.authenticate(u"mike", u"password"))
+            db.remove_user("mike")
+            db.logout()
+
+            self.assertRaises(OperationFailure,
+                              db.authenticate, "mike", "password")
+
+            # Add / authenticate / change password
+            self.assertRaises(OperationFailure,
+                              db.authenticate, "Gustave", u"Dor\xe9")
+            db.add_user("Gustave", u"Dor\xe9")
+            self.assertTrue(db.authenticate("Gustave", u"Dor\xe9"))
+            db.add_user("Gustave", "password")
+            db.logout()
+            self.assertRaises(OperationFailure,
+                              db.authenticate, "Gustave", u"Dor\xe9")
+            self.assertTrue(db.authenticate("Gustave", u"password"))
+
+            if not version.at_least(authed_client, (2, 5, 3, -1)):
+                # Add a readOnly user
+                db.add_user("Ross", "password", read_only=True)
+                db.logout()
+                self.assertTrue(db.authenticate("Ross", u"password"))
+                self.assertTrue(
+                    db.system.users.find({"readOnly": True}).count())
+                db.logout()
+
+        # Cleanup
+        finally:
+            remove_all_users(db)
+            self.client.admin.remove_user("admin")
+            self.client.admin.logout()
+
+    def test_make_user_readonly(self):
+        admin = self.client.admin
+        auth_context.client.admin.add_user('admin', 'pw')
+        admin.authenticate('admin', 'pw')
+
+        db = self.client.pymongo_test
+
+        try:
+            # Make a read-write user.
+            db.add_user('jesse', 'pw')
+            admin.logout()
+
+            # Check that we're read-write by default.
+            db.authenticate('jesse', 'pw')
+            db.collection.insert({})
+            db.logout()
+
+            # Make the user read-only.
+            admin.authenticate('admin', 'pw')
+            db.add_user('jesse', 'pw', read_only=True)
+            admin.logout()
+
+            db.authenticate('jesse', 'pw')
+            self.assertRaises(OperationFailure, db.collection.insert, {})
+        finally:
+            # Cleanup
+            admin.authenticate('admin', 'pw')
+            remove_all_users(db)
+            admin.remove_user("admin")
+            admin.logout()
+
+    def test_default_roles(self):
+        authed_client = auth_context.client
+        if not version.at_least(authed_client, (2, 5, 3, -1)):
+            raise SkipTest("Default roles only exist in MongoDB >= 2.5.3")
+
+        # "Admin" user
+        db = self.client.admin
+        authed_client.admin.add_user('admin', 'pass')
+        try:
+            db.authenticate('admin', 'pass')
+            info = db.command('usersInfo', 'admin')['users'][0]
+            self.assertEqual("root", info['roles'][0]['role'])
+
+            # Read only "admin" user
+            db.add_user('ro-admin', 'pass', read_only=True)
+            db.logout()
+            db.authenticate('ro-admin', 'pass')
+            info = db.command('usersInfo', 'ro-admin')['users'][0]
+            self.assertEqual("readAnyDatabase", info['roles'][0]['role'])
+            db.logout()
+
+        # Cleanup
+        finally:
+            db.authenticate('admin', 'pass')
+            db.remove_user('ro-admin')
+            db.remove_user('admin')
+            db.logout()
+
+        db.connection.disconnect()
+
+        # "Non-admin" user
+        db = self.client.pymongo_test
+        authed_client.pymongo_test.add_user('user', 'pass')
+        try:
+            db.authenticate('user', 'pass')
+            info = db.command('usersInfo', 'user')['users'][0]
+            self.assertEqual("dbOwner", info['roles'][0]['role'])
+
+            # Read only "Non-admin" user
+            db.add_user('ro-user', 'pass', read_only=True)
+            db.logout()
+            db.authenticate('ro-user', 'pass')
+            info = db.command('usersInfo', 'ro-user')['users'][0]
+            self.assertEqual("read", info['roles'][0]['role'])
+            db.logout()
+
+        # Cleanup
+        finally:
+            db.authenticate('user', 'pass')
+            remove_all_users(db)
+            db.logout()
+
+    def test_new_user_cmds(self):
+        authed_client = auth_context.client
+        if not version.at_least(authed_client, (2, 5, 3, -1)):
+            raise SkipTest("User manipulation through commands "
+                           "requires MongoDB >= 2.5.3")
+
+        db = self.client.pymongo_test
+        authed_client.pymongo_test.add_user("amalia", "password",
+                                            roles=["userAdmin"])
+        db.authenticate("amalia", "password")
+        try:
+            # This tests the ability to update user attributes.
+            db.add_user("amalia", "new_password",
+                        customData={"secret": "koalas"})
+
+            user_info = db.command("usersInfo", "amalia")
+            self.assertTrue(user_info["users"])
+            amalia_user = user_info["users"][0]
+            self.assertEqual(amalia_user["user"], "amalia")
+            self.assertEqual(amalia_user["customData"], {"secret": "koalas"})
+        finally:
+            db.remove_user("amalia")
+            db.logout()
+
+    def test_authenticate_and_safe(self):
+        db = auth_context.client.auth_test
+
+        db.add_user("bernie", "password",
+                    roles=["userAdmin", "dbAdmin", "readWrite"])
+        db.authenticate("bernie", "password")
+        try:
+            db.test.remove({})
+            self.assertTrue(db.test.insert({"bim": "baz"}))
+            self.assertEqual(1, db.test.count())
+
+            self.assertEqual(1,
+                             db.test.update({"bim": "baz"},
+                                            {"$set": {"bim": "bar"}}).get('n'))
+
+            self.assertEqual(1,
+                             db.test.remove({}).get('n'))
+
+            self.assertEqual(0, db.test.count())
+        finally:
+            db.remove_user("bernie")
+            db.logout()
+
+    def test_authenticate_and_request(self):
+        # Database.authenticate() needs to be in a request - check that it
+        # always runs in a request, and that it restores the request state
+        # (in or not in a request) properly when it's finished.
+        self.assertFalse(self.client.auto_start_request)
+        db = self.client.pymongo_test
+        auth_context.client.pymongo_test.add_user(
+            "mike", "password",
+            roles=["userAdmin", "dbAdmin", "readWrite"])
+        try:
+            self.assertFalse(self.client.in_request())
+            self.assertTrue(db.authenticate("mike", "password"))
+            self.assertFalse(self.client.in_request())
+
+            request_cx = get_client(auto_start_request=True)
+            request_db = request_cx.pymongo_test
+            self.assertTrue(request_db.authenticate("mike", "password"))
+            self.assertTrue(request_cx.in_request())
+        finally:
+            db.authenticate("mike", "password")
+            db.remove_user("mike")
+            db.logout()
+            request_db.logout()
+
+    def test_authenticate_multiple(self):
+        client = get_client()
+        authed_client = auth_context.client
+        if (is_mongos(authed_client) and not
+                version.at_least(authed_client, (2, 2, 0))):
+            raise SkipTest("Need mongos >= 2.2.0")
+
+        # Setup
+        authed_client.pymongo_test.test.drop()
+        authed_client.pymongo_test1.test.drop()
+        users_db = client.pymongo_test
+        admin_db = client.admin
+        other_db = client.pymongo_test1
+
+        authed_client.admin.add_user('admin', 'pass',
+                                     roles=["userAdminAnyDatabase", "dbAdmin",
+                                            "clusterAdmin", "readWrite"])
+        try:
+            self.assertTrue(admin_db.authenticate('admin', 'pass'))
+
+            if version.at_least(self.client, (2, 5, 3, -1)):
+                admin_db.add_user('ro-admin', 'pass',
+                                  roles=["userAdmin", "readAnyDatabase"])
+            else:
+                admin_db.add_user('ro-admin', 'pass', read_only=True)
+
+            users_db.add_user('user', 'pass',
+                              roles=["userAdmin", "readWrite"])
+
+            admin_db.logout()
+            self.assertRaises(OperationFailure, users_db.test.find_one)
+
+            # Regular user should be able to query its own db, but
+            # no other.
+            users_db.authenticate('user', 'pass')
+            self.assertEqual(0, users_db.test.count())
+            self.assertRaises(OperationFailure, other_db.test.find_one)
+
+            # Admin read-only user should be able to query any db,
+            # but not write.
+            admin_db.authenticate('ro-admin', 'pass')
+            self.assertEqual(0, other_db.test.count())
+            self.assertRaises(OperationFailure,
+                              other_db.test.insert, {})
+
+            # Force close all sockets
+            client.disconnect()
+
+            # We should still be able to write to the regular user's db
+            self.assertTrue(users_db.test.remove())
+            # And read from other dbs...
+            self.assertEqual(0, other_db.test.count())
+            # But still not write to other dbs...
+            self.assertRaises(OperationFailure,
+                              other_db.test.insert, {})
+
+        # Cleanup
+        finally:
+            admin_db.logout()
+            users_db.logout()
+            admin_db.authenticate('admin', 'pass')
+            remove_all_users(users_db)
+            admin_db.remove_user('ro-admin')
+            admin_db.remove_user('admin')
+
+
+class TestReplicaSetClientAuth(TestReplicaSetClientBase, TestRequestMixin):
+
+    def test_init_disconnected_with_auth_failure(self):
+        c = MongoReplicaSetClient(
+            "mongodb://user:pass@somedomainthatdoesntexist", replicaSet="rs",
+            connectTimeoutMS=1, _connect=False)
+
+        self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
+
+    def test_init_disconnected_with_auth(self):
+        c = self._get_client()
+
+        auth_context.client.admin.add_user("admin", "pass")
+        c.admin.authenticate("admin", "pass")
+        try:
+            c.pymongo_test.add_user("user", "pass",
+                                    roles=['readWrite', 'userAdmin'])
+
+            # Auth with lazy connection.
+            host = one(self.hosts)
+            uri = "mongodb://user:pass@%s:%d/pymongo_test?replicaSet=%s" % (
+                host[0], host[1], self.name)
+
+            authenticated_client = MongoReplicaSetClient(uri, _connect=False)
+            authenticated_client.pymongo_test.test.find_one()
+
+            # Wrong password.
+            bad_uri = ("mongodb://user:wrong@%s:%d/pymongo_test?replicaSet=%s"
+                       % (host[0], host[1], self.name))
+
+            bad_client = MongoReplicaSetClient(bad_uri, _connect=False)
+            self.assertRaises(
+                OperationFailure, bad_client.pymongo_test.test.find_one)
+
+        finally:
+            # Clean up.
+            remove_all_users(c.pymongo_test)
+            c.admin.remove_user('admin')
+
+    def test_lazy_auth_raises_operation_failure(self):
+        lazy_client = MongoReplicaSetClient(
+            "mongodb://user:wrong@%s/pymongo_test" % pair,
+            replicaSet=self.name,
+            _connect=False)
+
+        assertRaisesExactly(
+            OperationFailure, lazy_client.test.collection.find_one)
+
+    def test_copy_db(self):
+        authed_client = auth_context.client
+        if version.at_least(authed_client, (2, 7, 2)):
+            raise SkipTest("SERVER-17034")
+
+        authed_client.admin.add_user("admin", "password")
+        c = self._get_client()
+        c.admin.authenticate("admin", "password")
+        c.drop_database("pymongo_test1")
+        c.pymongo_test.test.insert({"foo": "bar"})
+
+        try:
+            c.pymongo_test.add_user("mike", "password")
+
+            self.assertRaises(OperationFailure, c.copy_database,
+                              "pymongo_test", "pymongo_test1",
+                              username="foo", password="bar")
+            self.assertFalse("pymongo_test1" in c.database_names())
+
+            self.assertRaises(OperationFailure, c.copy_database,
+                              "pymongo_test", "pymongo_test1",
+                              username="mike", password="bar")
+            self.assertFalse("pymongo_test1" in c.database_names())
+
+            c.copy_database("pymongo_test", "pymongo_test1",
+                            username="mike", password="password")
+            self.assertTrue("pymongo_test1" in c.database_names())
+            res = c.pymongo_test1.test.find_one(_must_use_master=True)
+            self.assertEqual("bar", res["foo"])
+        finally:
+            # Cleanup
+            remove_all_users(c.pymongo_test)
+            c.admin.remove_user("admin")
+        c.close()
+
+    def test_auth_network_error(self):
+        # Make sure there's no semaphore leak if we get a network error
+        # when authenticating a new socket with cached credentials.
+        # Get a client with one socket so we detect if it's leaked.
+
+        # Generous wait queue timeout in case the main thread contends
+        # with the monitor, though -- a semaphore leak will be detected
+        # eventually, even with a long timeout.
+        c = self._get_client(max_pool_size=1, waitQueueTimeoutMS=10000)
+
+        # Simulate an authenticate() call on a different socket.
+        credentials = auth._build_credentials_tuple(
+            'DEFAULT', 'admin',
+            unicode(db_user), unicode(db_pwd),
+            {})
+
+        c._cache_credentials('test', credentials, connect=False)
+
+        # Cause a network error on the actual socket.
+        pool = get_pool(c)
+        socket_info = one(pool.sockets)
+        socket_info.sock.close()
+
+        # In __check_auth, the client authenticates its socket with the
+        # new credential, but gets a socket.error. Reraised as AutoReconnect,
+        # unless periodic monitoring or Pool._check prevent the error.
+        try:
+            c.test.collection.find_one()
+        except AutoReconnect:
+            pass
+
+        # No semaphore leak, the pool is allowed to make a new socket.
+        c.test.collection.find_one()
+
+
+class TestBulkAuthorization(BulkTestBase):
+
+    def setUp(self):
+        super(TestBulkAuthorization, self).setUp()
+        self.client = client = get_client()
+        authed_client = auth_context.client
+        if not version.at_least(authed_client, (2, 5, 3)):
+            raise SkipTest('Need at least MongoDB 2.5.3 with auth')
+
+        db = client.pymongo_test
+        self.coll = db.test
+
+        authed_client.pymongo_test.test.drop()
+        authed_client.pymongo_test.add_user('dbOwner', 'pw', roles=['dbOwner'])
+        db.authenticate('dbOwner', 'pw')
+        db.add_user('readonly', 'pw', roles=['read'])
+        db.command(
+            'createRole', 'noremove',
+            privileges=[{
+                'actions': ['insert', 'update', 'find'],
+                'resource': {'db': 'pymongo_test', 'collection': 'test'}
+            }],
+            roles=[])
+
+        db.add_user('noremove', 'pw', roles=['noremove'])
+        db.logout()
+
+    def test_readonly(self):
+        # We test that an authorization failure aborts the batch and is raised
+        # as OperationFailure.
+        db = self.client.pymongo_test
+        db.authenticate('readonly', 'pw')
+        bulk = self.coll.initialize_ordered_bulk_op()
+        bulk.insert({'x': 1})
+        self.assertRaises(OperationFailure, bulk.execute)
+
+    def test_no_remove(self):
+        # We test that an authorization failure aborts the batch and is raised
+        # as OperationFailure.
+        db = self.client.pymongo_test
+        db.authenticate('noremove', 'pw')
+        bulk = self.coll.initialize_ordered_bulk_op()
+        bulk.insert({'x': 1})
+        bulk.find({'x': 2}).upsert().replace_one({'x': 2})
+        bulk.find({}).remove()  # Prohibited.
+        bulk.insert({'x': 3})   # Never attempted.
+        self.assertRaises(OperationFailure, bulk.execute)
+        self.assertEqual(set([1, 2]), set(self.coll.distinct('x')))
+
+    def tearDown(self):
+        db = self.client.pymongo_test
+        db.logout()
+        db.authenticate('dbOwner', 'pw')
+        db.command('dropRole', 'noremove')
+        remove_all_users(db)
+        db.logout()
+
+
+class BaseTestThreadsAuth(object):
+    """
+    Base test class for TestThreadsAuth and TestThreadsAuthReplicaSet. (This is
+    not itself a unittest.TestCase, otherwise it'd be run twice -- once when
+    nose imports this module, and once when nose imports
+    test_threads_replica_set_connection.py, which imports this module.)
+    """
+    def _get_client(self):
+        """
+        Intended for overriding in TestThreadsAuthReplicaSet. This method
+        returns a MongoClient here, and a MongoReplicaSetClient in
+        test_threads_replica_set_connection.py.
+        """
+        # Regular test client
+        return get_client()
+
+    def setUp(self):
+        client = self._get_client()
+        self.client = client
+        auth_context.client.admin.add_user('admin-user', 'password',
+                                           roles=['clusterAdmin',
+                                                  'dbAdminAnyDatabase',
+                                                  'readWriteAnyDatabase',
+                                                  'userAdminAnyDatabase'])
+        self.client.admin.authenticate("admin-user", "password")
+        self.client.auth_test.add_user("test-user", "password",
+                                       roles=['readWrite'])
+
+    def tearDown(self):
+        # Remove auth users from databases
+        self.client.admin.authenticate("admin-user", "password")
+        self.client.drop_database('auth_test')
+        remove_all_users(self.client.auth_test)
+        self.client.admin.remove_user('admin-user')
+        # Clear client reference so that RSC's monitor thread
+        # dies.
+        self.client = None
+
+    def test_auto_auth_login(self):
+        client = self._get_client()
+        self.assertRaises(OperationFailure, client.auth_test.test.find_one)
+
+        # Admin auth
+        client = self._get_client()
+        client.admin.authenticate("admin-user", "password")
+
+        nthreads = 10
+        threads = []
+        for _ in xrange(nthreads):
+            t = AutoAuthenticateThreads(client.auth_test.test, 100)
+            t.start()
+            threads.append(t)
+
+        joinall(threads)
+
+        for t in threads:
+            self.assertTrue(t.success)
+
+        # Database-specific auth
+        client = self._get_client()
+        client.auth_test.authenticate("test-user", "password")
+
+        threads = []
+        for _ in xrange(nthreads):
+            t = AutoAuthenticateThreads(client.auth_test.test, 100)
+            t.start()
+            threads.append(t)
+
+        joinall(threads)
+
+        for t in threads:
+            self.assertTrue(t.success)
+
+
+class TestThreadsAuth(BaseTestThreadsAuth, unittest.TestCase):
+    pass
+
+
+class TestThreadsAuthReplicaSet(TestReplicaSetClientBase, BaseTestThreadsAuth):
+
+    def setUp(self):
+        """
+        Prepare to test all the same things that TestThreads tests, but do it
+        with a replica-set client
+        """
+        TestReplicaSetClientBase.setUp(self)
+        BaseTestThreadsAuth.setUp(self)
+
+    def tearDown(self):
+        TestReplicaSetClientBase.tearDown(self)
+        BaseTestThreadsAuth.tearDown(self)
+
+    def _get_client(self):
+        """
+        Override TestThreadsAuth, so its tests run on a MongoReplicaSetClient
+        instead of a regular MongoClient.
+        """
+        return MongoReplicaSetClient(pair, replicaSet=self.name)
+
+
 if __name__ == "__main__":
     unittest.main()

test/test_binary.py

@@ -33,9 +33,14 @@ from bson.binary import *
 from bson.py3compat import b, binary_type
 from bson.son import SON
 from nose.plugins.skip import SkipTest
+from test import skip_restricted_localhost
 from test.test_client import get_client
 from pymongo.mongo_client import MongoClient
 
+
+setUpModule = skip_restricted_localhost
+
+
 class TestBinary(unittest.TestCase):
     def test_binary(self):
         a_string = "hello world"

test/test_bson.py

@@ -33,13 +33,15 @@ from nose.plugins.skip import SkipTest
 import bson
 from bson import (BSON,
                   decode_all,
+                  decode_file_iter,
+                  decode_iter,
                   is_valid,
                   Regex)
 from bson.binary import Binary, UUIDLegacy
 from bson.code import Code
 from bson.objectid import ObjectId
 from bson.dbref import DBRef
-from bson.py3compat import b
+from bson.py3compat import b, StringIO
 from bson.son import SON
 from bson.timestamp import Timestamp
 from bson.errors import (InvalidBSON,
@@ -156,6 +158,61 @@ class TestBSON(unittest.TestCase):
                                       "\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                                       "\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                                       "\x05\x00\x00\x00\x00")))
+        self.assertEqual([{"test": u"hello world"}, {}],
+                         list(decode_iter(
+                            b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
+                              "\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
+                              "\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
+                              "\x05\x00\x00\x00\x00"))))
+        self.assertEqual([{"test": u"hello world"}, {}],
+                         list(decode_file_iter(StringIO(
+                            b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
+                              "\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
+                              "\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
+                              "\x05\x00\x00\x00\x00")))))
+
+    def test_invalid_decodes(self):
+        # Invalid object size (not enough bytes in document for even
+        # an object size of first object.
+        # NOTE: decode_all and decode_iter don't care, not sure if they should?
+        self.assertRaises(InvalidBSON, list,
+                          decode_file_iter(StringIO(b("\x1B"))))
+
+        # An object size that's too small to even include the object size,
+        # but is correctly encoded, along with a correct EOO (and no data).
+        data = b("\x01\x00\x00\x00\x00")
+        self.assertRaises(InvalidBSON, decode_all, data)
+        self.assertRaises(InvalidBSON, list, decode_iter(data))
+        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
+
+        # One object, but with object size listed smaller than it is in the
+        # data.
+        data = b("\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
+                 "\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
+                 "\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
+                 "\x05\x00\x00\x00\x00")
+        self.assertRaises(InvalidBSON, decode_all, data)
+        self.assertRaises(InvalidBSON, list, decode_iter(data))
+        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
+
+        # One object, missing the EOO at the end.
+        data = b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
+                 "\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
+                 "\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
+                 "\x05\x00\x00\x00")
+        self.assertRaises(InvalidBSON, decode_all, data)
+        self.assertRaises(InvalidBSON, list, decode_iter(data))
+        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
+
+        # One object, sized correctly, with a spot for an EOO, but the EOO
+        # isn't 0x00.
+        data = b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
+                 "\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
+                 "\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
+                 "\x05\x00\x00\x00\xFF")
+        self.assertRaises(InvalidBSON, decode_all, data)
+        self.assertRaises(InvalidBSON, list, decode_iter(data))
+        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
 
     def test_data_timestamp(self):
         self.assertEqual({"test": Timestamp(4, 20)},

test/test_bulk.py

@@ -23,12 +23,14 @@ sys.path[0:0] = [""]
 
 from bson import InvalidDocument, SON
 from pymongo.errors import BulkWriteError, InvalidOperation, OperationFailure
-from test import version
+from pymongo.mongo_client import _partition_node, MongoClient
+from test import version, skip_restricted_localhost
 from test.test_client import get_client
-from test.utils import (oid_generated_on_client,
-                        remove_all_users,
-                        server_started_with_auth,
-                        server_started_with_nojournal)
+from test.utils import oid_generated_on_client, get_command_line
+
+
+setUpModule = skip_restricted_localhost
+
 
 class BulkTestBase(unittest.TestCase):
 
@@ -600,6 +602,28 @@ class TestBulk(BulkTestBase):
 
         self.assertEqual(1, self.coll.find({'x': 1}).count())
 
+    def test_client_generated_upsert_id(self):
+        batch = self.coll.initialize_ordered_bulk_op()
+        batch.find({'_id': 0}).upsert().update_one({'$set': {'a': 0}})
+        batch.find({'a': 1}).upsert().replace_one({'_id': 1})
+        if not version.at_least(self.coll.database.connection, (2, 6, 0)):
+            # This case is only possible in MongoDB versions before 2.6.
+            batch.find({'_id': 3}).upsert().replace_one({'_id': 2})
+        else:
+            # This is just here to make the counts right in all cases.
+            batch.find({'_id': 2}).upsert().replace_one({'_id': 2})
+        result = batch.execute()
+        self.assertEqualResponse(
+            {'nMatched': 0,
+             'nModified': 0,
+             'nUpserted': 3,
+             'nInserted': 0,
+             'nRemoved': 0,
+             'upserted': [{'index': 0, '_id': 0},
+                          {'index': 1, '_id': 1},
+                          {'index': 2, '_id': 2}]},
+            result)
+
     def test_single_ordered_batch(self):
         batch = self.coll.initialize_ordered_bulk_op()
         batch.insert({'a': 1})
@@ -893,42 +917,110 @@ class TestBulkWriteConcern(BulkTestBase):
         ismaster = client.test.command('ismaster')
         self.is_repl = bool(ismaster.get('setName'))
         self.w = len(ismaster.get("hosts", []))
+
+        self.secondary = None
+        if self.w > 1:
+            for member in ismaster['hosts']:
+                if member != ismaster['primary']:
+                    host, port = _partition_node(member)
+                    self.secondary = MongoClient(host, port)
+                    break
+
+        self.client = client
         self.coll = client.pymongo_test.test
         self.coll.remove()
 
+        # We tested wtimeout errors by specifying a write concern greater than
+        # the number of members, but in MongoDB 2.7.8+ this causes a different
+        # sort of error, "Not enough data-bearing nodes". In recent servers we
+        # use a failpoint to pause replication on a secondary.
+        self.need_replication_stopped = version.at_least(self.client,
+                                                         (2, 7, 8))
+
+        self.test_commands_enabled = ("enableTestCommands=1"
+                                      in get_command_line(self.client)["argv"])
+
+    def cause_wtimeout(self, batch):
+        if self.need_replication_stopped:
+            if not self.test_commands_enabled:
+                raise SkipTest("Test commands must be enabled.")
+
+            self.secondary.admin.command('configureFailPoint',
+                                         'rsSyncApplyStop',
+                                         mode='alwaysOn')
+
+            try:
+                return batch.execute({'w': self.w, 'wtimeout': 1})
+            finally:
+                self.secondary.admin.command('configureFailPoint',
+                                             'rsSyncApplyStop',
+                                             mode='off')
+        else:
+            return batch.execute({'w': self.w + 1, 'wtimeout': 1})
+
     def test_fsync_and_j(self):
+        if not version.at_least(self.client, (1, 8, 2)):
+            raise SkipTest("Need at least MongoDB 1.8.2")
         batch = self.coll.initialize_ordered_bulk_op()
         batch.insert({'a': 1})
         self.assertRaises(
             OperationFailure,
             batch.execute, {'fsync': True, 'j': True})
 
-    def test_j_without_journal(self):
-        client = self.coll.database.connection
-        if not server_started_with_nojournal(client):
-            raise SkipTest("Need mongod started with --nojournal")
-
-        # Using j=True without journaling is a hard failure.
-        batch = self.coll.initialize_ordered_bulk_op()
-        batch.insert({})
-        self.assertRaises(OperationFailure, batch.execute, {'j': True})
-
     def test_write_concern_failure_ordered(self):
+        if not self.is_repl:
+            raise SkipTest("Need a replica set to test.")
+
+        # Ensure we don't raise on wnote.
+        batch = self.coll.initialize_ordered_bulk_op()
+        batch.find({"something": "that does not exist"}).remove()
+        self.assertTrue(batch.execute({"w": self.w}))
 
         batch = self.coll.initialize_ordered_bulk_op()
         batch.insert({'a': 1})
         batch.insert({'a': 2})
 
-        # Using w > 1 with no replication is a hard failure.
-        if not self.is_repl:
-            self.assertRaises(OperationFailure,
-                              batch.execute, {'w': 5, 'wtimeout': 1})
-
         # Replication wtimeout is a 'soft' error.
         # It shouldn't stop batch processing.
+        try:
+            self.cause_wtimeout(batch)
+        except BulkWriteError, exc:
+            result = exc.details
+            self.assertEqual(exc.code, 65)
         else:
+            self.fail("Error not raised")
+
+        self.assertEqualResponse(
+            {'nMatched': 0,
+             'nModified': 0,
+             'nUpserted': 0,
+             'nInserted': 2,
+             'nRemoved': 0,
+             'upserted': [],
+             'writeErrors': []},
+            result)
+
+        # When talking to legacy servers there will be a
+        # write concern error for each operation.
+        self.assertTrue(len(result['writeConcernErrors']) > 0)
+
+        failed = result['writeConcernErrors'][0]
+        self.assertEqual(64, failed['code'])
+        self.assertTrue(isinstance(failed['errmsg'], basestring))
+
+        self.coll.remove()
+        self.coll.ensure_index('a', unique=True)
+
+        # Fail due to write concern support as well
+        # as duplicate key error on ordered batch.
+        try:
+            batch = self.coll.initialize_ordered_bulk_op()
+            batch.insert({'a': 1})
+            batch.find({'a': 3}).upsert().replace_one({'b': 1})
+            batch.insert({'a': 1})
+            batch.insert({'a': 2})
             try:
-                batch.execute({'w': self.w + 1, 'wtimeout': 1})
+                self.cause_wtimeout(batch)
             except BulkWriteError, exc:
                 result = exc.details
                 self.assertEqual(exc.code, 65)
@@ -938,76 +1030,68 @@ class TestBulkWriteConcern(BulkTestBase):
             self.assertEqualResponse(
                 {'nMatched': 0,
                  'nModified': 0,
-                 'nUpserted': 0,
-                 'nInserted': 2,
+                 'nUpserted': 1,
+                 'nInserted': 1,
                  'nRemoved': 0,
-                 'upserted': [],
-                 'writeErrors': []},
+                 'upserted': [{'index': 1, '_id': '...'}],
+                 'writeErrors': [
+                     {'index': 2,
+                      'code': 11000,
+                      'errmsg': '...',
+                      'op': {'_id': '...', 'a': 1}}]},
                 result)
 
-            # When talking to legacy servers there will be a
-            # write concern error for each operation.
-            self.assertTrue(len(result['writeConcernErrors']) > 0)
-
-            failed = result['writeConcernErrors'][0]
-            self.assertEqual(64, failed['code'])
-            self.assertTrue(isinstance(failed['errmsg'], basestring))
-
-            self.coll.remove()
-            self.coll.ensure_index('a', unique=True)
-
-            # Fail due to write concern support as well
-            # as duplicate key error on ordered batch.
-            try:
-                batch = self.coll.initialize_ordered_bulk_op()
-                batch.insert({'a': 1})
-                batch.find({'a': 3}).upsert().replace_one({'b': 1})
-                batch.insert({'a': 1})
-                batch.insert({'a': 2})
-                try:
-                    batch.execute({'w': self.w + 1, 'wtimeout': 1})
-                except BulkWriteError, exc:
-                    result = exc.details
-                    self.assertEqual(exc.code, 65)
-                else:
-                    self.fail("Error not raised")
-
-                self.assertEqualResponse(
-                    {'nMatched': 0,
-                     'nModified': 0,
-                     'nUpserted': 1,
-                     'nInserted': 1,
-                     'nRemoved': 0,
-                     'upserted': [{'index': 1, '_id': '...'}],
-                     'writeErrors': [
-                         {'index': 2,
-                          'code': 11000,
-                          'errmsg': '...',
-                          'op': {'_id': '...', 'a': 1}}]},
-                    result)
-
-                self.assertEqual(2, len(result['writeConcernErrors']))
-                failed = result['writeErrors'][0]
-                self.assertTrue("duplicate" in failed['errmsg'])
-            finally:
-                self.coll.drop_index([('a', 1)])
+            self.assertEqual(2, len(result['writeConcernErrors']))
+            failed = result['writeErrors'][0]
+            self.assertTrue("duplicate" in failed['errmsg'])
+        finally:
+            self.coll.drop_index([('a', 1)])
 
     def test_write_concern_failure_unordered(self):
+        if not self.is_repl:
+            raise SkipTest("Need a replica set to test.")
+
+        # Ensure we don't raise on wnote.
+        batch = self.coll.initialize_unordered_bulk_op()
+        batch.find({"something": "that does not exist"}).remove()
+        self.assertTrue(batch.execute({"w": self.w}))
 
         batch = self.coll.initialize_unordered_bulk_op()
         batch.insert({'a': 1})
         batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3, 'b': 1}})
         batch.insert({'a': 2})
 
-        # Using w > 1 with no replication is a hard failure.
-        if not self.is_repl:
-            self.assertRaises(OperationFailure,
-                              batch.execute, {'w': 5, 'wtimeout': 1})
         # Replication wtimeout is a 'soft' error.
         # It shouldn't stop batch processing.
+        try:
+            self.cause_wtimeout(batch)
+        except BulkWriteError, exc:
+            result = exc.details
+            self.assertEqual(exc.code, 65)
         else:
+            self.fail("Error not raised")
+
+        self.assertEqual(2, result['nInserted'])
+        self.assertEqual(1, result['nUpserted'])
+        self.assertEqual(0, len(result['writeErrors']))
+        # When talking to legacy servers there will be a
+        # write concern error for each operation.
+        self.assertTrue(len(result['writeConcernErrors']) > 1)
+
+        self.coll.remove()
+        self.coll.ensure_index('a', unique=True)
+
+        # Fail due to write concern support as well
+        # as duplicate key error on unordered batch.
+        try:
+            batch = self.coll.initialize_unordered_bulk_op()
+            batch.insert({'a': 1})
+            batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3,
+                                                               'b': 1}})
+            batch.insert({'a': 1})
+            batch.insert({'a': 2})
             try:
-                batch.execute({'w': self.w + 1, 'wtimeout': 1})
+                self.cause_wtimeout(batch)
             except BulkWriteError, exc:
                 result = exc.details
                 self.assertEqual(exc.code, 65)
@@ -1016,54 +1100,27 @@ class TestBulkWriteConcern(BulkTestBase):
 
             self.assertEqual(2, result['nInserted'])
             self.assertEqual(1, result['nUpserted'])
-            self.assertEqual(0, len(result['writeErrors']))
+            self.assertEqual(1, len(result['writeErrors']))
             # When talking to legacy servers there will be a
             # write concern error for each operation.
             self.assertTrue(len(result['writeConcernErrors']) > 1)
 
-            self.coll.remove()
-            self.coll.ensure_index('a', unique=True)
+            failed = result['writeErrors'][0]
+            self.assertEqual(2, failed['index'])
+            self.assertEqual(11000, failed['code'])
+            self.assertTrue(isinstance(failed['errmsg'], basestring))
+            self.assertEqual(1, failed['op']['a'])
 
-            # Fail due to write concern support as well
-            # as duplicate key error on unordered batch.
-            try:
-                batch = self.coll.initialize_unordered_bulk_op()
-                batch.insert({'a': 1})
-                batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3,
-                                                                   'b': 1}})
-                batch.insert({'a': 1})
-                batch.insert({'a': 2})
-                try:
-                    batch.execute({'w': self.w + 1, 'wtimeout': 1})
-                except BulkWriteError, exc:
-                    result = exc.details
-                    self.assertEqual(exc.code, 65)
-                else:
-                    self.fail("Error not raised")
+            failed = result['writeConcernErrors'][0]
+            self.assertEqual(64, failed['code'])
+            self.assertTrue(isinstance(failed['errmsg'], basestring))
 
-                self.assertEqual(2, result['nInserted'])
-                self.assertEqual(1, result['nUpserted'])
-                self.assertEqual(1, len(result['writeErrors']))
-                # When talking to legacy servers there will be a
-                # write concern error for each operation.
-                self.assertTrue(len(result['writeConcernErrors']) > 1)
-
-                failed = result['writeErrors'][0]
-                self.assertEqual(2, failed['index'])
-                self.assertEqual(11000, failed['code'])
-                self.assertTrue(isinstance(failed['errmsg'], basestring))
-                self.assertEqual(1, failed['op']['a'])
-
-                failed = result['writeConcernErrors'][0]
-                self.assertEqual(64, failed['code'])
-                self.assertTrue(isinstance(failed['errmsg'], basestring))
-
-                upserts = result['upserted']
-                self.assertEqual(1, len(upserts))
-                self.assertEqual(1, upserts[0]['index'])
-                self.assertTrue(upserts[0].get('_id'))
-            finally:
-                self.coll.drop_index([('a', 1)])
+            upserts = result['upserted']
+            self.assertEqual(1, len(upserts))
+            self.assertEqual(1, upserts[0]['index'])
+            self.assertTrue(upserts[0].get('_id'))
+        finally:
+            self.coll.drop_index([('a', 1)])
 
 
 class TestBulkNoResults(BulkTestBase):
@@ -1117,63 +1174,5 @@ class TestBulkNoResults(BulkTestBase):
         self.assertTrue(self.coll.find_one({'_id': 1}) is None)
 
 
-class TestBulkAuthorization(BulkTestBase):
-
-    def setUp(self):
-        super(TestBulkAuthorization, self).setUp()
-        self.client = client = get_client()
-        if (not server_started_with_auth(client)
-                or not version.at_least(client, (2, 5, 3))):
-            raise SkipTest('Need at least MongoDB 2.5.3 with auth')
-
-        db = client.pymongo_test
-        self.coll = db.test
-        self.coll.remove()
-
-        db.add_user('dbOwner', 'pw', roles=['dbOwner'])
-        db.authenticate('dbOwner', 'pw')
-        db.add_user('readonly', 'pw', roles=['read'])
-        db.command(
-            'createRole', 'noremove',
-            privileges=[{
-                'actions': ['insert', 'update', 'find'],
-                'resource': {'db': 'pymongo_test', 'collection': 'test'}
-            }],
-            roles=[])
-
-        db.add_user('noremove', 'pw', roles=['noremove'])
-        db.logout()
-
-    def test_readonly(self):
-        # We test that an authorization failure aborts the batch and is raised
-        # as OperationFailure.
-        db = self.client.pymongo_test
-        db.authenticate('readonly', 'pw')
-        bulk = self.coll.initialize_ordered_bulk_op()
-        bulk.insert({'x': 1})
-        self.assertRaises(OperationFailure, bulk.execute)
-
-    def test_no_remove(self):
-        # We test that an authorization failure aborts the batch and is raised
-        # as OperationFailure.
-        db = self.client.pymongo_test
-        db.authenticate('noremove', 'pw')
-        bulk = self.coll.initialize_ordered_bulk_op()
-        bulk.insert({'x': 1})
-        bulk.find({'x': 2}).upsert().replace_one({'x': 2})
-        bulk.find({}).remove()  # Prohibited.
-        bulk.insert({'x': 3})   # Never attempted.
-        self.assertRaises(OperationFailure, bulk.execute)
-        self.assertEqual(set([1, 2]), set(self.coll.distinct('x')))
-
-    def tearDown(self):
-        db = self.client.pymongo_test
-        db.logout()
-        db.authenticate('dbOwner', 'pw')
-        db.command('dropRole', 'noremove')
-        remove_all_users(db)
-        db.logout()
-
-
 if __name__ == "__main__":
     unittest.main()

test/test_client.py

@@ -22,6 +22,7 @@ import sys
 import time
 import thread
 import unittest
+import warnings
 
 
 sys.path[0:0] = [""]
@@ -33,33 +34,69 @@ from bson.tz_util import utc
 from pymongo.mongo_client import MongoClient
 from pymongo.database import Database
 from pymongo.pool import SocketInfo
-from pymongo import thread_util, common
+from pymongo import thread_util
 from pymongo.errors import (AutoReconnect,
                             ConfigurationError,
                             ConnectionFailure,
                             InvalidName,
-                            OperationFailure,
-                            PyMongoError)
-from test import version, host, port, pair
+                            OperationFailure)
+from pymongo.read_preferences import ReadPreference
+from test import version, host, port, pair, skip_restricted_localhost
 from test.pymongo_mocks import MockClient
 from test.utils import (assertRaisesExactly,
+                        catch_warnings,
                         delay,
                         is_mongos,
-                        remove_all_users,
                         server_is_master_with_slave,
                         server_started_with_auth,
                         TestRequestMixin,
                         _TestLazyConnectMixin,
+                        _TestExhaustCursorMixin,
                         lazy_client_trial,
                         NTHREADS,
                         get_pool)
 
+setUpModule = skip_restricted_localhost
+
 
 def get_client(*args, **kwargs):
     return MongoClient(host, port, *args, **kwargs)
 
 
 class TestClient(unittest.TestCase, TestRequestMixin):
+
+    def test_keyword_arg_defaults(self):
+        client = MongoClient(socketTimeoutMS=None,
+                             connectTimeoutMS=20000,
+                             waitQueueTimeoutMS=None,
+                             waitQueueMultiple=None,
+                             socketKeepAlive=False,
+                             auto_start_request=False,
+                             use_greenlets=False,
+                             replicaSet=None,
+                             read_preference=ReadPreference.PRIMARY,
+                             tag_sets=[{}],
+                             ssl=False,
+                             ssl_keyfile=None,
+                             ssl_certfile=None,
+                             ssl_ca_certs=None,
+                             _connect=False)
+        self.assertEqual(None, client._MongoClient__net_timeout)
+        # socket.Socket.settimeout takes a float in seconds
+        self.assertEqual(20.0, client._MongoClient__conn_timeout)
+        self.assertEqual(None, client._MongoClient__wait_queue_timeout)
+        self.assertEqual(None, client._MongoClient__wait_queue_multiple)
+        self.assertFalse(client._MongoClient__socket_keepalive)
+        self.assertFalse(client.auto_start_request)
+        self.assertFalse(client.use_greenlets)
+        self.assertEqual(None, client._MongoClient__repl)
+        self.assertEqual(ReadPreference.PRIMARY, client.read_preference)
+        self.assertEqual([{}], client.tag_sets)
+        self.assertFalse(client._MongoClient__use_ssl)
+        self.assertEqual(None, client._MongoClient__ssl_keyfile)
+        self.assertEqual(None, client._MongoClient__ssl_certfile)
+        self.assertEqual(None, client._MongoClient__ssl_ca_certs)
+
     def test_types(self):
         self.assertRaises(TypeError, MongoClient, 1)
         self.assertRaises(TypeError, MongoClient, 1.14)
@@ -224,71 +261,39 @@ class TestClient(unittest.TestCase, TestRequestMixin):
         # from a master in a master-slave pair.
         if server_is_master_with_slave(c):
             raise SkipTest("SERVER-2329")
-        # We test copy twice; once starting in a request and once not. In
-        # either case the copy should succeed (because it starts a request
-        # internally) and should leave us in the same state as before the copy.
-        c.start_request()
 
-        self.assertRaises(TypeError, c.copy_database, 4, "foo")
-        self.assertRaises(TypeError, c.copy_database, "foo", 4)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertRaises(TypeError, c.copy_database, 4, "foo")
+            self.assertRaises(TypeError, c.copy_database, "foo", 4)
+            self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
 
-        self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
-
-        c.pymongo_test.test.drop()
-        c.drop_database("pymongo_test1")
-        c.drop_database("pymongo_test2")
-        self.assertFalse("pymongo_test1" in c.database_names())
-        self.assertFalse("pymongo_test2" in c.database_names())
-
-        c.pymongo_test.test.insert({"foo": "bar"})
-
-        c.copy_database("pymongo_test", "pymongo_test1")
-        # copy_database() didn't accidentally end the request
-        self.assertTrue(c.in_request())
-
-        self.assertTrue("pymongo_test1" in c.database_names())
-        self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
-
-        c.end_request()
-        self.assertFalse(c.in_request())
-        c.copy_database("pymongo_test", "pymongo_test2",
-                        "%s:%d" % (host, port))
-        # copy_database() didn't accidentally restart the request
-        self.assertFalse(c.in_request())
-
-        self.assertTrue("pymongo_test2" in c.database_names())
-        self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
-
-        # See SERVER-6427 for mongos
-        if (version.at_least(c, (1, 3, 3, 1)) and
-            not is_mongos(c) and server_started_with_auth(c)):
+            c.pymongo_test.test.drop()
+            c.pymongo_test.test.insert({"foo": "bar"})
 
             c.drop_database("pymongo_test1")
+            self.assertFalse("pymongo_test1" in c.database_names())
 
-            c.admin.add_user("admin", "password")
-            c.admin.authenticate("admin", "password")
-            try:
-                c.pymongo_test.add_user("mike", "password")
-
-                self.assertRaises(OperationFailure, c.copy_database,
-                                  "pymongo_test", "pymongo_test1",
-                                  username="foo", password="bar")
-                self.assertFalse("pymongo_test1" in c.database_names())
-
-                self.assertRaises(OperationFailure, c.copy_database,
-                                  "pymongo_test", "pymongo_test1",
-                                  username="mike", password="bar")
-                self.assertFalse("pymongo_test1" in c.database_names())
+            c.copy_database("pymongo_test", "pymongo_test1")
+            self.assertTrue("pymongo_test1" in c.database_names())
+            self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
+            c.drop_database("pymongo_test1")
 
+            # XXX - SERVER-15318
+            if not (version.at_least(c, (2, 6, 4)) and is_mongos(c)):
+                self.assertFalse(c.in_request())
                 c.copy_database("pymongo_test", "pymongo_test1",
-                                username="mike", password="password")
+                                "%s:%d" % (host, port))
+                # copy_database() didn't accidentally restart the request
+                self.assertFalse(c.in_request())
+
                 self.assertTrue("pymongo_test1" in c.database_names())
                 self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
-            finally:
-                # Cleanup
-                remove_all_users(c.pymongo_test)
-                c.admin.remove_user("admin")
-                c.disconnect()
+
+            c.drop_database("pymongo_test1")
+        finally:
+            ctx.exit()
 
     def test_iteration(self):
         client = MongoClient(host, port)
@@ -315,11 +320,16 @@ class TestClient(unittest.TestCase, TestRequestMixin):
     def test_from_uri(self):
         c = MongoClient(host, port)
 
-        self.assertEqual(c, MongoClient("mongodb://%s:%d" % (host, port)))
-        self.assertTrue(MongoClient(
-            "mongodb://%s:%d" % (host, port), slave_okay=True).slave_okay)
-        self.assertTrue(MongoClient(
-            "mongodb://%s:%d/?slaveok=true;w=2" % (host, port)).slave_okay)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertEqual(c, MongoClient("mongodb://%s:%d" % (host, port)))
+            self.assertTrue(MongoClient(
+                "mongodb://%s:%d" % (host, port), slave_okay=True).slave_okay)
+            self.assertTrue(MongoClient(
+                "mongodb://%s:%d/?slaveok=true;w=2" % (host, port)).slave_okay)
+        finally:
+            ctx.exit()
 
     def test_get_default_database(self):
         c = MongoClient("mongodb://%s:%d/foo" % (host, port), _connect=False)
@@ -336,75 +346,13 @@ class TestClient(unittest.TestCase, TestRequestMixin):
         c = MongoClient(uri, _connect=False)
         self.assertEqual(Database(c, 'foo'), c.get_default_database())
 
-    def test_auth_from_uri(self):
-        c = MongoClient(host, port)
-        # Sharded auth not supported before MongoDB 2.0
-        if is_mongos(c) and not version.at_least(c, (2, 0, 0)):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-        if not server_started_with_auth(c):
-            raise SkipTest('Authentication is not enabled on server')
-
-        c.admin.add_user("admin", "pass")
-        c.admin.authenticate("admin", "pass")
-        try:
-            c.pymongo_test.add_user("user", "pass", roles=['userAdmin', 'readWrite'])
-
-            self.assertRaises(ConfigurationError, MongoClient,
-                              "mongodb://foo:bar@%s:%d" % (host, port))
-            self.assertRaises(ConfigurationError, MongoClient,
-                              "mongodb://admin:bar@%s:%d" % (host, port))
-            self.assertRaises(ConfigurationError, MongoClient,
-                              "mongodb://user:pass@%s:%d" % (host, port))
-            MongoClient("mongodb://admin:pass@%s:%d" % (host, port))
-
-            self.assertRaises(ConfigurationError, MongoClient,
-                              "mongodb://admin:pass@%s:%d/pymongo_test" %
-                              (host, port))
-            self.assertRaises(ConfigurationError, MongoClient,
-                              "mongodb://user:foo@%s:%d/pymongo_test" %
-                              (host, port))
-            MongoClient("mongodb://user:pass@%s:%d/pymongo_test" %
-                       (host, port))
-
-            # Auth with lazy connection.
-            MongoClient(
-                "mongodb://user:pass@%s:%d/pymongo_test" % (host, port),
-                _connect=False).pymongo_test.test.find_one()
-
-            # Wrong password.
-            bad_client = MongoClient(
-                "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port),
-                _connect=False)
-
-            self.assertRaises(OperationFailure,
-                              bad_client.pymongo_test.test.find_one)
-
-        finally:
-            # Clean up.
-            remove_all_users(c.pymongo_test)
-            remove_all_users(c.admin)
-
-    def test_lazy_auth_raises_operation_failure(self):
-        # Check if we have the prerequisites to run this test.
-        c = MongoClient(host, port)
-        if not server_started_with_auth(c):
-            raise SkipTest('Authentication is not enabled on server')
-
-        if is_mongos(c) and not version.at_least(c, (2, 0, 0)):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-
-        lazy_client = MongoClient(
-            "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port),
-            _connect=False)
-
-        assertRaisesExactly(
-            OperationFailure, lazy_client.test.collection.find_one)
-
     def test_unix_socket(self):
         if not hasattr(socket, "AF_UNIX"):
             raise SkipTest("UNIX-sockets are not supported on this system")
+        client = MongoClient(host, port)
         if (sys.platform == 'darwin' and
-            server_started_with_auth(MongoClient(host, port))):
+                server_started_with_auth(client) and
+                not version.at_least(client, (2, 7, 1))):
             raise SkipTest("SERVER-8492")
 
         mongodb_socket = '/tmp/mongodb-27017.sock'
@@ -576,6 +524,10 @@ class TestClient(unittest.TestCase, TestRequestMixin):
         self.assertEqual(pool.wait_queue_multiple, 2)
         self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6)
 
+    def test_socketKeepAlive(self):
+        client = MongoClient(host, port, socketKeepAlive=True)
+        self.assertTrue(get_pool(client).socket_keepalive)
+
     def test_tz_aware(self):
         self.assertRaises(ConfigurationError, MongoClient, tz_aware='foo')
 
@@ -990,6 +942,28 @@ with client.start_request() as request:
         client = get_client(_connect=False)
         client.pymongo_test.test.remove(w=0)
 
+    def test_kill_cursors_warning(self):
+        # If kill_cursors is called while the client is disconnected, it
+        # can't risk taking the lock to reconnect, in case it's being called
+        # from Cursor.__del__, see PYTHON-799. Test that it shows a warning
+        # in this case.
+        client = MongoClient(host, port)
+        collection = client.pymongo_test.test
+        collection.insert({} for _ in range(4))
+        cursor = collection.find().batch_size(1)
+        cursor.next()
+        client.disconnect()
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("error", UserWarning)
+            self.assertRaises(UserWarning, cursor.close)
+        finally:
+            ctx.exit()
+
+        # Reconnect.
+        collection.find_one()
+        cursor.close()
+
 
 class TestClientLazyConnect(unittest.TestCase, _TestLazyConnectMixin):
     def _get_client(self, **kwargs):
@@ -1101,5 +1075,10 @@ class TestMongoClientFailover(unittest.TestCase):
         c.db.collection.find_one()
 
 
+class TestExhaustCursor(_TestExhaustCursorMixin, unittest.TestCase):
+    def _get_client(self, **kwargs):
+        return get_client(**kwargs)
+
+
 if __name__ == "__main__":
     unittest.main()

test/test_collection.py

@@ -36,7 +36,7 @@ from bson.objectid import ObjectId
 from bson.py3compat import b
 from bson.son import SON, RE_TYPE
 from pymongo import (ASCENDING, DESCENDING, GEO2D,
-                     GEOHAYSTACK, GEOSPHERE, HASHED)
+                     GEOHAYSTACK, GEOSPHERE, HASHED, TEXT)
 from pymongo import message as message_module
 from pymongo.collection import Collection
 from pymongo.command_cursor import CommandCursor
@@ -51,10 +51,9 @@ from pymongo.errors import (DocumentTooLarge,
                             OperationFailure,
                             WTimeoutError)
 from test.test_client import get_client
-from test.utils import (is_mongos, joinall, enable_text_search, get_pool,
-                        oid_generated_on_client)
-from test import (qcheck,
-                  version)
+from test.utils import (catch_warnings, enable_text_search,
+                        get_pool, is_mongos, joinall, oid_generated_on_client)
+from test import qcheck, version, skip_restricted_localhost
 
 have_uuid = True
 try:
@@ -63,6 +62,9 @@ except ImportError:
     have_uuid = False
 
 
+setUpModule = skip_restricted_localhost
+
+
 class TestCollection(unittest.TestCase):
 
     def setUp(self):
@@ -120,8 +122,7 @@ class TestCollection(unittest.TestCase):
 
         db.test.drop_indexes()
         db.test.insert({})
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 1)
+        self.assertEqual(len(db.test.index_information()), 1)
 
         db.test.create_index("hello")
         db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)])
@@ -129,10 +130,7 @@ class TestCollection(unittest.TestCase):
         # Tuple instead of list.
         db.test.create_index((("world", ASCENDING),))
 
-        count = 0
-        for _ in db.system.indexes.find({"ns": u"pymongo_test.test"}):
-            count += 1
-        self.assertEqual(count, 4)
+        self.assertEqual(len(db.test.index_information()), 4)
 
         db.test.drop_indexes()
         ix = db.test.create_index([("hello", DESCENDING),
@@ -140,20 +138,14 @@ class TestCollection(unittest.TestCase):
         self.assertEqual(ix, "hello_world")
 
         db.test.drop_indexes()
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 1)
+        self.assertEqual(len(db.test.index_information()), 1)
         db.test.create_index("hello")
-        self.assertTrue(u"hello_1" in
-                        [a["name"] for a in db.system.indexes
-                         .find({"ns": u"pymongo_test.test"})])
+        self.assertTrue(u"hello_1" in db.test.index_information())
 
         db.test.drop_indexes()
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 1)
+        self.assertEqual(len(db.test.index_information()), 1)
         db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)])
-        self.assertTrue(u"hello_-1_world_1" in
-                        [a["name"] for a in db.system.indexes
-                         .find({"ns": u"pymongo_test.test"})])
+        self.assertTrue(u"hello_-1_world_1" in db.test.index_information())
 
         db.test.drop()
         db.test.insert({'a': 1})
@@ -226,19 +218,21 @@ class TestCollection(unittest.TestCase):
     def test_deprecated_ttl_index_kwarg(self):
         db = self.db
 
-        # In Python 2.6+ we could use the catch_warnings context
-        # manager to test this warning nicely. As we can't do that
-        # we must test raising errors before the ignore filter is applied.
-        warnings.simplefilter("error", DeprecationWarning)
+        ctx = catch_warnings()
         try:
+            warnings.simplefilter("error", DeprecationWarning)
             self.assertRaises(DeprecationWarning, lambda:
                               db.test.ensure_index("goodbye", ttl=10))
         finally:
-            warnings.resetwarnings()
-            warnings.simplefilter("ignore")
+            ctx.exit()
 
-        self.assertEqual("goodbye_1",
-                         db.test.ensure_index("goodbye", ttl=10))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertEqual("goodbye_1",
+                             db.test.ensure_index("goodbye", ttl=10))
+        finally:
+            ctx.exit()
         self.assertEqual(None, db.test.ensure_index("goodbye"))
 
     def test_ensure_unique_index_threaded(self):
@@ -269,49 +263,27 @@ class TestCollection(unittest.TestCase):
         self.assertEqual(10001, coll.count())
         coll.drop()
 
-    def test_index_on_binary(self):
-        db = self.db
-        db.drop_collection("test")
-        db.test.save({"bin": Binary(b("def"))})
-        db.test.save({"bin": Binary(b("abc"))})
-        db.test.save({"bin": Binary(b("ghi"))})
-
-        self.assertEqual(db.test.find({"bin": Binary(b("abc"))})
-                         .explain()["nscanned"], 3)
-
-        db.test.create_index("bin")
-        self.assertEqual(db.test.find({"bin": Binary(b("abc"))})
-                         .explain()["nscanned"], 1)
-
     def test_drop_index(self):
         db = self.db
         db.test.drop_indexes()
         db.test.create_index("hello")
         name = db.test.create_index("goodbye")
 
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 3)
+        self.assertEqual(len(db.test.index_information()), 3)
         self.assertEqual(name, "goodbye_1")
         db.test.drop_index(name)
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 2)
-        self.assertTrue(u"hello_1" in
-                        [a["name"] for a in db.system.indexes
-                         .find({"ns": u"pymongo_test.test"})])
+        self.assertEqual(len(db.test.index_information()), 2)
+        self.assertTrue(u"hello_1" in db.test.index_information())
 
         db.test.drop_indexes()
         db.test.create_index("hello")
         name = db.test.create_index("goodbye")
 
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 3)
         self.assertEqual(name, "goodbye_1")
+        self.assertEqual(len(db.test.index_information()), 3)
         db.test.drop_index([("goodbye", ASCENDING)])
-        self.assertEqual(db.system.indexes.find({"ns": u"pymongo_test.test"})
-                         .count(), 2)
-        self.assertTrue(u"hello_1" in
-                        [a["name"] for a in db.system.indexes
-                         .find({"ns": u"pymongo_test.test"})])
+        self.assertEqual(len(db.test.index_information()), 2)
+        self.assertTrue(u"hello_1" in db.test.index_information())
 
     def test_reindex(self):
         db = self.db
@@ -417,7 +389,7 @@ class TestCollection(unittest.TestCase):
 
         db = self.db
         db.test.drop_indexes()
-        self.assertEqual("t_text", db.test.create_index([("t", "text")]))
+        self.assertEqual("t_text", db.test.create_index([("t", TEXT)]))
         index_info = db.test.index_information()["t_text"]
         self.assertTrue("weights" in index_info)
 
@@ -448,12 +420,19 @@ class TestCollection(unittest.TestCase):
         self.assertEqual("geo_2dsphere",
                          db.test.create_index([("geo", GEOSPHERE)]))
 
+        for name, info in db.test.index_information().items():
+            field, idx_type = info['key'][0]
+            if field == 'geo' and idx_type == '2dsphere':
+                break
+        else:
+            self.fail("2dsphere index not found.")
+
         poly = {"type": "Polygon",
                 "coordinates": [[[40,5], [40,6], [41,6], [41,5], [40,5]]]}
         query = {"geo": {"$within": {"$geometry": poly}}}
 
-        cursor = db.test.find(query).explain()['cursor']
-        self.assertTrue('S2Cursor' in cursor or 'geo_2dsphere' in cursor)
+        # This query will error without a 2dsphere index.
+        db.test.find(query)
 
         db.test.drop_indexes()
 
@@ -466,8 +445,13 @@ class TestCollection(unittest.TestCase):
         self.assertEqual("a_hashed",
                          db.test.create_index([("a", HASHED)]))
 
-        self.assertEqual("BtreeCursor a_hashed",
-                db.test.find({'a': 1}).explain()['cursor'])
+        for name, info in db.test.index_information().items():
+            field, idx_type = info['key'][0]
+            if field == 'a' and idx_type == 'hashed':
+                break
+        else:
+            self.fail("hashed index not found.")
+
         db.test.drop_indexes()
 
     def test_index_sparse(self):
@@ -494,6 +478,8 @@ class TestCollection(unittest.TestCase):
         db.test.insert({'i': 3})
 
     def test_index_drop_dups(self):
+        if version.at_least(self.client, (2, 7)):
+            raise SkipTest("dropDups no longer supported in MongoDB >=2.7.")
         # Try dropping duplicates
         db = self.db
         self._drop_dups_setup(db)
@@ -599,15 +585,13 @@ class TestCollection(unittest.TestCase):
 
     def test_options(self):
         db = self.db
-        db.drop_collection("test")
-        db.test.save({})
-        self.assertEqual(db.test.options(), {})
-        self.assertEqual(db.test.doesnotexist.options(), {})
-
         db.drop_collection("test")
         if version.at_least(db.connection, (1, 9)):
             db.create_collection("test", capped=True, size=4096)
-            self.assertEqual(db.test.options(), {"capped": True, 'size': 4096})
+            result = db.test.options()
+            # mongos 2.2.x adds an $auth field when auth is enabled.
+            result.pop('$auth', None)
+            self.assertEqual(result, {"capped": True, 'size': 4096})
         else:
             db.create_collection("test", capped=True)
             self.assertEqual(db.test.options(), {"capped": True})
@@ -853,10 +837,15 @@ class TestCollection(unittest.TestCase):
         )
 
         # Misconfigured value for safe
-        self.assertRaises(
-            TypeError,
-            lambda: db.test.insert([{'i': 2}] * 2, safe=1),
-        )
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertRaises(
+                TypeError,
+                lambda: db.test.insert([{'i': 2}] * 2, safe=1),
+            )
+        finally:
+            ctx.exit()
 
     def test_insert_iterables(self):
         db = self.db
@@ -959,15 +948,19 @@ class TestCollection(unittest.TestCase):
         db.test.save({"hello": "world"})
         db.test.save({"hello": "mike"})
         db.test.save({"hello": "world"})
-        self.assertFalse(db.error())
 
         db.drop_collection("test")
         db.test.create_index("hello", unique=True)
 
         db.test.save({"hello": "world"})
         db.test.save({"hello": "mike"})
-        db.test.save({"hello": "world"}, w=0)
-        self.assertTrue(db.error())
+
+        self.client.start_request()
+        try:
+            db.test.save({"hello": "world"}, w=0)
+            self.assertTrue(db.command('getlasterror').get('err'))
+        finally:
+            self.client.end_request()
 
     def test_duplicate_key_error(self):
         db = self.db
@@ -979,10 +972,16 @@ class TestCollection(unittest.TestCase):
         db.test.insert({"_id": 2, "x": 2})
 
         # No error
-        db.test.insert({"_id": 1, "x": 1}, safe=False)
-        db.test.save({"_id": 1, "x": 1}, safe=False)
-        db.test.insert({"_id": 2, "x": 2}, safe=False)
-        db.test.save({"_id": 2, "x": 2}, safe=False)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            db.test.insert({"_id": 1, "x": 1}, safe=False)
+            db.test.save({"_id": 1, "x": 1}, safe=False)
+            db.test.insert({"_id": 2, "x": 2}, safe=False)
+            db.test.save({"_id": 2, "x": 2}, safe=False)
+        finally:
+            ctx.exit()
+
         db.test.insert({"_id": 1, "x": 1}, w=0)
         db.test.save({"_id": 1, "x": 1}, w=0)
         db.test.insert({"_id": 2, "x": 2}, w=0)
@@ -1044,11 +1043,11 @@ class TestCollection(unittest.TestCase):
         docs.append({"five": 5})
 
         db.test.insert(docs, manipulate=False, w=0)
-        self.assertEqual(11000, db.error()['code'])
+        self.assertEqual(11000, db.command('getlasterror')['code'])
         self.assertEqual(1, db.test.count())
 
         db.test.insert(docs, manipulate=False, continue_on_error=True, w=0)
-        self.assertEqual(11000, db.error()['code'])
+        self.assertEqual(11000, db.command('getlasterror')['code'])
         self.assertEqual(4, db.test.count())
 
         db.drop_collection("test")
@@ -1058,11 +1057,11 @@ class TestCollection(unittest.TestCase):
         docs[2]["_id"] = oid
 
         db.test.insert(docs, manipulate=False, w=0)
-        self.assertEqual(11000, db.error()['code'])
+        self.assertEqual(11000, db.command('getlasterror')['code'])
         self.assertEqual(3, db.test.count())
 
         db.test.insert(docs, manipulate=False, continue_on_error=True, w=0)
-        self.assertEqual(11000, db.error()['code'])
+        self.assertEqual(11000, db.command('getlasterror')['code'])
         self.assertEqual(6, db.test.count())
 
     def test_error_code(self):
@@ -1100,7 +1099,7 @@ class TestCollection(unittest.TestCase):
         a = {"hello": "world"}
         db.test.insert(a)
         db.test.insert(a, w=0)
-        self.assertTrue("E11000" in db.error()["err"])
+        self.assertTrue("E11000" in db.command('getlasterror')["err"])
 
         self.assertRaises(OperationFailure, db.test.insert, a)
 
@@ -1206,11 +1205,11 @@ class TestCollection(unittest.TestCase):
             None, db.test.update({"_id": id}, {"$inc": {"x": 1}}, w=0))
 
         if v19:
-            self.assertTrue("E11000" in db.error()["err"])
+            self.assertTrue("E11000" in db.command('getlasterror')["err"])
         elif v113minus:
-            self.assertTrue(db.error()["err"].startswith("E11001"))
+            self.assertTrue(db.command('getlasterror')["err"].startswith("E11001"))
         else:
-            self.assertTrue(db.error()["err"].startswith("E12011"))
+            self.assertTrue(db.command('getlasterror')["err"].startswith("E12011"))
 
         self.assertRaises(OperationFailure, db.test.update,
                           {"_id": id}, {"$inc": {"x": 1}})
@@ -1271,7 +1270,7 @@ class TestCollection(unittest.TestCase):
 
         db.test.save({"hello": "world"})
         db.test.save({"hello": "world"}, w=0)
-        self.assertTrue("E11000" in db.error()["err"])
+        self.assertTrue("E11000" in db.command('getlasterror')["err"])
 
         self.assertRaises(OperationFailure, db.test.save,
                           {"hello": "world"})
@@ -1311,18 +1310,31 @@ class TestCollection(unittest.TestCase):
         ismaster = self.client.admin.command("ismaster")
         if ismaster.get("setName"):
             w = len(ismaster["hosts"]) + 1
-            self.assertRaises(WTimeoutError, self.db.test.save,
-                              {"x": 1}, w=w, wtimeout=1)
-            self.assertRaises(WTimeoutError, self.db.test.insert,
-                              {"x": 1}, w=w, wtimeout=1)
-            self.assertRaises(WTimeoutError, self.db.test.update,
-                              {"x": 1}, {"y": 2}, w=w, wtimeout=1)
-            self.assertRaises(WTimeoutError, self.db.test.remove,
-                              {"x": 1}, w=w, wtimeout=1)
+
+            # MongoDB 2.8+ raises error code 100, CannotSatisfyWriteConcern,
+            # if w > number of members. Older versions just time out after 1 ms
+            # as if they had enough secondaries but some are lagging. They
+            # return an error with 'wtimeout': True and no code.
+            def wtimeout_err(f, *args, **kwargs):
+                try:
+                    f(*args, **kwargs)
+                except WTimeoutError:
+                    pass
+                except OperationFailure, exc:
+                    self.assertTrue(exc.code == 100,
+                                    "Unexpected error: %r" % exc)
+                else:
+                    self.fail("%s should have failed" % f)
+
+            coll = self.db.test
+            wtimeout_err(coll.save, {"x": 1}, w=w, wtimeout=1)
+            wtimeout_err(coll.insert, {"x": 1}, w=w, wtimeout=1)
+            wtimeout_err(coll.remove, {"x": 1}, w=w, wtimeout=1)
+            wtimeout_err(coll.update, {"x": 1}, {"y": 2}, w=w, wtimeout=1)
 
             try:
                 self.db.test.save({"x": 1}, w=w, wtimeout=1)
-            except WTimeoutError, exc:
+            except OperationFailure, exc:
                 # Just check that we set the error document. Fields
                 # vary by MongoDB version.
                 self.assertTrue(exc.details is not None)
@@ -1330,8 +1342,9 @@ class TestCollection(unittest.TestCase):
                 self.fail("WTimeoutError was not raised")
 
         # can't use fsync and j options together
-        self.assertRaises(OperationFailure, self.db.test.insert,
-                          {"_id": 1}, j=True, fsync=True)
+        if version.at_least(self.client, (1, 8, 2)):
+            self.assertRaises(OperationFailure, self.db.test.insert,
+                              {"_id": 1}, j=True, fsync=True)
 
     def test_manual_last_error(self):
         self.db.test.save({"x": 1}, w=0)
@@ -1417,6 +1430,22 @@ class TestCollection(unittest.TestCase):
                 expected_sum,
                 sum(doc['_id'] for doc in cursor))
 
+    def test_aggregation_cursor_alive(self):
+        if not version.at_least(self.db.connection, (2, 5, 1)):
+            raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1")
+        self.db.test.remove()
+        self.db.test.insert([{} for _ in range(3)])
+        cursor = self.db.test.aggregate(pipeline=[], cursor={'batchSize': 2})
+        n = 0
+        while True:
+            cursor.next()
+            n += 1
+            if 3 == n:
+                self.assertFalse(cursor.alive)
+                break
+
+            self.assertTrue(cursor.alive)
+
     def test_parallel_scan(self):
         if is_mongos(self.db.connection):
             raise SkipTest("mongos does not support parallel_scan")
@@ -2064,8 +2093,17 @@ class TestCollection(unittest.TestCase):
     # (Shame on me)
     def test_bad_encode(self):
         c = self.db.test
+        c.drop()
         self.assertRaises(InvalidDocument, c.save, {"x": c})
 
+        class BadGetAttr(dict):
+            def __getattr__(self, name):
+                pass
+
+        bad = BadGetAttr([('foo', 'bar')])
+        c.insert({'bad': bad})
+        self.assertEqual('bar', c.find_one()['bad']['foo'])
+
     def test_bad_dbref(self):
         c = self.db.test
         c.drop()
@@ -2194,53 +2232,71 @@ class TestCollection(unittest.TestCase):
                                     as_class=ExtendedDict)
         self.assertTrue(isinstance(result, ExtendedDict))
 
+    def test_update_backward_compat(self):
+        # MongoDB versions >= 2.6.0 don't return the updatedExisting field
+        # and return upsert _id in an array subdocument. This test should
+        # pass regardless of server version or type (mongod/s).
+        c = self.db.test
+        c.drop()
+        oid = ObjectId()
+        res = c.update({'_id': oid}, {'$set': {'a': 'a'}}, upsert=True)
+        self.assertFalse(res.get('updatedExisting'))
+        self.assertEqual(oid, res.get('upserted'))
+
+        res = c.update({'_id': oid}, {'$set': {'b': 'b'}})
+        self.assertTrue(res.get('updatedExisting'))
+
     def test_find_and_modify_with_sort(self):
         c = self.db.test
         c.drop()
         for j in xrange(5):
             c.insert({'j': j, 'i': 0})
 
-        sort={'j': DESCENDING}
-        self.assertEqual(4, c.find_and_modify({},
-                                              {'$inc': {'i': 1}},
-                                              sort=sort)['j'])
-        sort={'j': ASCENDING}
-        self.assertEqual(0, c.find_and_modify({},
-                                              {'$inc': {'i': 1}},
-                                              sort=sort)['j'])
-        sort=[('j', DESCENDING)]
-        self.assertEqual(4, c.find_and_modify({},
-                                              {'$inc': {'i': 1}},
-                                              sort=sort)['j'])
-        sort=[('j', ASCENDING)]
-        self.assertEqual(0, c.find_and_modify({},
-                                              {'$inc': {'i': 1}},
-                                              sort=sort)['j'])
-        sort=SON([('j', DESCENDING)])
-        self.assertEqual(4, c.find_and_modify({},
-                                              {'$inc': {'i': 1}},
-                                              sort=sort)['j'])
-        sort=SON([('j', ASCENDING)])
-        self.assertEqual(0, c.find_and_modify({},
-                                              {'$inc': {'i': 1}},
-                                              sort=sort)['j'])
+        ctx = catch_warnings()
         try:
-            from collections import OrderedDict
-            sort=OrderedDict([('j', DESCENDING)])
+            warnings.simplefilter("ignore", DeprecationWarning)
+            sort={'j': DESCENDING}
             self.assertEqual(4, c.find_and_modify({},
                                                   {'$inc': {'i': 1}},
                                                   sort=sort)['j'])
-            sort=OrderedDict([('j', ASCENDING)])
+            sort={'j': ASCENDING}
             self.assertEqual(0, c.find_and_modify({},
                                                   {'$inc': {'i': 1}},
                                                   sort=sort)['j'])
-        except ImportError:
-            pass
-        # Test that a standard dict with two keys is rejected.
-        sort={'j': DESCENDING, 'foo': DESCENDING}
-        self.assertRaises(TypeError, c.find_and_modify, {},
-                                                         {'$inc': {'i': 1}},
-                                                         sort=sort)
+            sort=[('j', DESCENDING)]
+            self.assertEqual(4, c.find_and_modify({},
+                                                  {'$inc': {'i': 1}},
+                                                  sort=sort)['j'])
+            sort=[('j', ASCENDING)]
+            self.assertEqual(0, c.find_and_modify({},
+                                                  {'$inc': {'i': 1}},
+                                                  sort=sort)['j'])
+            sort=SON([('j', DESCENDING)])
+            self.assertEqual(4, c.find_and_modify({},
+                                                  {'$inc': {'i': 1}},
+                                                  sort=sort)['j'])
+            sort=SON([('j', ASCENDING)])
+            self.assertEqual(0, c.find_and_modify({},
+                                                  {'$inc': {'i': 1}},
+                                                  sort=sort)['j'])
+            try:
+                from collections import OrderedDict
+                sort=OrderedDict([('j', DESCENDING)])
+                self.assertEqual(4, c.find_and_modify({},
+                                                      {'$inc': {'i': 1}},
+                                                      sort=sort)['j'])
+                sort=OrderedDict([('j', ASCENDING)])
+                self.assertEqual(0, c.find_and_modify({},
+                                                      {'$inc': {'i': 1}},
+                                                      sort=sort)['j'])
+            except ImportError:
+                pass
+            # Test that a standard dict with two keys is rejected.
+            sort={'j': DESCENDING, 'foo': DESCENDING}
+            self.assertRaises(TypeError, c.find_and_modify,
+                              {}, {'$inc': {'i': 1}}, sort=sort)
+        finally:
+            ctx.exit()
 
     def test_find_with_nested(self):
         if not version.at_least(self.db.connection, (2, 0, 0)):
@@ -2327,6 +2383,43 @@ class TestCollection(unittest.TestCase):
         for doc in c.find(compile_re=False):
             self.assertTrue(isinstance(doc['r'], Regex))
 
+    def test_find_and_modify_with_manipulator(self):
+        class AddCollectionNameManipulator(SONManipulator):
+            def will_copy(self):
+                return True
+
+            def transform_incoming(self, son, collection):
+                copy = SON(son)
+                if 'collection' in copy:
+                    del copy['collection']
+                return copy
+
+            def transform_outgoing(self, son, collection):
+                copy = SON(son)
+                copy['collection'] = collection.name
+                return copy
+
+        self.db.add_son_manipulator(AddCollectionNameManipulator())
+
+        c = self.db.test
+        c.drop()
+        c.insert({'_id': 1, 'i': 1})
+
+        # Test correct findAndModify
+        # With manipulators
+        self.assertEqual({'_id': 1, 'i': 1, 'collection': 'test'},
+                         c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}},
+                                           manipulate=True))
+        self.assertEqual({'_id': 1, 'i': 3, 'collection': 'test'},
+                         c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}},
+                                           new=True, manipulate=True))
+        # With out manipulators
+        self.assertEqual({'_id': 1, 'i': 3},
+                         c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}))
+        self.assertEqual({'_id': 1, 'i': 5},
+                         c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}},
+                                           new=True))
+
 
 if __name__ == "__main__":
     unittest.main()

test/test_common.py

@@ -30,8 +30,8 @@ from pymongo.connection import Connection
 from pymongo.mongo_client import MongoClient
 from pymongo.mongo_replica_set_client import MongoReplicaSetClient
 from pymongo.errors import ConfigurationError, OperationFailure
-from test import host, port, pair, version
-from test.utils import drop_collections
+from test import host, port, pair, version, skip_restricted_localhost
+from test.utils import catch_warnings, drop_collections
 
 have_uuid = True
 try:
@@ -40,15 +40,16 @@ except ImportError:
     have_uuid = False
 
 
+setUpModule = skip_restricted_localhost
+
+
 class TestCommon(unittest.TestCase):
 
     def test_baseobject(self):
 
-        # In Python 2.6+ we could use the catch_warnings context
-        # manager to test this warning nicely. As we can't do that
-        # we must test raising errors before the ignore filter is applied.
-        warnings.simplefilter("error", UserWarning)
+        ctx = catch_warnings()
         try:
+            warnings.simplefilter("error", UserWarning)
             self.assertRaises(UserWarning, lambda:
                     MongoClient(host, port, wtimeout=1000, w=0))
             try:
@@ -61,191 +62,195 @@ class TestCommon(unittest.TestCase):
             except UserWarning:
                 self.fail()
         finally:
-            warnings.resetwarnings()
-            warnings.simplefilter("ignore")
+            ctx.exit()
 
         # Connection tests
-        c = Connection(pair)
-        self.assertFalse(c.slave_okay)
-        self.assertFalse(c.safe)
-        self.assertEqual({}, c.get_lasterror_options())
-        db = c.pymongo_test
-        db.drop_collection("test")
-        self.assertFalse(db.slave_okay)
-        self.assertFalse(db.safe)
-        self.assertEqual({}, db.get_lasterror_options())
-        coll = db.test
-        self.assertFalse(coll.slave_okay)
-        self.assertFalse(coll.safe)
-        self.assertEqual({}, coll.get_lasterror_options())
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            c = Connection(pair)
+            self.assertFalse(c.slave_okay)
+            self.assertFalse(c.safe)
+            self.assertEqual({}, c.get_lasterror_options())
+            db = c.pymongo_test
+            db.drop_collection("test")
+            self.assertFalse(db.slave_okay)
+            self.assertFalse(db.safe)
+            self.assertEqual({}, db.get_lasterror_options())
+            coll = db.test
+            self.assertFalse(coll.slave_okay)
+            self.assertFalse(coll.safe)
+            self.assertEqual({}, coll.get_lasterror_options())
 
-        self.assertEqual((False, {}), coll._get_write_mode())
-        coll.safe = False
-        coll.write_concern.update(w=1)
-        self.assertEqual((True, {}), coll._get_write_mode())
-        coll.write_concern.update(w=3)
-        self.assertEqual((True, {'w': 3}), coll._get_write_mode())
+            self.assertEqual((False, {}), coll._get_write_mode())
+            coll.safe = False
+            coll.write_concern.update(w=1)
+            self.assertEqual((True, {"w": 1}), coll._get_write_mode())
+            coll.write_concern.update(w=3)
+            self.assertEqual((True, {'w': 3}), coll._get_write_mode())
 
-        coll.safe = True
-        coll.write_concern.update(w=0)
-        self.assertEqual((False, {}), coll._get_write_mode())
+            coll.safe = True
+            coll.write_concern.update(w=0)
+            self.assertEqual((False, {}), coll._get_write_mode())
 
-        coll = db.test
-        cursor = coll.find()
-        self.assertFalse(cursor._Cursor__slave_okay)
-        cursor = coll.find(slave_okay=True)
-        self.assertTrue(cursor._Cursor__slave_okay)
+            coll = db.test
+            cursor = coll.find()
+            self.assertFalse(cursor._Cursor__slave_okay)
+            cursor = coll.find(slave_okay=True)
+            self.assertTrue(cursor._Cursor__slave_okay)
 
-        # MongoClient test
-        c = MongoClient(pair)
-        self.assertFalse(c.slave_okay)
-        self.assertTrue(c.safe)
-        self.assertEqual({}, c.get_lasterror_options())
-        db = c.pymongo_test
-        db.drop_collection("test")
-        self.assertFalse(db.slave_okay)
-        self.assertTrue(db.safe)
-        self.assertEqual({}, db.get_lasterror_options())
-        coll = db.test
-        self.assertFalse(coll.slave_okay)
-        self.assertTrue(coll.safe)
-        self.assertEqual({}, coll.get_lasterror_options())
+            # MongoClient test
+            c = MongoClient(pair)
+            self.assertFalse(c.slave_okay)
+            self.assertTrue(c.safe)
+            self.assertEqual({}, c.get_lasterror_options())
+            db = c.pymongo_test
+            db.drop_collection("test")
+            self.assertFalse(db.slave_okay)
+            self.assertTrue(db.safe)
+            self.assertEqual({}, db.get_lasterror_options())
+            coll = db.test
+            self.assertFalse(coll.slave_okay)
+            self.assertTrue(coll.safe)
+            self.assertEqual({}, coll.get_lasterror_options())
 
-        self.assertEqual((True, {}), coll._get_write_mode())
-        coll.safe = False
-        coll.write_concern.update(w=1)
-        self.assertEqual((True, {}), coll._get_write_mode())
-        coll.write_concern.update(w=3)
-        self.assertEqual((True, {'w': 3}), coll._get_write_mode())
+            self.assertEqual((True, {}), coll._get_write_mode())
+            coll.safe = False
+            coll.write_concern.update(w=1)
+            self.assertEqual((True, {"w": 1}), coll._get_write_mode())
+            coll.write_concern.update(w=3)
+            self.assertEqual((True, {'w': 3}), coll._get_write_mode())
 
-        coll.safe = True
-        coll.write_concern.update(w=0)
-        self.assertEqual((False, {}), coll._get_write_mode())
+            coll.safe = True
+            coll.write_concern.update(w=0)
+            self.assertEqual((False, {}), coll._get_write_mode())
 
-        coll = db.test
-        cursor = coll.find()
-        self.assertFalse(cursor._Cursor__slave_okay)
-        cursor = coll.find(slave_okay=True)
-        self.assertTrue(cursor._Cursor__slave_okay)
+            coll = db.test
+            cursor = coll.find()
+            self.assertFalse(cursor._Cursor__slave_okay)
+            cursor = coll.find(slave_okay=True)
+            self.assertTrue(cursor._Cursor__slave_okay)
 
-        # Setting any safe operations overrides explicit safe
-        self.assertTrue(MongoClient(host, port, wtimeout=1000, safe=False).safe)
+            # Setting any safe operations overrides explicit safe
+            self.assertTrue(MongoClient(host, port, wtimeout=1000, safe=False).safe)
 
-        c = MongoClient(pair, slaveok=True, w='majority',
-                        wtimeout=300, fsync=True, j=True)
-        self.assertTrue(c.slave_okay)
-        self.assertTrue(c.safe)
-        d = {'w': 'majority', 'wtimeout': 300, 'fsync': True, 'j': True}
-        self.assertEqual(d, c.get_lasterror_options())
-        db = c.pymongo_test
-        self.assertTrue(db.slave_okay)
-        self.assertTrue(db.safe)
-        self.assertEqual(d, db.get_lasterror_options())
-        coll = db.test
-        self.assertTrue(coll.slave_okay)
-        self.assertTrue(coll.safe)
-        self.assertEqual(d, coll.get_lasterror_options())
-        cursor = coll.find()
-        self.assertTrue(cursor._Cursor__slave_okay)
-        cursor = coll.find(slave_okay=False)
-        self.assertFalse(cursor._Cursor__slave_okay)
+            c = MongoClient(pair, slaveok=True, w='majority',
+                            wtimeout=300, fsync=True, j=True)
+            self.assertTrue(c.slave_okay)
+            self.assertTrue(c.safe)
+            d = {'w': 'majority', 'wtimeout': 300, 'fsync': True, 'j': True}
+            self.assertEqual(d, c.get_lasterror_options())
+            db = c.pymongo_test
+            self.assertTrue(db.slave_okay)
+            self.assertTrue(db.safe)
+            self.assertEqual(d, db.get_lasterror_options())
+            coll = db.test
+            self.assertTrue(coll.slave_okay)
+            self.assertTrue(coll.safe)
+            self.assertEqual(d, coll.get_lasterror_options())
+            cursor = coll.find()
+            self.assertTrue(cursor._Cursor__slave_okay)
+            cursor = coll.find(slave_okay=False)
+            self.assertFalse(cursor._Cursor__slave_okay)
 
-        c = MongoClient('mongodb://%s/?'
-                       'w=2;wtimeoutMS=300;fsync=true;'
-                       'journal=true' % (pair,))
-        self.assertTrue(c.safe)
-        d = {'w': 2, 'wtimeout': 300, 'fsync': True, 'j': True}
-        self.assertEqual(d, c.get_lasterror_options())
+            c = MongoClient('mongodb://%s/?'
+                           'w=2;wtimeoutMS=300;fsync=true;'
+                           'journal=true' % (pair,))
+            self.assertTrue(c.safe)
+            d = {'w': 2, 'wtimeout': 300, 'fsync': True, 'j': True}
+            self.assertEqual(d, c.get_lasterror_options())
 
-        c = MongoClient('mongodb://%s/?'
-                       'slaveok=true;w=1;wtimeout=300;'
-                       'fsync=true;j=true' % (pair,))
-        self.assertTrue(c.slave_okay)
-        self.assertTrue(c.safe)
-        d = {'w': 1, 'wtimeout': 300, 'fsync': True, 'j': True}
-        self.assertEqual(d, c.get_lasterror_options())
-        self.assertEqual(d, c.write_concern)
-        db = c.pymongo_test
-        self.assertTrue(db.slave_okay)
-        self.assertTrue(db.safe)
-        self.assertEqual(d, db.get_lasterror_options())
-        self.assertEqual(d, db.write_concern)
-        coll = db.test
-        self.assertTrue(coll.slave_okay)
-        self.assertTrue(coll.safe)
-        self.assertEqual(d, coll.get_lasterror_options())
-        self.assertEqual(d, coll.write_concern)
-        cursor = coll.find()
-        self.assertTrue(cursor._Cursor__slave_okay)
-        cursor = coll.find(slave_okay=False)
-        self.assertFalse(cursor._Cursor__slave_okay)
+            c = MongoClient('mongodb://%s/?'
+                           'slaveok=true;w=1;wtimeout=300;'
+                           'fsync=true;j=true' % (pair,))
+            self.assertTrue(c.slave_okay)
+            self.assertTrue(c.safe)
+            d = {'w': 1, 'wtimeout': 300, 'fsync': True, 'j': True}
+            self.assertEqual(d, c.get_lasterror_options())
+            self.assertEqual(d, c.write_concern)
+            db = c.pymongo_test
+            self.assertTrue(db.slave_okay)
+            self.assertTrue(db.safe)
+            self.assertEqual(d, db.get_lasterror_options())
+            self.assertEqual(d, db.write_concern)
+            coll = db.test
+            self.assertTrue(coll.slave_okay)
+            self.assertTrue(coll.safe)
+            self.assertEqual(d, coll.get_lasterror_options())
+            self.assertEqual(d, coll.write_concern)
+            cursor = coll.find()
+            self.assertTrue(cursor._Cursor__slave_okay)
+            cursor = coll.find(slave_okay=False)
+            self.assertFalse(cursor._Cursor__slave_okay)
 
-        c.unset_lasterror_options()
-        self.assertTrue(c.slave_okay)
-        self.assertTrue(c.safe)
-        c.safe = False
-        self.assertFalse(c.safe)
-        c.slave_okay = False
-        self.assertFalse(c.slave_okay)
-        self.assertEqual({}, c.get_lasterror_options())
-        self.assertEqual({}, c.write_concern)
-        db = c.pymongo_test
-        self.assertFalse(db.slave_okay)
-        self.assertFalse(db.safe)
-        self.assertEqual({}, db.get_lasterror_options())
-        self.assertEqual({}, db.write_concern)
-        coll = db.test
-        self.assertFalse(coll.slave_okay)
-        self.assertFalse(coll.safe)
-        self.assertEqual({}, coll.get_lasterror_options())
-        self.assertEqual({}, coll.write_concern)
-        cursor = coll.find()
-        self.assertFalse(cursor._Cursor__slave_okay)
-        cursor = coll.find(slave_okay=True)
-        self.assertTrue(cursor._Cursor__slave_okay)
+            c.unset_lasterror_options()
+            self.assertTrue(c.slave_okay)
+            self.assertTrue(c.safe)
+            c.safe = False
+            self.assertFalse(c.safe)
+            c.slave_okay = False
+            self.assertFalse(c.slave_okay)
+            self.assertEqual({}, c.get_lasterror_options())
+            self.assertEqual({}, c.write_concern)
+            db = c.pymongo_test
+            self.assertFalse(db.slave_okay)
+            self.assertFalse(db.safe)
+            self.assertEqual({}, db.get_lasterror_options())
+            self.assertEqual({}, db.write_concern)
+            coll = db.test
+            self.assertFalse(coll.slave_okay)
+            self.assertFalse(coll.safe)
+            self.assertEqual({}, coll.get_lasterror_options())
+            self.assertEqual({}, coll.write_concern)
+            cursor = coll.find()
+            self.assertFalse(cursor._Cursor__slave_okay)
+            cursor = coll.find(slave_okay=True)
+            self.assertTrue(cursor._Cursor__slave_okay)
 
-        coll.set_lasterror_options(fsync=True)
-        self.assertEqual({'fsync': True}, coll.get_lasterror_options())
-        self.assertEqual({'fsync': True}, coll.write_concern)
-        self.assertEqual({}, db.get_lasterror_options())
-        self.assertEqual({}, db.write_concern)
-        self.assertFalse(db.safe)
-        self.assertEqual({}, c.get_lasterror_options())
-        self.assertEqual({}, c.write_concern)
-        self.assertFalse(c.safe)
+            coll.set_lasterror_options(fsync=True)
+            self.assertEqual({'fsync': True}, coll.get_lasterror_options())
+            self.assertEqual({'fsync': True}, coll.write_concern)
+            self.assertEqual({}, db.get_lasterror_options())
+            self.assertEqual({}, db.write_concern)
+            self.assertFalse(db.safe)
+            self.assertEqual({}, c.get_lasterror_options())
+            self.assertEqual({}, c.write_concern)
+            self.assertFalse(c.safe)
 
-        db.set_lasterror_options(w='majority')
-        self.assertEqual({'fsync': True}, coll.get_lasterror_options())
-        self.assertEqual({'fsync': True}, coll.write_concern)
-        self.assertEqual({'w': 'majority'}, db.get_lasterror_options())
-        self.assertEqual({'w': 'majority'}, db.write_concern)
-        self.assertEqual({}, c.get_lasterror_options())
-        self.assertEqual({}, c.write_concern)
-        self.assertFalse(c.safe)
-        db.slave_okay = True
-        self.assertTrue(db.slave_okay)
-        self.assertFalse(c.slave_okay)
-        self.assertFalse(coll.slave_okay)
-        cursor = coll.find()
-        self.assertFalse(cursor._Cursor__slave_okay)
-        cursor = db.coll2.find()
-        self.assertTrue(cursor._Cursor__slave_okay)
-        cursor = db.coll2.find(slave_okay=False)
-        self.assertFalse(cursor._Cursor__slave_okay)
+            db.set_lasterror_options(w='majority')
+            self.assertEqual({'fsync': True}, coll.get_lasterror_options())
+            self.assertEqual({'fsync': True}, coll.write_concern)
+            self.assertEqual({'w': 'majority'}, db.get_lasterror_options())
+            self.assertEqual({'w': 'majority'}, db.write_concern)
+            self.assertEqual({}, c.get_lasterror_options())
+            self.assertEqual({}, c.write_concern)
+            self.assertFalse(c.safe)
+            db.slave_okay = True
+            self.assertTrue(db.slave_okay)
+            self.assertFalse(c.slave_okay)
+            self.assertFalse(coll.slave_okay)
+            cursor = coll.find()
+            self.assertFalse(cursor._Cursor__slave_okay)
+            cursor = db.coll2.find()
+            self.assertTrue(cursor._Cursor__slave_okay)
+            cursor = db.coll2.find(slave_okay=False)
+            self.assertFalse(cursor._Cursor__slave_okay)
 
-        self.assertRaises(ConfigurationError, coll.set_lasterror_options, foo=20)
-        self.assertRaises(TypeError, coll._BaseObject__set_slave_okay, 20)
-        self.assertRaises(TypeError, coll._BaseObject__set_safe, 20)
+            self.assertRaises(ConfigurationError, coll.set_lasterror_options, foo=20)
+            self.assertRaises(TypeError, coll._BaseObject__set_slave_okay, 20)
+            self.assertRaises(TypeError, coll._BaseObject__set_safe, 20)
 
-        coll.remove()
-        self.assertEqual(None, coll.find_one(slave_okay=True))
-        coll.unset_lasterror_options()
-        coll.set_lasterror_options(w=4, wtimeout=10)
-        # Fails if we don't have 4 active nodes or we don't have replication...
-        self.assertRaises(OperationFailure, coll.insert, {'foo': 'bar'})
-        # Succeeds since we override the lasterror settings per query.
-        self.assertTrue(coll.insert({'foo': 'bar'}, fsync=True))
-        drop_collections(db)
+            coll.remove()
+            self.assertEqual(None, coll.find_one(slave_okay=True))
+            coll.unset_lasterror_options()
+            coll.set_lasterror_options(w=4, wtimeout=10)
+            # Fails if we don't have 4 active nodes or we don't have replication...
+            self.assertRaises(OperationFailure, coll.insert, {'foo': 'bar'})
+            # Succeeds since we override the lasterror settings per query.
+            self.assertTrue(coll.insert({'foo': 'bar'}, fsync=True))
+            drop_collections(db)
+        finally:
+            ctx.exit()
 
     def test_uuid_subtype(self):
         if not have_uuid:
@@ -439,30 +444,36 @@ class TestCommon(unittest.TestCase):
         m = MongoClient(pair, w=0)
         coll = m.pymongo_test.write_concern_test
         coll.drop()
-        doc = {"_id": ObjectId()}
-        coll.insert(doc)
-        self.assertTrue(coll.insert(doc, safe=False))
-        self.assertTrue(coll.insert(doc, w=0))
-        self.assertTrue(coll.insert(doc))
-        self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
-        self.assertRaises(OperationFailure, coll.insert, doc, w=1)
 
-        m = MongoClient(pair)
-        coll = m.pymongo_test.write_concern_test
-        self.assertTrue(coll.insert(doc, safe=False))
-        self.assertTrue(coll.insert(doc, w=0))
-        self.assertRaises(OperationFailure, coll.insert, doc)
-        self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
-        self.assertRaises(OperationFailure, coll.insert, doc, w=1)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            doc = {"_id": ObjectId()}
+            coll.insert(doc)
+            self.assertTrue(coll.insert(doc, safe=False))
+            self.assertTrue(coll.insert(doc, w=0))
+            self.assertTrue(coll.insert(doc))
+            self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
+            self.assertRaises(OperationFailure, coll.insert, doc, w=1)
 
-        m = MongoClient("mongodb://%s/" % (pair,))
-        self.assertTrue(m.safe)
-        coll = m.pymongo_test.write_concern_test
-        self.assertRaises(OperationFailure, coll.insert, doc)
-        m = MongoClient("mongodb://%s/?w=0" % (pair,))
-        self.assertFalse(m.safe)
-        coll = m.pymongo_test.write_concern_test
-        self.assertTrue(coll.insert(doc))
+            m = MongoClient(pair)
+            coll = m.pymongo_test.write_concern_test
+            self.assertTrue(coll.insert(doc, safe=False))
+            self.assertTrue(coll.insert(doc, w=0))
+            self.assertRaises(OperationFailure, coll.insert, doc)
+            self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
+            self.assertRaises(OperationFailure, coll.insert, doc, w=1)
+
+            m = MongoClient("mongodb://%s/" % (pair,))
+            self.assertTrue(m.safe)
+            coll = m.pymongo_test.write_concern_test
+            self.assertRaises(OperationFailure, coll.insert, doc)
+            m = MongoClient("mongodb://%s/?w=0" % (pair,))
+            self.assertFalse(m.safe)
+            coll = m.pymongo_test.write_concern_test
+            self.assertTrue(coll.insert(doc))
+        finally:
+            ctx.exit()
 
         # Equality tests
         self.assertEqual(m, MongoClient("mongodb://%s/?w=0" % (pair,)))
@@ -478,30 +489,36 @@ class TestCommon(unittest.TestCase):
         m = MongoReplicaSetClient(pair, replicaSet=setname, w=0)
         coll = m.pymongo_test.write_concern_test
         coll.drop()
-        doc = {"_id": ObjectId()}
-        coll.insert(doc)
-        self.assertTrue(coll.insert(doc, safe=False))
-        self.assertTrue(coll.insert(doc, w=0))
-        self.assertTrue(coll.insert(doc))
-        self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
-        self.assertRaises(OperationFailure, coll.insert, doc, w=1)
 
-        m = MongoReplicaSetClient(pair, replicaSet=setname)
-        coll = m.pymongo_test.write_concern_test
-        self.assertTrue(coll.insert(doc, safe=False))
-        self.assertTrue(coll.insert(doc, w=0))
-        self.assertRaises(OperationFailure, coll.insert, doc)
-        self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
-        self.assertRaises(OperationFailure, coll.insert, doc, w=1)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            doc = {"_id": ObjectId()}
+            coll.insert(doc)
+            self.assertTrue(coll.insert(doc, safe=False))
+            self.assertTrue(coll.insert(doc, w=0))
+            self.assertTrue(coll.insert(doc))
+            self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
+            self.assertRaises(OperationFailure, coll.insert, doc, w=1)
 
-        m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname))
-        self.assertTrue(m.safe)
-        coll = m.pymongo_test.write_concern_test
-        self.assertRaises(OperationFailure, coll.insert, doc)
-        m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname))
-        self.assertFalse(m.safe)
-        coll = m.pymongo_test.write_concern_test
-        self.assertTrue(coll.insert(doc))
+            m = MongoReplicaSetClient(pair, replicaSet=setname)
+            coll = m.pymongo_test.write_concern_test
+            self.assertTrue(coll.insert(doc, safe=False))
+            self.assertTrue(coll.insert(doc, w=0))
+            self.assertRaises(OperationFailure, coll.insert, doc)
+            self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
+            self.assertRaises(OperationFailure, coll.insert, doc, w=1)
+
+            m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname))
+            self.assertTrue(m.safe)
+            coll = m.pymongo_test.write_concern_test
+            self.assertRaises(OperationFailure, coll.insert, doc)
+            m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname))
+            self.assertFalse(m.safe)
+            coll = m.pymongo_test.write_concern_test
+            self.assertTrue(coll.insert(doc))
+        finally:
+            ctx.exit()
 
 
 if __name__ == "__main__":

test/test_cursor.py

@@ -19,6 +19,8 @@ import random
 import re
 import sys
 import unittest
+import warnings
+
 sys.path[0:0] = [""]
 
 from nose.plugins.skip import SkipTest
@@ -35,9 +37,13 @@ from pymongo.database import Database
 from pymongo.errors import (InvalidOperation,
                             OperationFailure,
                             ExecutionTimeout)
-from test import version
+from test import version, skip_restricted_localhost
 from test.test_client import get_client
-from test.utils import is_mongos, get_command_line, server_started_with_auth
+from test.utils import (catch_warnings, is_mongos,
+                        get_command_line, server_started_with_auth)
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestCursor(unittest.TestCase):
@@ -128,16 +134,12 @@ class TestCursor(unittest.TestCase):
 
     def test_explain(self):
         a = self.db.test.find()
-        b = a.explain()
+        a.explain()
         for _ in a:
             break
-        c = a.explain()
-        del b["millis"]
-        b.pop("oldPlan", None)
-        del c["millis"]
-        c.pop("oldPlan", None)
-        self.assertEqual(b, c)
-        self.assertTrue("cursor" in b)
+        b = a.explain()
+        # "cursor" pre MongoDB 2.7.6, "executionStats" post
+        self.assertTrue("cursor" in b or "executionStats" in b)
 
     def test_hint(self):
         db = self.db
@@ -154,15 +156,13 @@ class TestCursor(unittest.TestCase):
                           db.test.find({"num": 17, "foo": 17})
                           .hint([("foo", ASCENDING)]).explain)
 
-        index = db.test.create_index("num")
+        spec = [("num", DESCENDING)]
+        index = db.test.create_index(spec)
 
-        spec = [("num", ASCENDING)]
-        self.assertEqual(db.test.find({}).explain()["cursor"], "BasicCursor")
-        self.assertEqual(db.test.find({}).hint(spec).explain()["cursor"],
-                         "BtreeCursor %s" % index)
-        self.assertEqual(db.test.find({}).hint(spec).hint(None)
-                         .explain()["cursor"],
-                         "BasicCursor")
+        first = db.test.find().next()
+        self.assertEqual(0, first.get('num'))
+        first = db.test.find().hint(spec).next()
+        self.assertEqual(99, first.get('num'))
         self.assertRaises(OperationFailure,
                           db.test.find({"num": 17, "foo": 17})
                           .hint([("foo", ASCENDING)]).explain)
@@ -173,7 +173,18 @@ class TestCursor(unittest.TestCase):
             break
         self.assertRaises(InvalidOperation, a.hint, spec)
 
-        self.assertRaises(TypeError, db.test.find().hint, index)
+    def test_hint_by_name(self):
+        db = self.db
+        db.test.drop()
+
+        for i in range(100):
+            db.test.insert({'i': i})
+
+        db.test.create_index([('i', DESCENDING)], name='fooindex')
+        first = db.test.find().next()
+        self.assertEqual(0, first.get('i'))
+        first = db.test.find().hint('fooindex').next()
+        self.assertEqual(99, first.get('i'))
 
     def test_limit(self):
         db = self.db
@@ -495,6 +506,40 @@ class TestCursor(unittest.TestCase):
 
         self.assertEqual(0, db.test.acollectionthatdoesntexist.find().count())
 
+    def test_count_with_hint(self):
+        collection = self.db.test
+        collection.drop()
+
+        collection.save({'i': 1})
+        collection.save({'i': 2})
+        self.assertEqual(2, collection.find().count())
+
+        collection.create_index([('i', 1)])
+
+        self.assertEqual(1, collection.find({'i': 1}).hint("_id_").count())
+        self.assertEqual(2, collection.find().hint("_id_").count())
+
+        if version.at_least(self.client, (2, 6, 0)):
+            # Count supports hint
+            self.assertRaises(OperationFailure,
+                              collection.find({'i': 1}).hint("BAD HINT").count)
+        else:
+            # Hint is ignored
+            self.assertEqual(
+                1, collection.find({'i': 1}).hint("BAD HINT").count())
+
+        # Create a sparse index which should have no entries.
+        collection.create_index([('x', 1)], sparse=True)
+
+        if version.at_least(self.client, (2, 6, 0)):
+            # Count supports hint
+            self.assertEqual(0, collection.find({'i': 1}).hint("x_1").count())
+        else:
+            # Hint is ignored
+            self.assertEqual(1, collection.find({'i': 1}).hint("x_1").count())
+
+        self.assertEqual(2, collection.find().hint("x_1").count())
+
     def test_where(self):
         db = self.db
         db.test.drop()
@@ -1100,8 +1145,11 @@ self.assertFalse(c2.alive)
                 pass
 
         client = self.db.connection
+        ctx = catch_warnings()
         try:
+            warnings.simplefilter("ignore", DeprecationWarning)
             client.set_cursor_manager(CManager)
+
             docs = []
             cursor = self.db.test.find().batch_size(10)
             docs.append(cursor.next())
@@ -1115,6 +1163,21 @@ self.assertFalse(c2.alive)
             self.assertEqual(len(docs), 200)
         finally:
             client.set_cursor_manager(CursorManager)
+            ctx.exit()
+
+    def test_alive(self):
+        self.db.test.remove()
+        self.db.test.insert([{} for _ in range(3)])
+        cursor = self.db.test.find().batch_size(2)
+        n = 0
+        while True:
+            cursor.next()
+            n += 1
+            if 3 == n:
+                self.assertFalse(cursor.alive)
+                break
+
+            self.assertTrue(cursor.alive)
 
 if __name__ == "__main__":
     unittest.main()

test/test_database.py

@@ -15,7 +15,6 @@
 """Test the database module."""
 
 import datetime
-import os
 import re
 import sys
 import warnings
@@ -39,19 +38,22 @@ from pymongo import (ALL,
 from pymongo.collection import Collection
 from pymongo.database import Database
 from pymongo.errors import (CollectionInvalid,
-                            ConfigurationError,
                             ExecutionTimeout,
                             InvalidName,
                             OperationFailure)
 from pymongo.son_manipulator import (AutoReference,
                                      NamespaceInjector,
+                                     SONManipulator,
                                      ObjectIdShuffler)
-from test import version
-from test.utils import (get_command_line, is_mongos,
-                        remove_all_users, server_started_with_auth)
+from test import version, skip_restricted_localhost
+from test.utils import (catch_warnings, get_command_line,
+                        is_mongos, server_started_with_auth)
 from test.test_client import get_client
 
 
+setUpModule = skip_restricted_localhost
+
+
 class TestDatabase(unittest.TestCase):
 
     def setUp(self):
@@ -103,14 +105,13 @@ class TestDatabase(unittest.TestCase):
         self.assertRaises(InvalidName, db.create_collection, "coll..ection")
 
         test = db.create_collection("test")
+        self.assertTrue(u"test" in db.collection_names())
         test.save({"hello": u"world"})
         self.assertEqual(db.test.find_one()["hello"], "world")
-        self.assertTrue(u"test" in db.collection_names())
 
         db.drop_collection("test.foo")
         db.create_collection("test.foo")
         self.assertTrue(u"test.foo" in db.collection_names())
-        self.assertEqual(db.test.foo.options(), {})
         self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
 
     def test_collection_names(self):
@@ -252,36 +253,40 @@ class TestDatabase(unittest.TestCase):
         if is_mongos(self.client):
             raise SkipTest('getpreverror not supported by mongos')
         db = self.client.pymongo_test
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            db.reset_error_history()
+            self.assertEqual(None, db.error())
+            self.assertEqual(None, db.previous_error())
 
-        db.reset_error_history()
-        self.assertEqual(None, db.error())
-        self.assertEqual(None, db.previous_error())
+            db.command("forceerror", check=False)
+            self.assertTrue(db.error())
+            self.assertTrue(db.previous_error())
 
-        db.command("forceerror", check=False)
-        self.assertTrue(db.error())
-        self.assertTrue(db.previous_error())
+            db.command("forceerror", check=False)
+            self.assertTrue(db.error())
+            prev_error = db.previous_error()
+            self.assertEqual(prev_error["nPrev"], 1)
+            del prev_error["nPrev"]
+            prev_error.pop("lastOp", None)
+            error = db.error()
+            error.pop("lastOp", None)
+            # getLastError includes "connectionId" in recent
+            # server versions, getPrevError does not.
+            error.pop("connectionId", None)
+            self.assertEqual(error, prev_error)
 
-        db.command("forceerror", check=False)
-        self.assertTrue(db.error())
-        prev_error = db.previous_error()
-        self.assertEqual(prev_error["nPrev"], 1)
-        del prev_error["nPrev"]
-        prev_error.pop("lastOp", None)
-        error = db.error()
-        error.pop("lastOp", None)
-        # getLastError includes "connectionId" in recent
-        # server versions, getPrevError does not.
-        error.pop("connectionId", None)
-        self.assertEqual(error, prev_error)
+            db.test.find_one()
+            self.assertEqual(None, db.error())
+            self.assertTrue(db.previous_error())
+            self.assertEqual(db.previous_error()["nPrev"], 2)
 
-        db.test.find_one()
-        self.assertEqual(None, db.error())
-        self.assertTrue(db.previous_error())
-        self.assertEqual(db.previous_error()["nPrev"], 2)
-
-        db.reset_error_history()
-        self.assertEqual(None, db.error())
-        self.assertEqual(None, db.previous_error())
+            db.reset_error_history()
+            self.assertEqual(None, db.error())
+            self.assertEqual(None, db.previous_error())
+        finally:
+            ctx.exit()
 
     def test_command(self):
         db = self.client.admin
@@ -330,11 +335,16 @@ class TestDatabase(unittest.TestCase):
         db.test.remove({})
         db.test.save({"i": 1})
 
-        db.test.update({"i": 1}, {"$set": {"i": 2}}, w=0)
-        self.assertTrue(db.last_status()["updatedExisting"])
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            db.test.update({"i": 1}, {"$set": {"i": 2}}, w=0)
+            self.assertTrue(db.last_status()["updatedExisting"])
 
-        db.test.update({"i": 1}, {"$set": {"i": 500}}, w=0)
-        self.assertFalse(db.last_status()["updatedExisting"])
+            db.test.update({"i": 1}, {"$set": {"i": 500}}, w=0)
+            self.assertFalse(db.last_status()["updatedExisting"])
+        finally:
+            ctx.exit()
 
     def test_password_digest(self):
         self.assertRaises(TypeError, auth._password_digest, 5)
@@ -350,319 +360,6 @@ class TestDatabase(unittest.TestCase):
         self.assertEqual(auth._password_digest("Gustave", u"Dor\xe9"),
                          u"81e0e2364499209f466e75926a162d73")
 
-    def test_authenticate_add_remove_user(self):
-        if (is_mongos(self.client) and not
-            version.at_least(self.client, (2, 0, 0))):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-
-        db = self.client.pymongo_test
-
-        # Configuration errors
-        self.assertRaises(ValueError, db.add_user, "user", '')
-        self.assertRaises(TypeError, db.add_user, "user", 'password', 15)
-        self.assertRaises(ConfigurationError, db.add_user,
-                          "user", 'password', 'True')
-        self.assertRaises(ConfigurationError, db.add_user,
-                          "user", 'password', True, roles=['read'])
-
-        if version.at_least(self.client, (2, 5, 3, -1)):
-            warnings.simplefilter("error", DeprecationWarning)
-            try:
-                self.assertRaises(DeprecationWarning, db.add_user,
-                                  "user", "password")
-                self.assertRaises(DeprecationWarning, db.add_user,
-                                  "user", "password", True)
-            finally:
-                warnings.resetwarnings()
-                warnings.simplefilter("ignore")
-
-            self.assertRaises(ConfigurationError, db.add_user,
-                              "user", "password", digestPassword=True)
-
-        self.client.admin.add_user("admin", "password")
-        self.client.admin.authenticate("admin", "password")
-
-        try:
-            # Add / authenticate / remove
-            db.add_user("mike", "password")
-            self.assertRaises(TypeError, db.authenticate, 5, "password")
-            self.assertRaises(TypeError, db.authenticate, "mike", 5)
-            self.assertRaises(OperationFailure,
-                              db.authenticate, "mike", "not a real password")
-            self.assertRaises(OperationFailure,
-                              db.authenticate, "faker", "password")
-            self.assertTrue(db.authenticate("mike", "password"))
-            db.logout()
-            self.assertTrue(db.authenticate(u"mike", u"password"))
-            db.remove_user("mike")
-            db.logout()
-
-            self.assertRaises(OperationFailure,
-                              db.authenticate, "mike", "password")
-
-            # Add / authenticate / change password
-            self.assertRaises(OperationFailure,
-                              db.authenticate, "Gustave", u"Dor\xe9")
-            db.add_user("Gustave", u"Dor\xe9")
-            self.assertTrue(db.authenticate("Gustave", u"Dor\xe9"))
-            db.add_user("Gustave", "password")
-            db.logout()
-            self.assertRaises(OperationFailure,
-                              db.authenticate, "Gustave", u"Dor\xe9")
-            self.assertTrue(db.authenticate("Gustave", u"password"))
-
-            if not version.at_least(self.client, (2, 5, 3, -1)):
-                # Add a readOnly user
-                db.add_user("Ross", "password", read_only=True)
-                db.logout()
-                self.assertTrue(db.authenticate("Ross", u"password"))
-                self.assertTrue(db.system.users.find({"readOnly": True}).count())
-                db.logout()
-
-        # Cleanup
-        finally:
-            remove_all_users(db)
-            self.client.admin.remove_user("admin")
-            self.client.admin.logout()
-
-    def test_make_user_readonly(self):
-        if (is_mongos(self.client)
-                and not version.at_least(self.client, (2, 0, 0))):
-            raise SkipTest('Auth with sharding requires MongoDB >= 2.0.0')
-
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-
-        admin = self.client.admin
-        admin.add_user('admin', 'pw')
-        admin.authenticate('admin', 'pw')
-
-        db = self.client.pymongo_test
-
-        try:
-            # Make a read-write user.
-            db.add_user('jesse', 'pw')
-            admin.logout()
-
-            # Check that we're read-write by default.
-            db.authenticate('jesse', 'pw')
-            db.collection.insert({})
-            db.logout()
-
-            # Make the user read-only.
-            admin.authenticate('admin', 'pw')
-            db.add_user('jesse', 'pw', read_only=True)
-            admin.logout()
-
-            db.authenticate('jesse', 'pw')
-            self.assertRaises(OperationFailure, db.collection.insert, {})
-        finally:
-            # Cleanup
-            admin.authenticate('admin', 'pw')
-            remove_all_users(db)
-            admin.remove_user("admin")
-            admin.logout()
-
-    def test_default_roles(self):
-        if not version.at_least(self.client, (2, 5, 3, -1)):
-            raise SkipTest("Default roles only exist in MongoDB >= 2.5.3")
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-
-        # "Admin" user
-        db = self.client.admin
-        db.add_user('admin', 'pass')
-        try:
-            db.authenticate('admin', 'pass')
-            info = db.command('usersInfo', 'admin')['users'][0]
-            self.assertEqual("root", info['roles'][0]['role'])
-
-            # Read only "admin" user
-            db.add_user('ro-admin', 'pass', read_only=True)
-            db.logout()
-            db.authenticate('ro-admin', 'pass')
-            info = db.command('usersInfo', 'ro-admin')['users'][0]
-            self.assertEqual("readAnyDatabase", info['roles'][0]['role'])
-            db.logout()
-
-        # Cleanup
-        finally:
-            db.authenticate('admin', 'pass')
-            remove_all_users(db)
-            db.logout()
-
-        db.connection.disconnect()
-
-        # "Non-admin" user
-        db = self.client.pymongo_test
-        db.add_user('user', 'pass')
-        try:
-            db.authenticate('user', 'pass')
-            info = db.command('usersInfo', 'user')['users'][0]
-            self.assertEqual("dbOwner", info['roles'][0]['role'])
-
-            # Read only "Non-admin" user
-            db.add_user('ro-user', 'pass', read_only=True)
-            db.logout()
-            db.authenticate('ro-user', 'pass')
-            info = db.command('usersInfo', 'ro-user')['users'][0]
-            self.assertEqual("read", info['roles'][0]['role'])
-            db.logout()
-
-        # Cleanup
-        finally:
-            db.authenticate('user', 'pass')
-            remove_all_users(db)
-            db.logout()
-
-    def test_new_user_cmds(self):
-        if not version.at_least(self.client, (2, 5, 3, -1)):
-            raise SkipTest("User manipulation through commands "
-                           "requires MongoDB >= 2.5.3")
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-
-        db = self.client.pymongo_test
-        db.add_user("amalia", "password", roles=["userAdmin"])
-        db.authenticate("amalia", "password")
-        try:
-            # This tests the ability to update user attributes.
-            db.add_user("amalia", "new_password",
-                        customData={"secret": "koalas"})
-
-            user_info = db.command("usersInfo", "amalia")
-            self.assertTrue(user_info["users"])
-            amalia_user = user_info["users"][0]
-            self.assertEqual(amalia_user["user"], "amalia")
-            self.assertEqual(amalia_user["customData"], {"secret": "koalas"})
-        finally:
-            db.remove_user("amalia")
-            db.logout()
-
-    def test_authenticate_and_safe(self):
-        if (is_mongos(self.client) and not
-            version.at_least(self.client, (2, 0, 0))):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-        db = self.client.auth_test
-
-        db.add_user("bernie", "password",
-                    roles=["userAdmin", "dbAdmin", "readWrite"])
-        db.authenticate("bernie", "password")
-        try:
-            db.test.remove({})
-            self.assertTrue(db.test.insert({"bim": "baz"}))
-            self.assertEqual(1, db.test.count())
-
-            self.assertEqual(1,
-                             db.test.update({"bim": "baz"},
-                                            {"$set": {"bim": "bar"}}).get('n'))
-
-            self.assertEqual(1,
-                             db.test.remove({}).get('n'))
-
-            self.assertEqual(0, db.test.count())
-        finally:
-            db.remove_user("bernie")
-            db.logout()
-
-    def test_authenticate_and_request(self):
-        if (is_mongos(self.client) and not
-            version.at_least(self.client, (2, 0, 0))):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-        if not server_started_with_auth(self.client):
-            raise SkipTest('Authentication is not enabled on server')
-
-        # Database.authenticate() needs to be in a request - check that it
-        # always runs in a request, and that it restores the request state
-        # (in or not in a request) properly when it's finished.
-        self.assertFalse(self.client.auto_start_request)
-        db = self.client.pymongo_test
-        db.add_user("mike", "password",
-                    roles=["userAdmin", "dbAdmin", "readWrite"])
-        try:
-            self.assertFalse(self.client.in_request())
-            self.assertTrue(db.authenticate("mike", "password"))
-            self.assertFalse(self.client.in_request())
-
-            request_cx = get_client(auto_start_request=True)
-            request_db = request_cx.pymongo_test
-            self.assertTrue(request_db.authenticate("mike", "password"))
-            self.assertTrue(request_cx.in_request())
-        finally:
-            db.authenticate("mike", "password")
-            db.remove_user("mike")
-            db.logout()
-            request_db.logout()
-
-    def test_authenticate_multiple(self):
-        client = get_client()
-        if (is_mongos(client) and not
-            version.at_least(self.client, (2, 0, 0))):
-            raise SkipTest("Auth with sharding requires MongoDB >= 2.0.0")
-        if not server_started_with_auth(client):
-            raise SkipTest("Authentication is not enabled on server")
-
-        # Setup
-        users_db = client.pymongo_test
-        admin_db = client.admin
-        other_db = client.pymongo_test1
-        users_db.test.remove()
-        other_db.test.remove()
-
-        admin_db.add_user('admin', 'pass',
-                          roles=["userAdminAnyDatabase", "dbAdmin",
-                                 "clusterAdmin", "readWrite"])
-        try:
-            self.assertTrue(admin_db.authenticate('admin', 'pass'))
-
-            if version.at_least(self.client, (2, 5, 3, -1)):
-                admin_db.add_user('ro-admin', 'pass',
-                                  roles=["userAdmin", "readAnyDatabase"])
-            else:
-                admin_db.add_user('ro-admin', 'pass', read_only=True)
-
-            users_db.add_user('user', 'pass',
-                              roles=["userAdmin", "readWrite"])
-
-            admin_db.logout()
-            self.assertRaises(OperationFailure, users_db.test.find_one)
-
-            # Regular user should be able to query its own db, but
-            # no other.
-            users_db.authenticate('user', 'pass')
-            self.assertEqual(0, users_db.test.count())
-            self.assertRaises(OperationFailure, other_db.test.find_one)
-
-            # Admin read-only user should be able to query any db,
-            # but not write.
-            admin_db.authenticate('ro-admin', 'pass')
-            self.assertEqual(0, other_db.test.count())
-            self.assertRaises(OperationFailure,
-                              other_db.test.insert, {})
-
-            # Force close all sockets
-            client.disconnect()
-
-            # We should still be able to write to the regular user's db
-            self.assertTrue(users_db.test.remove())
-            # And read from other dbs...
-            self.assertEqual(0, other_db.test.count())
-            # But still not write to other dbs...
-            self.assertRaises(OperationFailure,
-                              other_db.test.insert, {})
-
-        # Cleanup
-        finally:
-            admin_db.logout()
-            users_db.logout()
-            admin_db.authenticate('admin', 'pass')
-            remove_all_users(users_db)
-            remove_all_users(admin_db)
-
     def test_id_ordering(self):
         # PyMongo attempts to have _id show up first
         # when you iterate key/value pairs in a document.
@@ -703,6 +400,16 @@ class TestDatabase(unittest.TestCase):
         db.test.save(obj)
         self.assertEqual(obj, db.dereference(DBRef("test", 4)))
 
+    def test_deref_kwargs(self):
+        db = self.client.pymongo_test
+        db.test.remove({})
+
+        db.test.insert({"_id": 4, "foo": "bar"})
+        self.assertEqual(SON([("foo", "bar")]),
+                         db.dereference(DBRef("test", 4),
+                                        fields={"_id": False},
+                                        as_class=SON))
+
     def test_eval(self):
         db = self.client.pymongo_test
         db.test.remove({})
@@ -946,19 +653,46 @@ class TestDatabase(unittest.TestCase):
         else:
             self.fail("_check_command_response didn't raise OperationFailure")
 
-    def test_command_read_pref_warning(self):
-        warnings.simplefilter("error", UserWarning)
+    def test_mongos_response(self):
+        error_document = {
+            'ok': 0,
+            'errmsg': 'outer',
+            'raw': {'shard0/host0,host1': {'ok': 0, 'errmsg': 'inner'}}}
+
         try:
-            self.assertRaises(UserWarning, self.client.pymongo_test.command,
-                              'ping', read_preference=ReadPreference.SECONDARY)
-            try:
-                self.client.pymongo_test.command(
-                    'dbStats', read_preference=ReadPreference.SECONDARY)
-            except UserWarning:
-                self.fail("Shouldn't have raised UserWarning.")
-        finally:
-            warnings.resetwarnings()
-            warnings.simplefilter("ignore")
+            helpers._check_command_response(error_document, reset=None)
+        except OperationFailure, exc:
+            self.assertEqual('inner', str(exc))
+        else:
+            self.fail('OperationFailure not raised')
+
+        # If a shard has no primary and you run a command like dbstats, which
+        # cannot be run on a secondary, mongos's response includes empty "raw"
+        # errors. See SERVER-15428.
+        error_document = {
+            'ok': 0,
+            'errmsg': 'outer',
+            'raw': {'shard0/host0,host1': {}}}
+
+        try:
+            helpers._check_command_response(error_document, reset=None)
+        except OperationFailure, exc:
+            self.assertEqual('outer', str(exc))
+        else:
+            self.fail('OperationFailure not raised')
+
+        # Raw error has ok: 0 but no errmsg. Not a known case, but test it.
+        error_document = {
+            'ok': 0,
+            'errmsg': 'outer',
+            'raw': {'shard0/host0,host1': {'ok': 0}}}
+
+        try:
+            helpers._check_command_response(error_document, reset=None)
+        except OperationFailure, exc:
+            self.assertEqual('outer', str(exc))
+        else:
+            self.fail('OperationFailure not raised')
 
     def test_command_max_time_ms(self):
         if not version.at_least(self.client, (2, 5, 3, -1)):
@@ -989,6 +723,79 @@ class TestDatabase(unittest.TestCase):
                                       "maxTimeAlwaysTimeOut",
                                       mode="off")
 
+    def test_object_to_dict_transformer(self):
+        # PYTHON-709: Some users rely on their custom SONManipulators to run
+        # before any other checks, so they can insert non-dict objects and
+        # have them dictified before the _id is inserted or any other
+        # processing.
+        class Thing(object):
+            def __init__(self, value):
+                self.value = value
+
+        class ThingTransformer(SONManipulator):
+            def transform_incoming(self, thing, collection):
+                return {'value': thing.value}
+
+        db = self.client.foo
+        db.add_son_manipulator(ThingTransformer())
+        t = Thing('value')
+
+        db.test.remove()
+        db.test.insert([t])
+        out = db.test.find_one()
+        self.assertEqual('value', out.get('value'))
+
+    def test_son_manipulator_outgoing(self):
+        class Thing(object):
+            def __init__(self, value):
+                self.value = value
+
+        class ThingTransformer(SONManipulator):
+            def transform_outgoing(self, doc, collection):
+                # We don't want this applied to the command return
+                # value in pymongo.cursor.Cursor.
+                if 'value' in doc:
+                    return Thing(doc['value'])
+                return doc
+
+        db = self.client.foo
+        db.add_son_manipulator(ThingTransformer())
+
+        db.test.remove()
+        db.test.insert({'value': 'value'})
+        out = db.test.find_one()
+        self.assertTrue(isinstance(out, Thing))
+        self.assertEqual('value', out.value)
+
+        if version.at_least(self.client, (2, 6)):
+            out = db.test.aggregate([], cursor={}).next()
+            self.assertTrue(isinstance(out, Thing))
+            self.assertEqual('value', out.value)
+
+    def test_son_manipulator_inheritance(self):
+        class Thing(object):
+            def __init__(self, value):
+                self.value = value
+
+        class ThingTransformer(SONManipulator):
+            def transform_incoming(self, thing, collection):
+                return {'value': thing.value}
+
+            def transform_outgoing(self, son, collection):
+                return Thing(son['value'])
+
+        class Child(ThingTransformer):
+            pass
+
+        db = self.client.foo
+        db.add_son_manipulator(Child())
+        t = Thing('value')
+
+        db.test.remove()
+        db.test.insert([t])
+        out = db.test.find_one()
+        self.assertTrue(isinstance(out, Thing))
+        self.assertEqual('value', out.value)
 
 if __name__ == "__main__":
     unittest.main()

test/test_grid_file.py

@@ -39,7 +39,10 @@ from gridfs.errors import (NoFile,
 from pymongo import MongoClient
 from pymongo.errors import ConnectionFailure
 from test.test_client import get_client
-from test import qcheck
+from test import qcheck, skip_restricted_localhost
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestGridFile(unittest.TestCase):
@@ -586,6 +589,9 @@ with GridOut(self.db.fs, infile._id) as outfile:
         outfile.read()
         outfile.filename
 
+        outfile = GridOut(fs, infile._id, _connect=False)
+        outfile.readchunk()
+
     def test_grid_in_lazy_connect(self):
         client = MongoClient('badhost', _connect=False)
         fs = client.db.fs

test/test_gridfs.py

@@ -16,25 +16,29 @@
 
 """Tests for the gridfs package.
 """
-import sys
-sys.path[0:0] = [""]
-
-from pymongo.mongo_client import MongoClient
-from pymongo.errors import ConnectionFailure
-from pymongo.read_preferences import ReadPreference
-from test.test_replica_set_client import TestReplicaSetClientBase
-
 import datetime
-import unittest
+import sys
 import threading
 import time
+import unittest
+import warnings
+
+sys.path[0:0] = [""]
+
 import gridfs
 
 from bson.py3compat import b, StringIO
-from gridfs.errors import (FileExists,
-                           NoFile)
+from gridfs.errors import (FileExists, NoFile)
+from pymongo.errors import ConnectionFailure
+from pymongo.mongo_client import MongoClient
+from pymongo.read_preferences import ReadPreference
+from test import skip_restricted_localhost
 from test.test_client import get_client
-from test.utils import joinall
+from test.test_replica_set_client import TestReplicaSetClientBase
+from test.utils import catch_warnings, joinall
+
+
+setUpModule = skip_restricted_localhost
 
 
 class JustWrite(threading.Thread):
@@ -396,11 +400,35 @@ class TestGridfs(unittest.TestCase):
         cursor.close()
         self.assertRaises(TypeError, self.fs.find, {}, {"_id": True})
 
+    def test_gridfs_find_one(self):
+        self.assertEqual(None, self.fs.find_one())
+
+        id1 = self.fs.put(b('test1'), filename='file1')
+        self.assertEqual(b('test1'), self.fs.find_one().read())
+
+        id2 = self.fs.put(b('test2'), filename='file2', meta='data')
+        self.assertEqual(b('test1'), self.fs.find_one(id1).read())
+        self.assertEqual(b('test2'), self.fs.find_one(id2).read())
+
+        self.assertEqual(b('test1'),
+                         self.fs.find_one({'filename': 'file1'}).read())
+
+        self.assertEqual('data', self.fs.find_one(id2).meta)
+
+    def test_grid_in_non_int_chunksize(self):
+        # Lua, and perhaps other buggy GridFS clients, store size as a float.
+        data = b('data')
+        self.fs.put(data, filename='f')
+        self.db.fs.files.update({'filename': 'f'},
+                                {'$set': {'chunkSize': 100.0}})
+
+        self.assertEqual(data, self.fs.get_version('f').read())
+
 
 class TestGridfsReplicaSet(TestReplicaSetClientBase):
     def test_gridfs_replica_set(self):
         rsc = self._get_client(
-            w=self.w, wtimeout=5000,
+            w=self.w, wtimeout=30000,
             read_preference=ReadPreference.SECONDARY)
 
         try:
@@ -416,20 +444,25 @@ class TestGridfsReplicaSet(TestReplicaSetClientBase):
         primary_connection = MongoClient(primary_host, primary_port)
 
         secondary_host, secondary_port = self.secondaries[0]
-        for secondary_connection in [
-            MongoClient(secondary_host, secondary_port, slave_okay=True),
-            MongoClient(secondary_host, secondary_port,
-                        read_preference=ReadPreference.SECONDARY),
-        ]:
-            primary_connection.pymongo_test.drop_collection("fs.files")
-            primary_connection.pymongo_test.drop_collection("fs.chunks")
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            for secondary_connection in [
+                MongoClient(secondary_host, secondary_port, slave_okay=True),
+                MongoClient(secondary_host, secondary_port,
+                            read_preference=ReadPreference.SECONDARY),
+            ]:
+                primary_connection.pymongo_test.drop_collection("fs.files")
+                primary_connection.pymongo_test.drop_collection("fs.chunks")
 
-            # Should detect it's connected to secondary and not attempt to
-            # create index
-            fs = gridfs.GridFS(secondary_connection.pymongo_test)
+                # Should detect it's connected to secondary and not attempt to
+                # create index
+                fs = gridfs.GridFS(secondary_connection.pymongo_test)
 
-            # This won't detect secondary, raises error
-            self.assertRaises(ConnectionFailure, fs.put, b('foo'))
+                # This won't detect secondary, raises error
+                self.assertRaises(ConnectionFailure, fs.put, b('foo'))
+        finally:
+            ctx.exit()
 
     def test_gridfs_secondary_lazy(self):
         # Should detect it's connected to secondary and not attempt to

test/test_json_util.py

@@ -25,7 +25,7 @@ sys.path[0:0] = [""]
 
 import bson
 from bson.py3compat import b
-from bson import json_util
+from bson import json_util, EPOCH_AWARE
 from bson.binary import Binary, MD5_SUBTYPE, USER_DEFINED_SUBTYPE
 from bson.code import Code
 from bson.dbref import DBRef
@@ -37,6 +37,7 @@ from bson.son import RE_TYPE
 from bson.timestamp import Timestamp
 from bson.tz_util import utc
 
+from test import skip_restricted_localhost
 from test.test_client import get_client
 
 PY3 = sys.version_info[0] == 3
@@ -82,6 +83,37 @@ class TestJsonUtil(unittest.TestCase):
         self.round_trip({"date": datetime.datetime(2009, 12, 9, 15,
                                                    49, 45, 191000, utc)})
 
+        jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000+0000"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000+00:00"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000Z"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        # No explicit offset
+        jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        # Localtime behind UTC
+        jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000-0800"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000-08:00"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        # Localtime ahead of UTC
+        jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000+0100"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+        jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000+01:00"}}'
+        self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"])
+
+        dtm = datetime.datetime(1, 1, 1, 1, 1, 1, 0, utc)
+        jsn = '{"dt": {"$date": -62135593139000}}'
+        self.assertEqual(dtm, json_util.loads(jsn)["dt"])
+        jsn = '{"dt": {"$date": {"$numberLong": "-62135593139000"}}}'
+        self.assertEqual(dtm, json_util.loads(jsn)["dt"])
+
+        # Test support for microsecond accuracy
+        dtm = datetime.datetime(2014, 9, 17, 22, 41, 22, 201000, utc)
+        jsn = '{"dt": { "$date" : "2014-09-17T15:41:22.201-0700"}}'
+        self.assertEqual(dtm, json_util.loads(jsn)["dt"])
+
     def test_regex_object_hook(self):
         # simplejson or the builtin json module.
         from bson.json_util import json
@@ -149,14 +181,14 @@ class TestJsonUtil(unittest.TestCase):
         self.round_trip({"m": MaxKey()})
 
     def test_timestamp(self):
-        res = json_util.dumps({"ts": Timestamp(4, 13)}, default=json_util.default)
+        dct = {"ts": Timestamp(4, 13)}
+        res = json_util.dumps(dct, default=json_util.default)
         if not PY24:
             # Check order.
-            self.assertEqual('{"ts": {"t": 4, "i": 13}}', res)
+            self.assertEqual('{"ts": {"$timestamp": {"t": 4, "i": 13}}}', res)
 
-        dct = json_util.loads(res)
-        self.assertEqual(dct['ts']['t'], 4)
-        self.assertEqual(dct['ts']['i'], 13)
+        rtdct = json_util.loads(res)
+        self.assertEqual(dct, rtdct)
 
     def test_uuid(self):
         if not bson.has_uuid():
@@ -220,7 +252,16 @@ class TestJsonUtil(unittest.TestCase):
             # Check order.
             self.assertEqual('{"$code": "return z", "$scope": {"z": 2}}', res)
 
+    def test_undefined(self):
+        json = '{"name": {"$undefined": true}}'
+        self.assertEqual(json_util.loads(json)['name'], None)
+
+    def test_numberlong(self):
+        json = '{"weight": {"$numberLong": 65535}}'
+        self.assertEqual(json_util.loads(json)['weight'], long(65535))
+
     def test_cursor(self):
+        skip_restricted_localhost()
         db = self.db
 
         db.drop_collection("test")

test/test_legacy_connections.py

@@ -17,6 +17,7 @@
 
 import sys
 import unittest
+import warnings
 
 sys.path[0:0] = [""]
 
@@ -26,29 +27,38 @@ import pymongo
 from pymongo.connection import Connection
 from pymongo.replica_set_connection import ReplicaSetConnection
 from pymongo.errors import ConfigurationError
-from test import host, port, pair
+from test import host, port, pair, skip_restricted_localhost
 from test.test_replica_set_client import TestReplicaSetClientBase
-from test.utils import get_pool
+from test.utils import catch_warnings, get_pool
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestConnection(unittest.TestCase):
     def test_connection(self):
         c = Connection(host, port)
-        self.assertTrue(c.auto_start_request)
-        self.assertEqual(None, c.max_pool_size)
-        self.assertFalse(c.slave_okay)
-        self.assertFalse(c.safe)
-        self.assertEqual({}, c.get_lasterror_options())
 
-        # Connection's writes are unacknowledged by default
-        doc = {"_id": ObjectId()}
-        coll = c.pymongo_test.write_concern_test
-        coll.drop()
-        coll.insert(doc)
-        coll.insert(doc)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertTrue(c.auto_start_request)
+            self.assertEqual(None, c.max_pool_size)
+            self.assertFalse(c.slave_okay)
+            self.assertFalse(c.safe)
+            self.assertEqual({}, c.get_lasterror_options())
 
-        c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
-        self.assertTrue(c.safe)
+            # Connection's writes are unacknowledged by default
+            doc = {"_id": ObjectId()}
+            coll = c.pymongo_test.write_concern_test
+            coll.drop()
+            coll.insert(doc)
+            coll.insert(doc)
+
+            c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
+            self.assertTrue(c.safe)
+        finally:
+            ctx.exit()
 
         # To preserve legacy Connection's behavior, max_size should be None.
         # Pool should handle this without error.
@@ -73,23 +83,29 @@ class TestConnection(unittest.TestCase):
 class TestReplicaSetConnection(TestReplicaSetClientBase):
     def test_replica_set_connection(self):
         c = ReplicaSetConnection(pair, replicaSet=self.name)
-        self.assertTrue(c.auto_start_request)
-        self.assertEqual(None, c.max_pool_size)
-        self.assertFalse(c.slave_okay)
-        self.assertFalse(c.safe)
-        self.assertEqual({}, c.get_lasterror_options())
 
-        # ReplicaSetConnection's writes are unacknowledged by default
-        doc = {"_id": ObjectId()}
-        coll = c.pymongo_test.write_concern_test
-        coll.drop()
-        coll.insert(doc)
-        coll.insert(doc)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertTrue(c.auto_start_request)
+            self.assertEqual(None, c.max_pool_size)
+            self.assertFalse(c.slave_okay)
+            self.assertFalse(c.safe)
+            self.assertEqual({}, c.get_lasterror_options())
 
-        c = ReplicaSetConnection("mongodb://%s:%s/?replicaSet=%s&safe=true" % (
-            host, port, self.name))
+            # ReplicaSetConnection's writes are unacknowledged by default
+            doc = {"_id": ObjectId()}
+            coll = c.pymongo_test.write_concern_test
+            coll.drop()
+            coll.insert(doc)
+            coll.insert(doc)
 
-        self.assertTrue(c.safe)
+            c = ReplicaSetConnection("mongodb://%s:%s/?replicaSet=%s&safe=true" % (
+                host, port, self.name))
+
+            self.assertTrue(c.safe)
+        finally:
+            ctx.exit()
 
         # To preserve legacy ReplicaSetConnection's behavior, max_size should
         # be None. Pool should handle this without error.

test/test_master_slave_connection.py

@@ -15,11 +15,12 @@
 """Test for master slave connections."""
 
 import datetime
-import os
 import sys
 import threading
 import time
 import unittest
+import warnings
+
 sys.path[0:0] = [""]
 
 from nose.plugins.skip import SkipTest
@@ -34,8 +35,14 @@ from pymongo.database import Database
 from pymongo.mongo_client import MongoClient
 from pymongo.collection import Collection
 from pymongo.master_slave_connection import MasterSlaveConnection
-from test import host, port, host2, port2, host3, port3
-from test.utils import TestRequestMixin, get_pool
+from test import (host, port,
+                  host2, port2,
+                  host3, port3,
+                  skip_restricted_localhost)
+from test.utils import TestRequestMixin, catch_warnings, get_pool
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestMasterSlaveConnection(unittest.TestCase, TestRequestMixin):
@@ -59,10 +66,13 @@ class TestMasterSlaveConnection(unittest.TestCase, TestRequestMixin):
         if not self.slaves:
             raise SkipTest("Not connected to master-slave set")
 
+        self.ctx = catch_warnings()
+        warnings.simplefilter("ignore", DeprecationWarning)
         self.client = MasterSlaveConnection(self.master, self.slaves)
         self.db = self.client.pymongo_test
 
     def tearDown(self):
+        self.ctx.exit()
         try:
             self.db.test.drop_indexes()
         except Exception:
@@ -331,39 +341,17 @@ class TestMasterSlaveConnection(unittest.TestCase, TestRequestMixin):
         self.assertRaises(OperationFailure,
                           self.db.test.save, {'username': 'mike'})
 
-    # NOTE this test is non-deterministic, but I expect
-    # some failures unless the db is pulling instantaneously...
-    def test_insert_find_one_with_slaves(self):
-        count = 0
-        for i in range(100):
-            self.db.test.remove({})
-            self.db.test.insert({"x": i})
-            try:
-                if i != self.db.test.find_one()["x"]:
-                    count += 1
-            except:
-                count += 1
-        self.assertTrue(count)
-
-    # NOTE this test is non-deterministic, but hopefully we pause long enough
-    # for the slaves to pull...
-    def test_insert_find_one_with_pause(self):
-        count = 0
-
-        self.db.test.remove({})
-        self.db.test.insert({"x": 5586})
-        time.sleep(11)
-        for _ in range(10):
-            try:
-                if 5586 != self.db.test.find_one()["x"]:
-                    count += 1
-            except:
-                count += 1
-        self.assertFalse(count)
+    def test_insert(self):
+        w = len(self.slaves) + 1
+        self.db.test.remove(w=w)
+        self.assertEqual(0, self.db.test.count())
+        doc = {}
+        self.db.test.insert(doc, w=w)
+        self.assertEqual(doc, self.db.test.find_one())
 
     def test_kill_cursor_explicit(self):
         c = self.client
-        c.slave_okay = True
+        c.read_preference = ReadPreference.SECONDARY_PREFERRED
         db = c.pymongo_test
 
         test = db.master_slave_test_kill_cursor_explicit
@@ -412,60 +400,65 @@ class TestMasterSlaveConnection(unittest.TestCase, TestRequestMixin):
         self.assertRaises(OperationFailure, lambda: list(cursor2))
 
     def test_base_object(self):
-        c = self.client
-        self.assertFalse(c.slave_okay)
-        self.assertTrue(bool(c.read_preference))
-        self.assertTrue(c.safe)
-        self.assertEqual({}, c.get_lasterror_options())
-        db = c.pymongo_test
-        self.assertFalse(db.slave_okay)
-        self.assertTrue(bool(c.read_preference))
-        self.assertTrue(db.safe)
-        self.assertEqual({}, db.get_lasterror_options())
-        coll = db.test
-        coll.drop()
-        self.assertFalse(coll.slave_okay)
-        self.assertTrue(bool(c.read_preference))
-        self.assertTrue(coll.safe)
-        self.assertEqual({}, coll.get_lasterror_options())
-        cursor = coll.find()
-        self.assertFalse(cursor._Cursor__slave_okay)
-        self.assertTrue(bool(cursor._Cursor__read_preference))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            c = self.client
+            self.assertFalse(c.slave_okay)
+            self.assertTrue(bool(c.read_preference))
+            self.assertTrue(c.safe)
+            self.assertEqual({}, c.get_lasterror_options())
+            db = c.pymongo_test
+            self.assertFalse(db.slave_okay)
+            self.assertTrue(bool(c.read_preference))
+            self.assertTrue(db.safe)
+            self.assertEqual({}, db.get_lasterror_options())
+            coll = db.test
+            coll.drop()
+            self.assertFalse(coll.slave_okay)
+            self.assertTrue(bool(c.read_preference))
+            self.assertTrue(coll.safe)
+            self.assertEqual({}, coll.get_lasterror_options())
+            cursor = coll.find()
+            self.assertFalse(cursor._Cursor__slave_okay)
+            self.assertTrue(bool(cursor._Cursor__read_preference))
 
-        w = 1 + len(self.slaves)
-        wtimeout=10000 # Wait 10 seconds for replication to complete
-        c.set_lasterror_options(w=w, wtimeout=wtimeout)
-        self.assertFalse(c.slave_okay)
-        self.assertTrue(bool(c.read_preference))
-        self.assertTrue(c.safe)
-        self.assertEqual({'w': w, 'wtimeout': wtimeout}, c.get_lasterror_options())
-        db = c.pymongo_test
-        self.assertFalse(db.slave_okay)
-        self.assertTrue(bool(c.read_preference))
-        self.assertTrue(db.safe)
-        self.assertEqual({'w': w, 'wtimeout': wtimeout}, db.get_lasterror_options())
-        coll = db.test
-        self.assertFalse(coll.slave_okay)
-        self.assertTrue(bool(c.read_preference))
-        self.assertTrue(coll.safe)
-        self.assertEqual({'w': w, 'wtimeout': wtimeout},
-                         coll.get_lasterror_options())
-        cursor = coll.find()
-        self.assertFalse(cursor._Cursor__slave_okay)
-        self.assertTrue(bool(cursor._Cursor__read_preference))
+            w = 1 + len(self.slaves)
+            wtimeout=10000 # Wait 10 seconds for replication to complete
+            c.set_lasterror_options(w=w, wtimeout=wtimeout)
+            self.assertFalse(c.slave_okay)
+            self.assertTrue(bool(c.read_preference))
+            self.assertTrue(c.safe)
+            self.assertEqual({'w': w, 'wtimeout': wtimeout}, c.get_lasterror_options())
+            db = c.pymongo_test
+            self.assertFalse(db.slave_okay)
+            self.assertTrue(bool(c.read_preference))
+            self.assertTrue(db.safe)
+            self.assertEqual({'w': w, 'wtimeout': wtimeout}, db.get_lasterror_options())
+            coll = db.test
+            self.assertFalse(coll.slave_okay)
+            self.assertTrue(bool(c.read_preference))
+            self.assertTrue(coll.safe)
+            self.assertEqual({'w': w, 'wtimeout': wtimeout},
+                             coll.get_lasterror_options())
+            cursor = coll.find()
+            self.assertFalse(cursor._Cursor__slave_okay)
+            self.assertTrue(bool(cursor._Cursor__read_preference))
 
-        coll.insert({'foo': 'bar'})
-        self.assertEqual(1, coll.find({'foo': 'bar'}).count())
-        self.assertTrue(coll.find({'foo': 'bar'}))
-        coll.remove({'foo': 'bar'})
-        self.assertEqual(0, coll.find({'foo': 'bar'}).count())
+            coll.insert({'foo': 'bar'})
+            self.assertEqual(1, coll.find({'foo': 'bar'}).count())
+            self.assertTrue(coll.find({'foo': 'bar'}))
+            coll.remove({'foo': 'bar'})
+            self.assertEqual(0, coll.find({'foo': 'bar'}).count())
 
-        c.safe = False
-        c.unset_lasterror_options()
-        self.assertFalse(self.client.slave_okay)
-        self.assertTrue(bool(self.client.read_preference))
-        self.assertFalse(self.client.safe)
-        self.assertEqual({}, self.client.get_lasterror_options())
+            c.safe = False
+            c.unset_lasterror_options()
+            self.assertFalse(self.client.slave_okay)
+            self.assertTrue(bool(self.client.read_preference))
+            self.assertFalse(self.client.safe)
+            self.assertEqual({}, self.client.get_lasterror_options())
+        finally:
+            ctx.exit()
 
     def test_document_class(self):
         c = MasterSlaveConnection(self.master, self.slaves)

test/test_mongos_ha.py

@@ -21,9 +21,13 @@ import unittest
 sys.path[0:0] = [""]
 
 from pymongo.errors import AutoReconnect
+from test import skip_restricted_localhost
 from test.pymongo_mocks import MockClient
 
 
+setUpModule = skip_restricted_localhost
+
+
 class FindOne(threading.Thread):
     def __init__(self, client):
         super(FindOne, self).__init__()
@@ -119,6 +123,37 @@ class TestMongosHA(unittest.TestCase):
         # Down host is still in list.
         self.assertEqual(len(mock_hosts), len(client.nodes))
 
+    def test_acceptable_latency(self):
+        client = MockClient(
+            standalones=[],
+            members=[],
+            mongoses=['a:1', 'b:2', 'c:3'],
+            host='a:1,b:2,c:3',
+            secondaryAcceptableLatencyMS=7)
+
+        self.assertEqual(7, client.secondary_acceptable_latency_ms)
+        # No error
+        client.db.collection.find_one()
+
+        client = MockClient(
+            standalones=[],
+            members=[],
+            mongoses=['a:1', 'b:2', 'c:3'],
+            host='a:1,b:2,c:3',
+            secondaryAcceptableLatencyMS=0)
+
+        self.assertEqual(0, client.secondary_acceptable_latency_ms)
+        # No error
+        client.db.collection.find_one()
+        # Our chosen mongos goes down.
+        client.kill_host('%s:%s' % (client.host, client.port))
+        try:
+            client.db.collection.find_one()
+        except:
+            pass
+        # No error
+        client.db.collection.find_one()
+
 
 if __name__ == "__main__":
     unittest.main()

test/test_objectid.py

@@ -18,14 +18,13 @@ import datetime
 import pickle
 import unittest
 import sys
-import time
 sys.path[0:0] = [""]
 
 from nose.plugins.skip import SkipTest
 
 from bson.errors import InvalidId
 from bson.objectid import ObjectId
-from bson.py3compat import b, binary_type
+from bson.py3compat import b
 from bson.tz_util import (FixedOffset,
                           utc)
 
@@ -181,6 +180,7 @@ class TestObjectId(unittest.TestCase):
         self.assertEqual(oid_1_9, oid_1_10)
 
     def test_is_valid(self):
+        self.assertFalse(ObjectId.is_valid(None))
         self.assertFalse(ObjectId.is_valid(4))
         self.assertFalse(ObjectId.is_valid(175.0))
         self.assertFalse(ObjectId.is_valid({"test": 4}))

test/test_pooling.py

@@ -23,13 +23,16 @@ sys.path[0:0] = [""]
 
 from nose.plugins.skip import SkipTest
 
-from test import host, port
+from test import host, port, skip_restricted_localhost
 from test.test_pooling_base import (
     _TestPooling, _TestMaxPoolSize, _TestMaxOpenSockets,
-    _TestPoolSocketSharing, _TestWaitQueueMultiple, one)
+    _TestWaitQueueMultiple, one)
 from test.utils import get_pool
 
 
+setUpModule = skip_restricted_localhost
+
+
 class TestPoolingThreads(_TestPooling, unittest.TestCase):
     use_greenlets = False
 
@@ -165,10 +168,6 @@ class TestMaxPoolSizeThreads(_TestMaxPoolSize, unittest.TestCase):
     use_greenlets = False
 
 
-class TestPoolSocketSharingThreads(_TestPoolSocketSharing, unittest.TestCase):
-    use_greenlets = False
-
-
 class TestMaxOpenSocketsThreads(_TestMaxOpenSockets, unittest.TestCase):
     use_greenlets = False
 

test/test_pooling_base.py

@@ -134,7 +134,8 @@ class NonUnique(MongoThread):
         for _ in xrange(N):
             self.client.start_request()
             self.db.unique.insert({"_id": "jesse"}, w=0)
-            self.ut.assertNotEqual(None, self.db.error())
+            self.ut.assertNotEqual(None,
+                                   self.db.command('getlasterror').get('err'))
             self.client.end_request()
 
 
@@ -152,7 +153,7 @@ class NoRequest(MongoThread):
         errors = 0
         for _ in xrange(N):
             self.db.unique.insert({"_id": "jesse"}, w=0)
-            if not self.db.error():
+            if not self.db.command("getlasterror").get("err"):
                 errors += 1
 
         self.client.end_request()
@@ -1141,99 +1142,3 @@ class _TestWaitQueueMultiple(_TestPoolingBase):
 
         for socket_info in socks:
             socket_info.close()
-
-
-class _TestPoolSocketSharing(_TestPoolingBase):
-    """Directly test that two simultaneous operations don't share a socket. To
-    be run both with threads and with greenlets.
-    """
-    def _test_pool(self, use_request):
-        """
-        Test that the connection pool prevents both threads and greenlets from
-        using a socket at the same time.
-
-        Sequence:
-        gr0: start a slow find()
-        gr1: start a fast find()
-        gr1: get results
-        gr0: get results
-        """
-        cx = get_client(
-            use_greenlets=self.use_greenlets,
-            auto_start_request=False
-        )
-
-        db = cx.pymongo_test
-        db.test.remove()
-        db.test.insert({'_id': 1})
-
-        history = []
-
-        def find_fast():
-            if use_request:
-                cx.start_request()
-
-            history.append('find_fast start')
-
-            # With greenlets and the old connection._Pool, this would throw
-            # AssertionError: "This event is already used by another
-            # greenlet"
-            self.assertEqual({'_id': 1}, db.test.find_one())
-            history.append('find_fast done')
-
-            if use_request:
-                cx.end_request()
-
-        def find_slow():
-            if use_request:
-                cx.start_request()
-
-            history.append('find_slow start')
-
-            # Javascript function that pauses N seconds per document
-            fn = delay(10)
-            if (is_mongos(db.connection) or not
-                version.at_least(db.connection, (1, 7, 2))):
-                # mongos doesn't support eval so we have to use $where
-                # which is less reliable in this context.
-                self.assertEqual(1, db.test.find({"$where": fn}).count())
-            else:
-                # 'nolock' allows find_fast to start and finish while we're
-                # waiting for this to complete.
-                self.assertEqual({'ok': 1.0, 'retval': True},
-                                 db.command('eval', fn, nolock=True))
-
-            history.append('find_slow done')
-
-            if use_request:
-                cx.end_request()
-
-        if self.use_greenlets:
-            gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast)
-            gr0.start()
-            gr1.start_later(.1)
-        else:
-            gr0 = threading.Thread(target=find_slow)
-            gr0.setDaemon(True)
-            gr1 = threading.Thread(target=find_fast)
-            gr1.setDaemon(True)
-
-            gr0.start()
-            time.sleep(.1)
-            gr1.start()
-
-        gr0.join()
-        gr1.join()
-
-        self.assertEqual([
-            'find_slow start',
-            'find_fast start',
-            'find_fast done',
-            'find_slow done',
-        ], history)
-
-    def test_pool(self):
-        self._test_pool(use_request=False)
-
-    def test_pool_request(self):
-        self._test_pool(use_request=True)

test/test_pooling_gevent.py

@@ -23,11 +23,14 @@ from nose.plugins.skip import SkipTest
 
 from pymongo import pool
 from pymongo.errors import ConfigurationError
-from test import host, port
+from test import host, port, skip_restricted_localhost
 from test.utils import looplet
 from test.test_pooling_base import (
     _TestPooling, _TestMaxPoolSize, _TestMaxOpenSockets,
-    _TestPoolSocketSharing, _TestWaitQueueMultiple, has_gevent)
+    _TestWaitQueueMultiple, has_gevent)
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestPoolingGevent(_TestPooling, unittest.TestCase):
@@ -181,10 +184,6 @@ class TestMaxPoolSizeGevent(_TestMaxPoolSize, unittest.TestCase):
     use_greenlets = True
 
 
-class TestPoolSocketSharingGevent(_TestPoolSocketSharing, unittest.TestCase):
-    use_greenlets = True
-
-
 class TestMaxOpenSocketsGevent(_TestMaxOpenSockets, unittest.TestCase):
     use_greenlets = True
 

test/test_pymongo.py

@@ -15,7 +15,6 @@
 """Test the pymongo module itself."""
 
 import unittest
-import os
 import sys
 sys.path[0:0] = [""]
 

test/test_read_preferences.py

@@ -14,9 +14,9 @@
 
 """Test the replica_set_connection module."""
 import random
-
 import sys
 import unittest
+import warnings
 
 from nose.plugins.skip import SkipTest
 
@@ -24,6 +24,8 @@ sys.path[0:0] = [""]
 
 from bson.son import SON
 from pymongo.cursor import _QUERY_OPTIONS
+from pymongo.master_slave_connection import MasterSlaveConnection
+from pymongo.mongo_client import MongoClient
 from pymongo.mongo_replica_set_client import MongoReplicaSetClient
 from pymongo.read_preferences import (ReadPreference, modes, MovingAverage,
                                       secondary_ok_commands)
@@ -31,7 +33,11 @@ from pymongo.errors import ConfigurationError
 
 from test.test_replica_set_client import TestReplicaSetClientBase
 from test.test_client import get_client
-from test import version, utils, host, port
+from test import version, utils, host, port, skip_restricted_localhost
+from test.utils import catch_warnings
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestReadPreferencesBase(TestReplicaSetClientBase):
@@ -76,6 +82,24 @@ class TestReadPreferencesBase(TestReplicaSetClientBase):
             expected, used))
 
 
+class TestSlaveOkayMetadataCommands(TestReadPreferencesBase):
+
+    def test_slave_okay_metadata_commands(self):
+
+        secondaries = iter(self._get_client().secondaries)
+        host, port = secondaries.next()
+        # Direct connection to a secondary.
+        client = MongoClient(host, port)
+        self.assertFalse(client.is_primary)
+        self.assertEqual(client.read_preference, ReadPreference.PRIMARY)
+
+        # No error.
+        client.database_names()
+        client.pymongo_test.collection_names()
+        client.pymongo_test.test.options()
+        client.pymongo_test.test.index_information()
+
+
 class TestReadPreferences(TestReadPreferencesBase):
     def test_mode_validation(self):
         # 'modes' are imported from read_preferences.py
@@ -129,6 +153,14 @@ class TestReadPreferences(TestReadPreferencesBase):
             secondaryacceptablelatencyms=666
         ).secondary_acceptable_latency_ms)
 
+        self.assertEqual(0, self._get_client(
+            secondaryacceptablelatencyms=0
+        ).secondary_acceptable_latency_ms)
+
+        self.assertRaises(ConfigurationError,
+                          self._get_client,
+                          secondaryacceptablelatencyms=-1)
+
     def test_primary(self):
         self.assertReadsFrom('primary',
             read_preference=ReadPreference.PRIMARY)
@@ -274,12 +306,50 @@ class TestCommandAndReadPreference(TestReplicaSetClientBase):
                                 "Some members not used for NEAREST: %s" % (
                                     unused))
 
+    def test_command_read_pref_warning(self):
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("error", UserWarning)
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertRaises(UserWarning, self.c.pymongo_test.command,
+                              'ping', read_preference=ReadPreference.SECONDARY)
+            try:
+                self.c.pymongo_test.command('dbStats',
+                    read_preference=ReadPreference.SECONDARY_PREFERRED)
+            except UserWarning:
+                self.fail("Shouldn't have raised UserWarning.")
+
+            primary = MongoClient(host, port)
+            try:
+                primary.pymongo_test.command('ping',
+                    read_preference=ReadPreference.SECONDARY_PREFERRED)
+            except UserWarning:
+                self.fail("Shouldn't have raised UserWarning.")
+
+            secondary_addr = iter(self.c.secondaries).next()
+            secondary = MongoClient(*secondary_addr)
+            msclient = MasterSlaveConnection(primary, [secondary])
+            self.assertRaises(UserWarning, msclient.pymongo_test.command,
+                              'ping', read_preference=ReadPreference.SECONDARY)
+            try:
+                msclient.pymongo_test.command('dbStats',
+                    read_preference=ReadPreference.SECONDARY_PREFERRED)
+            except UserWarning:
+                self.fail("Shouldn't have raised UserWarning.")
+        finally:
+            ctx.exit()
+
     def test_command(self):
         # Test generic 'command' method. Some commands obey read preference,
         # most don't.
         # Disobedient commands, always go to primary
-        self._test_fn(False, lambda: self.c.pymongo_test.command('ping'))
-        self._test_fn(False, lambda: self.c.admin.command('buildinfo'))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            self._test_fn(False, lambda: self.c.pymongo_test.command('ping'))
+            self._test_fn(False, lambda: self.c.admin.command('buildinfo'))
+        finally:
+            ctx.exit()
 
         # Obedient commands.
         self._test_fn(True, lambda: self.c.pymongo_test.command('group', {
@@ -303,23 +373,21 @@ class TestCommandAndReadPreference(TestReplicaSetClientBase):
 
         # Distinct
         self._test_fn(True, lambda: self.c.pymongo_test.command(
-            'distinct', 'test', key={'a': 1}))
+            'distinct', 'test', key='a'))
         self._test_fn(True, lambda: self.c.pymongo_test.command(
-            'distinct', 'test', key={'a': 1}, query={'a': 1}))
+            'distinct', 'test', key='a', query={'a': 1}))
         self._test_fn(True, lambda: self.c.pymongo_test.command(SON([
-            ('distinct', 'test'), ('key', {'a': 1}), ('query', {'a': 1})])))
+            ('distinct', 'test'), ('key', 'a'), ('query', {'a': 1})])))
 
-        # Geo stuff. Make sure a 2d index is created and replicated
-        self.c.pymongo_test.system.indexes.insert({
-            'key' : { 'location' : '2d' }, 'ns' : 'pymongo_test.test',
-            'name' : 'location_2d' }, w=self.w)
+        # Geo stuff.
+        self.c.pymongo_test.test.create_index([('location', '2d')])
 
-        self.c.pymongo_test.system.indexes.insert(SON([
-            ('ns', 'pymongo_test.test'),
-            ('key', SON([('location', 'geoHaystack'), ('key', 1)])),
-            ('bucketSize', 100),
-            ('name', 'location_geoHaystack'),
-        ]), w=self.w)
+        self.c.pymongo_test.test.create_index([('location', 'geoHaystack'),
+                                               ('key', 1)], bucketSize=100)
+
+        # Attempt to await replication of indexes replicated.
+        self.c.pymongo_test.test2.insert({}, w=self.w)
+        self.c.pymongo_test.test2.remove({}, w=self.w)
 
         self._test_fn(True, lambda: self.c.pymongo_test.command(
             'geoNear', 'test', near=[0, 0]))
@@ -340,46 +408,31 @@ class TestCommandAndReadPreference(TestReplicaSetClientBase):
                 ('pipeline', [])
             ])))
 
-        # Text search.
-        if version.at_least(self.c, (2, 3, 2)):
-            utils.enable_text_search(self.c)
-            db = self.c.pymongo_test
-
-            # Only way to create an index and wait for all members to build it.
-            index = {
-                'ns': 'pymongo_test.test',
-                'name': 't_text',
-                'key': {'t': 'text'}}
-
-            db.system.indexes.insert(
-                index, manipulate=False, check_keys=False, w=self.w)
-
-            self._test_fn(True, lambda: self.c.pymongo_test.command(SON([
-                ('text', 'test'),
-                ('search', 'foo')])))
-
-            self.c.pymongo_test.test.drop_indexes()
-
     def test_map_reduce_command(self):
         # mapreduce fails if no collection
         self.c.pymongo_test.test.insert({}, w=self.w)
 
         # Non-inline mapreduce always goes to primary, doesn't obey read prefs.
         # Test with command in a SON and with kwargs
-        self._test_fn(False, lambda: self.c.pymongo_test.command(SON([
-            ('mapreduce', 'test'),
-            ('map', 'function() { }'),
-            ('reduce', 'function() { }'),
-            ('out', 'mr_out')
-        ])))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            self._test_fn(False, lambda: self.c.pymongo_test.command(SON([
+                ('mapreduce', 'test'),
+                ('map', 'function() { }'),
+                ('reduce', 'function() { }'),
+                ('out', 'mr_out')
+            ])))
 
-        self._test_fn(False, lambda: self.c.pymongo_test.command(
-            'mapreduce', 'test', map='function() { }',
-            reduce='function() { }', out='mr_out'))
+            self._test_fn(False, lambda: self.c.pymongo_test.command(
+                'mapreduce', 'test', map='function() { }',
+                reduce='function() { }', out='mr_out'))
 
-        self._test_fn(False, lambda: self.c.pymongo_test.command(
-            'mapreduce', 'test', map='function() { }',
-            reduce='function() { }', out={'replace': 'some_collection'}))
+            self._test_fn(False, lambda: self.c.pymongo_test.command(
+                'mapreduce', 'test', map='function() { }',
+                reduce='function() { }', out={'replace': 'some_collection'}))
+        finally:
+            ctx.exit()
 
         # Inline mapreduce obeys read prefs
         self._test_fn(True, lambda: self.c.pymongo_test.command(
@@ -405,32 +458,47 @@ class TestCommandAndReadPreference(TestReplicaSetClientBase):
 
         # Aggregate with $out always goes to primary, doesn't obey read prefs.
         # Test aggregate command sent directly to db.command.
-        self._test_fn(False, lambda: self.c.pymongo_test.command(
-            "aggregate", "test",
-            pipeline=[{"$match": {"x": 1}}, {"$out": "agg_out"}]
-        ))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            self._test_fn(False, lambda: self.c.pymongo_test.command(
+                "aggregate", "test",
+                pipeline=[{"$match": {"x": 1}}, {"$out": "agg_out"}]
+            ))
 
-        # Test aggregate when sent through the collection aggregate function.
-        self._test_fn(False, lambda: self.c.pymongo_test.test.aggregate(
-            [{"$match": {"x": 2}}, {"$out": "agg_out"}]
-        ))
+            # Test aggregate when sent through the collection aggregate function.
+            self._test_fn(False, lambda: self.c.pymongo_test.test.aggregate(
+                [{"$match": {"x": 2}}, {"$out": "agg_out"}]
+            ))
+        finally:
+            ctx.exit()
 
         self.c.pymongo_test.drop_collection("test")
         self.c.pymongo_test.drop_collection("agg_out")
 
     def test_create_collection(self):
         # Collections should be created on primary, obviously
-        self._test_fn(False, lambda: self.c.pymongo_test.command(
-            'create', 'some_collection%s' % random.randint(0, sys.maxint)))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            self._test_fn(False, lambda: self.c.pymongo_test.command(
+                'create', 'some_collection%s' % random.randint(0, sys.maxint)))
 
-        self._test_fn(False, lambda: self.c.pymongo_test.create_collection(
-            'some_collection%s' % random.randint(0, sys.maxint)))
+            self._test_fn(False, lambda: self.c.pymongo_test.create_collection(
+                'some_collection%s' % random.randint(0, sys.maxint)))
+        finally:
+            ctx.exit()
 
     def test_drop_collection(self):
-        self._test_fn(False, lambda: self.c.pymongo_test.drop_collection(
-            'some_collection'))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            self._test_fn(False, lambda: self.c.pymongo_test.drop_collection(
+                'some_collection'))
 
-        self._test_fn(False, lambda: self.c.pymongo_test.some_collection.drop())
+            self._test_fn(False, lambda: self.c.pymongo_test.some_collection.drop())
+        finally:
+            ctx.exit()
 
     def test_group(self):
         self._test_fn(True, lambda: self.c.pymongo_test.test.group(
@@ -440,8 +508,13 @@ class TestCommandAndReadPreference(TestReplicaSetClientBase):
         # mapreduce fails if no collection
         self.c.pymongo_test.test.insert({}, w=self.w)
 
-        self._test_fn(False, lambda: self.c.pymongo_test.test.map_reduce(
-            'function() { }', 'function() { }', 'mr_out'))
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            self._test_fn(False, lambda: self.c.pymongo_test.test.map_reduce(
+                'function() { }', 'function() { }', 'mr_out'))
+        finally:
+            ctx.exit()
 
         self._test_fn(True, lambda: self.c.pymongo_test.test.map_reduce(
             'function() { }', 'function() { }', {'inline': 1}))
@@ -517,83 +590,93 @@ class TestMongosConnection(unittest.TestCase):
         NEAREST = ReadPreference.NEAREST
         SLAVE_OKAY = _QUERY_OPTIONS['slave_okay']
 
-        # Test non-PRIMARY modes which can be combined with tags
-        for kwarg, value, mongos_mode in (
-            ('read_preference', PRIMARY_PREFERRED, 'primaryPreferred'),
-            ('read_preference', SECONDARY, 'secondary'),
-            ('read_preference', SECONDARY_PREFERRED, 'secondaryPreferred'),
-            ('read_preference', NEAREST, 'nearest'),
-            ('slave_okay', True, 'secondaryPreferred'),
-            ('slave_okay', False, 'primary')
-        ):
-            for tag_sets in (
-                None, [{}]
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            # Test non-PRIMARY modes which can be combined with tags
+            for kwarg, value, mongos_mode in (
+                ('read_preference', PRIMARY_PREFERRED, 'primaryPreferred'),
+                ('read_preference', SECONDARY, 'secondary'),
+                ('read_preference', SECONDARY_PREFERRED, 'secondaryPreferred'),
+                ('read_preference', NEAREST, 'nearest'),
+                ('slave_okay', True, 'secondaryPreferred'),
+                ('slave_okay', False, 'primary')
             ):
-                # Create a client e.g. with read_preference=NEAREST or
-                # slave_okay=True
-                c = get_client(tag_sets=tag_sets, **{kwarg: value})
+                for tag_sets in (
+                    None, [{}]
+                ):
+                    # Create a client e.g. with read_preference=NEAREST or
+                    # slave_okay=True
+                    c = get_client(tag_sets=tag_sets, **{kwarg: value})
 
-                self.assertEqual(is_mongos, c.is_mongos)
-                cursor = c.pymongo_test.test.find()
-                if is_mongos:
-                    # We don't set $readPreference for SECONDARY_PREFERRED
-                    # unless tags are in use. slaveOkay has the same effect.
-                    if mongos_mode == 'secondaryPreferred':
-                        self.assertEqual(
-                            None,
-                            cursor._Cursor__query_spec().get('$readPreference'))
+                    self.assertEqual(is_mongos, c.is_mongos)
+                    cursor = c.pymongo_test.test.find()
+                    if is_mongos:
+                        # We don't set $readPreference for SECONDARY_PREFERRED
+                        # unless tags are in use. slaveOkay has the same effect.
+                        if mongos_mode == 'secondaryPreferred':
+                            self.assertEqual(
+                                None,
+                                cursor._Cursor__query_spec().get('$readPreference'))
 
-                        self.assertTrue(
-                            cursor._Cursor__query_options() & SLAVE_OKAY)
+                            self.assertTrue(
+                                cursor._Cursor__query_options() & SLAVE_OKAY)
 
-                    # Don't send $readPreference for PRIMARY either
-                    elif mongos_mode == 'primary':
-                        self.assertEqual(
-                            None,
-                            cursor._Cursor__query_spec().get('$readPreference'))
+                        # Don't send $readPreference for PRIMARY either
+                        elif mongos_mode == 'primary':
+                            self.assertEqual(
+                                None,
+                                cursor._Cursor__query_spec().get('$readPreference'))
 
-                        self.assertFalse(
-                            cursor._Cursor__query_options() & SLAVE_OKAY)
+                            self.assertFalse(
+                                cursor._Cursor__query_options() & SLAVE_OKAY)
+                        else:
+                            self.assertEqual(
+                                {'mode': mongos_mode},
+                                cursor._Cursor__query_spec().get('$readPreference'))
+
+                            self.assertTrue(
+                                cursor._Cursor__query_options() & SLAVE_OKAY)
                     else:
+                        self.assertFalse(
+                            '$readPreference' in cursor._Cursor__query_spec())
+
+                for tag_sets in (
+                    [{'dc': 'la'}],
+                    [{'dc': 'la'}, {'dc': 'sf'}],
+                    [{'dc': 'la'}, {'dc': 'sf'}, {}],
+                ):
+                    if kwarg == 'slave_okay':
+                        # Can't use tags with slave_okay True or False, need a
+                        # real read preference
+                        self.assertRaises(
+                            ConfigurationError,
+                            get_client, tag_sets=tag_sets, **{kwarg: value})
+
+                        continue
+
+                    c = get_client(tag_sets=tag_sets, **{kwarg: value})
+
+                    self.assertEqual(is_mongos, c.is_mongos)
+                    cursor = c.pymongo_test.test.find()
+                    if is_mongos:
                         self.assertEqual(
-                            {'mode': mongos_mode},
+                            {'mode': mongos_mode, 'tags': tag_sets},
                             cursor._Cursor__query_spec().get('$readPreference'))
-
-                        self.assertTrue(
-                            cursor._Cursor__query_options() & SLAVE_OKAY)
-                else:
-                    self.assertFalse(
-                        '$readPreference' in cursor._Cursor__query_spec())
-
-            for tag_sets in (
-                [{'dc': 'la'}],
-                [{'dc': 'la'}, {'dc': 'sf'}],
-                [{'dc': 'la'}, {'dc': 'sf'}, {}],
-            ):
-                if kwarg == 'slave_okay':
-                    # Can't use tags with slave_okay True or False, need a
-                    # real read preference
-                    self.assertRaises(
-                        ConfigurationError,
-                        get_client, tag_sets=tag_sets, **{kwarg: value})
-
-                    continue
-
-                c = get_client(tag_sets=tag_sets, **{kwarg: value})
-
-                self.assertEqual(is_mongos, c.is_mongos)
-                cursor = c.pymongo_test.test.find()
-                if is_mongos:
-                    self.assertEqual(
-                        {'mode': mongos_mode, 'tags': tag_sets},
-                        cursor._Cursor__query_spec().get('$readPreference'))
-                else:
-                    self.assertFalse(
-                        '$readPreference' in cursor._Cursor__query_spec())
+                    else:
+                        self.assertFalse(
+                            '$readPreference' in cursor._Cursor__query_spec())
+        finally:
+            ctx.exit()
 
     def test_only_secondary_ok_commands_have_read_prefs(self):
         c = get_client(read_preference=ReadPreference.SECONDARY)
-        is_mongos = utils.is_mongos(c)
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", UserWarning)
+            is_mongos = utils.is_mongos(c)
+        finally:
+            ctx.exit()
         if not is_mongos:
             raise SkipTest("Only mongos have read_prefs added to the spec")
 

test/test_replica_set_client.py

@@ -18,6 +18,7 @@
 
 import copy
 import datetime
+import random
 import signal
 import socket
 import sys
@@ -26,6 +27,7 @@ import thread
 import threading
 import traceback
 import unittest
+import warnings
 
 sys.path[0:0] = [""]
 
@@ -35,8 +37,8 @@ from bson.son import SON
 from bson.tz_util import utc
 from pymongo.mongo_client import MongoClient
 from pymongo.read_preferences import ReadPreference
-from pymongo.member import PRIMARY, SECONDARY, OTHER
-from pymongo.mongo_replica_set_client import MongoReplicaSetClient
+from pymongo.member import SECONDARY, Member
+from pymongo.mongo_replica_set_client import MongoReplicaSetClient, Monitor
 from pymongo.mongo_replica_set_client import _partition_node, have_gevent
 from pymongo.database import Database
 from pymongo.pool import SocketInfo
@@ -45,13 +47,15 @@ from pymongo.errors import (AutoReconnect,
                             ConnectionFailure,
                             InvalidName,
                             OperationFailure, InvalidOperation)
-from test import version, port, pair
+from test import version, port, pair, skip_restricted_localhost, auth_context
 from test.pymongo_mocks import MockReplicaSetClient
 from test.utils import (
     delay, assertReadFrom, assertReadFromAll, read_from_which_host,
-    remove_all_users, assertRaisesExactly, TestRequestMixin, one,
-    server_started_with_auth, pools_from_rs_client, get_pool,
-    _TestLazyConnectMixin)
+    assertRaisesExactly, TestRequestMixin, one, pools_from_rs_client, get_pool,
+    _TestLazyConnectMixin, _TestExhaustCursorMixin, catch_warnings)
+
+
+setUpModule = skip_restricted_localhost
 
 
 class TestReplicaSetClientAgainstStandalone(unittest.TestCase):
@@ -82,7 +86,10 @@ class TestReplicaSetClientBase(unittest.TestCase):
             self.arbiters = set([_partition_node(h)
                                  for h in response.get("arbiters", [])])
 
-            repl_set_status = client.admin.command('replSetGetStatus')
+            # Cannot run replSetGetStatus in MongoDB >= 2.7.1 under auth once a
+            # user has been added.
+            repl_set_status = auth_context.client.admin.command(
+                'replSetGetStatus')
             primary_info = [
                 m for m in repl_set_status['members']
                 if m['stateStr'] == 'PRIMARY'
@@ -121,6 +128,43 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
             standardMsg = '%r is not an instance of %r' % (obj, cls)
             self.fail(self._formatMessage(msg, standardMsg))
 
+    def test_keyword_arg_defaults(self):
+        client = MongoReplicaSetClient(socketTimeoutMS=None,
+                                       connectTimeoutMS=20000,
+                                       waitQueueTimeoutMS=None,
+                                       waitQueueMultiple=None,
+                                       socketKeepAlive=False,
+                                       auto_start_request=False,
+                                       use_greenlets=False,
+                                       replicaSet='myreplset',  # Required
+                                       read_preference=ReadPreference.PRIMARY,
+                                       tag_sets=[{}],
+                                       ssl=False,
+                                       ssl_keyfile=None,
+                                       ssl_certfile=None,
+                                       ssl_ca_certs=None,
+                                       _connect=False)
+        self.assertEqual(None,
+                         client._MongoReplicaSetClient__net_timeout)
+        # socket.Socket.settimeout takes a float in seconds
+        self.assertEqual(20.0,
+                         client._MongoReplicaSetClient__conn_timeout)
+        self.assertEqual(None,
+                         client._MongoReplicaSetClient__wait_queue_timeout)
+        self.assertEqual(None,
+                         client._MongoReplicaSetClient__wait_queue_multiple)
+        self.assertFalse(client._MongoReplicaSetClient__socket_keepalive)
+        self.assertFalse(client.auto_start_request)
+        self.assertFalse(client.use_greenlets)
+        self.assertEqual('myreplset',
+                         client._MongoReplicaSetClient__name)
+        self.assertEqual(ReadPreference.PRIMARY, client.read_preference)
+        self.assertEqual([{}], client.tag_sets)
+        self.assertFalse(client._MongoReplicaSetClient__use_ssl)
+        self.assertEqual(None, client._MongoReplicaSetClient__ssl_keyfile)
+        self.assertEqual(None, client._MongoReplicaSetClient__ssl_certfile)
+        self.assertEqual(None, client._MongoReplicaSetClient__ssl_ca_certs)
+
     def test_init_disconnected(self):
         c = self._get_client(_connect=False)
 
@@ -167,44 +211,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
 
         self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
 
-    def test_init_disconnected_with_auth_failure(self):
-        c = MongoReplicaSetClient(
-            "mongodb://user:pass@somedomainthatdoesntexist", replicaSet="rs",
-            connectTimeoutMS=1, _connect=False)
-
-        self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
-
-    def test_init_disconnected_with_auth(self):
-        c = self._get_client()
-        if not server_started_with_auth(c):
-            raise SkipTest('Authentication is not enabled on server')
-
-        c.admin.add_user("admin", "pass")
-        c.admin.authenticate("admin", "pass")
-        try:
-            c.pymongo_test.add_user("user", "pass", roles=['readWrite', 'userAdmin'])
-
-            # Auth with lazy connection.
-            host = one(self.hosts)
-            uri = "mongodb://user:pass@%s:%d/pymongo_test?replicaSet=%s" % (
-                host[0], host[1], self.name)
-
-            authenticated_client = MongoReplicaSetClient(uri, _connect=False)
-            authenticated_client.pymongo_test.test.find_one()
-
-            # Wrong password.
-            bad_uri = "mongodb://user:wrong@%s:%d/pymongo_test?replicaSet=%s" % (
-                host[0], host[1], self.name)
-
-            bad_client = MongoReplicaSetClient(bad_uri, _connect=False)
-            self.assertRaises(
-                OperationFailure, bad_client.pymongo_test.test.find_one)
-
-        finally:
-            # Clean up.
-            remove_all_users(c.pymongo_test)
-            remove_all_users(c.admin)
-
     def test_connect(self):
         assertRaisesExactly(ConnectionFailure, MongoReplicaSetClient,
                           "somedomainthatdoesntexist.org:27017",
@@ -260,7 +266,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
                                  read_preference=ReadPreference.SECONDARY,
                                  tag_sets=copy.deepcopy(tag_sets),
                                  secondary_acceptable_latency_ms=77)
-        c.admin.command('ping')
         self.assertEqual(c.primary, self.primary)
         self.assertEqual(c.hosts, self.hosts)
         self.assertEqual(c.arbiters, self.arbiters)
@@ -341,20 +346,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
         finally:
             socket.socket.sendall = old_sendall
 
-    def test_lazy_auth_raises_operation_failure(self):
-        # Check if we have the prerequisites to run this test.
-        c = self._get_client()
-        if not server_started_with_auth(c):
-            raise SkipTest('Authentication is not enabled on server')
-
-        lazy_client = MongoReplicaSetClient(
-            "mongodb://user:wrong@%s/pymongo_test" % pair,
-            replicaSet=self.name,
-            _connect=False)
-
-        assertRaisesExactly(
-            OperationFailure, lazy_client.test.collection.find_one)
-
     def test_operations(self):
         c = self._get_client()
 
@@ -428,72 +419,25 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
 
     def test_copy_db(self):
         c = self._get_client()
-        # We test copy twice; once starting in a request and once not. In
-        # either case the copy should succeed (because it starts a request
-        # internally) and should leave us in the same state as before the copy.
-        c.start_request()
+        ctx = catch_warnings()
+        try:
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertRaises(TypeError, c.copy_database, 4, "foo")
+            self.assertRaises(TypeError, c.copy_database, "foo", 4)
+            self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
 
-        self.assertRaises(TypeError, c.copy_database, 4, "foo")
-        self.assertRaises(TypeError, c.copy_database, "foo", 4)
+            c.pymongo_test.test.drop()
+            c.pymongo_test.test.insert({"foo": "bar"})
 
-        self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
-
-        c.pymongo_test.test.drop()
-        c.drop_database("pymongo_test1")
-        c.drop_database("pymongo_test2")
-
-        c.pymongo_test.test.insert({"foo": "bar"})
-
-        self.assertFalse("pymongo_test1" in c.database_names())
-        self.assertFalse("pymongo_test2" in c.database_names())
-
-        c.copy_database("pymongo_test", "pymongo_test1")
-        # copy_database() didn't accidentally end the request
-        self.assertTrue(c.in_request())
-
-        self.assertTrue("pymongo_test1" in c.database_names())
-        self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
-
-        c.end_request()
-
-        self.assertFalse(c.in_request())
-        c.copy_database("pymongo_test", "pymongo_test2", pair)
-        # copy_database() didn't accidentally restart the request
-        self.assertFalse(c.in_request())
-
-        time.sleep(1)
-
-        self.assertTrue("pymongo_test2" in c.database_names())
-        self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
-
-        if version.at_least(c, (1, 3, 3, 1)) and server_started_with_auth(c):
             c.drop_database("pymongo_test1")
+            self.assertFalse("pymongo_test1" in c.database_names())
 
-            c.admin.add_user("admin", "password")
-            c.admin.authenticate("admin", "password")
-            try:
-                c.pymongo_test.add_user("mike", "password")
-
-                self.assertRaises(OperationFailure, c.copy_database,
-                                  "pymongo_test", "pymongo_test1",
-                                  username="foo", password="bar")
-                self.assertFalse("pymongo_test1" in c.database_names())
-
-                self.assertRaises(OperationFailure, c.copy_database,
-                                  "pymongo_test", "pymongo_test1",
-                                  username="mike", password="bar")
-                self.assertFalse("pymongo_test1" in c.database_names())
-
-                c.copy_database("pymongo_test", "pymongo_test1",
-                                username="mike", password="password")
-                self.assertTrue("pymongo_test1" in c.database_names())
-                res = c.pymongo_test1.test.find_one(_must_use_master=True)
-                self.assertEqual("bar", res["foo"])
-            finally:
-                # Cleanup
-                remove_all_users(c.pymongo_test)
-                c.admin.remove_user("admin")
-        c.close()
+            c.copy_database("pymongo_test", "pymongo_test1")
+            self.assertTrue("pymongo_test1" in c.database_names())
+            self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
+            c.drop_database("pymongo_test1")
+        finally:
+            ctx.exit()
 
     def test_get_default_database(self):
         host = one(self.hosts)
@@ -748,6 +692,11 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
         self.assertEqual(pool.wait_queue_multiple, 2)
         self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6)
 
+    def test_socketKeepAlive(self):
+        client = self._get_client(socketKeepAlive=True)
+        pool = get_pool(client)
+        self.assertTrue(pool.socket_keepalive)
+
     def test_tz_aware(self):
         self.assertRaises(ConfigurationError, MongoReplicaSetClient,
                           tz_aware='foo', replicaSet=self.name)
@@ -1077,6 +1026,28 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
 
         client.close()
 
+    def test_zero_latency(self):
+        orig_interval = Monitor._refresh_interval
+        Monitor._refresh_interval = 1e9
+        ping_times = set()
+        # Generate unique ping times.
+        while len(ping_times) < len(self.hosts):
+            ping_times.add(random.random())
+        for ping_time, host in zip(ping_times, self.hosts):
+            Member._host_to_ping_time[host] = ping_time
+        try:
+            client = self._get_client()
+            host = read_from_which_host(
+                client, ReadPreference.NEAREST, None, 0)
+            for _ in range(5):
+                self.assertEqual(
+                    host,
+                    read_from_which_host(
+                        client, ReadPreference.NEAREST, None, 0))
+        finally:
+            Monitor._refresh_interval = orig_interval
+            Member._host_to_ping_time.clear()
+
     def test_pinned_member(self):
         latency = 1000 * 1000
         client = self._get_client(secondary_acceptable_latency_ms=latency)
@@ -1238,5 +1209,12 @@ class TestReplicaSetClientMaxWriteBatchSize(unittest.TestCase):
         self.assertEqual(c.max_write_batch_size, 2)
 
 
+class TestReplicaSetClientExhaustCursor(
+        _TestExhaustCursorMixin,
+        TestReplicaSetClientBase):
+
+    # Base class implements _get_client already.
+    pass
+
 if __name__ == "__main__":
     unittest.main()

test/test_replica_set_reconfig.py

@@ -21,9 +21,13 @@ sys.path[0:0] = [""]
 
 from pymongo.errors import ConfigurationError, ConnectionFailure
 from pymongo import ReadPreference
+from test import skip_restricted_localhost
 from test.pymongo_mocks import MockClient, MockReplicaSetClient
 
 
+setUpModule = skip_restricted_localhost
+
+
 class TestSecondaryBecomesStandalone(unittest.TestCase):
     # An administrator removes a secondary from a 3-node set and
     # brings it back up as standalone, without updating the other

test/test_son.py

@@ -67,6 +67,15 @@ class TestSON(unittest.TestCase):
                                      ('mike', 'awesome'),
                                      ('hello', 'world'))))
 
+        # Embedded SON.
+        d4 = SON([('blah', {'foo': SON()})])
+        self.assertEqual(d4, {'blah': {'foo': {}}})
+        self.assertEqual(d4, {'blah': {'foo': SON()}})
+        self.assertNotEqual(d4, {'blah': {'foo': []}})
+
+        # Original data unaffected.
+        self.assertEqual(SON, d4['blah']['foo'].__class__)
+
     def test_to_dict(self):
         a1 = SON()
         b2 = SON([("blah", SON())])
@@ -81,6 +90,9 @@ class TestSON(unittest.TestCase):
         self.assertEqual(dict, c3.to_dict()["blah"][0].__class__)
         self.assertEqual(dict, d4.to_dict()["blah"]["foo"].__class__)
 
+        # Original data unaffected.
+        self.assertEqual(SON, d4['blah']['foo'].__class__)
+
     def test_pickle(self):
 
         simple_son = SON([])
@@ -143,6 +155,47 @@ class TestSON(unittest.TestCase):
         self.assertEqual(reflexive_son.keys(), reflexive_son1.keys())
         self.assertEqual(id(reflexive_son1), id(reflexive_son1["reflexive"]))
 
+    def test_iteration(self):
+        """
+        Test __iter__
+        """
+        # test success case
+        test_son = SON([(1, 100), (2, 200), (3, 300)])
+        for ele in test_son:
+            self.assertEqual(ele * 100, test_son[ele])
+
+    def test_contains_has(self):
+        """
+        has_key and __contains__
+        """
+        test_son = SON([(1, 100), (2, 200), (3, 300)])
+        self.assertTrue(1 in test_son)
+        self.assertTrue(2 in test_son, "in failed")
+        self.assertFalse(22 in test_son, "in succeeded when it shouldn't")
+        self.assertTrue(test_son.has_key(2), "has_key failed")
+        self.assertFalse(test_son.has_key(22), "has_key succeeded when it shouldn't")
+
+    def test_clears(self):
+        """
+        Test clear()
+        """
+        test_son = SON([(1, 100), (2, 200), (3, 300)])
+        test_son.clear()
+        self.assertFalse(1 in test_son)
+        self.assertEqual(0, len(test_son))
+        self.assertEqual(0, len(test_son.keys()))
+        self.assertEqual({}, test_son.to_dict())
+
+    def test_len(self):
+        """
+        Test len
+        """
+        test_son = SON()
+        self.assertEqual(0, len(test_son))
+        test_son = SON([(1, 100), (2, 200), (3, 300)])
+        self.assertEqual(3, len(test_son))
+        test_son.popitem()
+        self.assertEqual(2, len(test_son))
 
 if __name__ == "__main__":
     unittest.main()

test/test_ssl.py

@@ -19,20 +19,20 @@ import socket
 import sys
 import unittest
 
+sys.path[0:0] = [""]
+
 try:
     from ssl import CertificateError
 except ImportError:
     # Backport.
     from pymongo.ssl_match_hostname import CertificateError
 
-sys.path[0:0] = [""]
-
 from urllib import quote_plus
 
 from nose.plugins.skip import SkipTest
 
 from pymongo import MongoClient, MongoReplicaSetClient
-from pymongo.common import HAS_SSL
+from pymongo.common import HAS_SSL, validate_cert_reqs
 from pymongo.errors import (ConfigurationError,
                             ConnectionFailure,
                             OperationFailure)
@@ -52,9 +52,9 @@ MONGODB_X509_USERNAME = (
 
 # To fully test this start a mongod instance (built with SSL support) like so:
 # mongod --dbpath /path/to/data/directory --sslOnNormalPorts \
-# --sslPEMKeyFile /path/to/mongo/jstests/libs/server.pem \
-# --sslCAFile /path/to/mongo/jstests/libs/ca.pem \
-# --sslCRLFile /path/to/mongo/jstests/libs/crl.pem \
+# --sslPEMKeyFile /path/to/pymongo/test/certificates/server.pem \
+# --sslCAFile /path/to/pymongo/test/certificates/ca.pem \
+# --sslCRLFile /path/to/pymongo/test/certificates/crl.pem \
 # --sslWeakCertificateValidation
 # Also, make sure you have 'server' as an alias for localhost in /etc/hosts
 #
@@ -85,17 +85,19 @@ if HAS_SSL:
         MongoClient(host, port, connectTimeoutMS=100, ssl=True)
         SIMPLE_SSL = True
     except ConnectionFailure:
-        # Is MongoDB configured with server.pem, ca.pem, and crl.pem from
-        # mongodb jstests/lib?
-        try:
-            MongoClient(host, port, connectTimeoutMS=100, ssl=True,
-                        ssl_certfile=CLIENT_PEM)
-            CERT_SSL = True
-        except ConnectionFailure:
-            pass
+        pass
 
-        if CERT_SSL:
-            SERVER_IS_RESOLVABLE = is_server_resolvable()
+    # Is MongoDB configured with server.pem, ca.pem, and crl.pem from
+    # mongodb jstests/lib?
+    try:
+        MongoClient(host, port, connectTimeoutMS=100, ssl=True,
+                    ssl_certfile=CLIENT_PEM)
+        CERT_SSL = True
+    except ConnectionFailure:
+        pass
+
+    if CERT_SSL:
+        SERVER_IS_RESOLVABLE = is_server_resolvable()
 
 
 class TestClientSSL(unittest.TestCase):
@@ -123,7 +125,6 @@ class TestClientSSL(unittest.TestCase):
                           ssl_certfile=CLIENT_PEM)
 
     def test_config_ssl(self):
-        """Tests various ssl configurations"""
         self.assertRaises(ConfigurationError, MongoClient, ssl='foo')
         self.assertRaises(ConfigurationError,
                           MongoClient,
@@ -199,6 +200,40 @@ class TestClientSSL(unittest.TestCase):
                           ssl_keyfile=CLIENT_PEM,
                           ssl_certfile=CLIENT_PEM)
 
+        if HAS_SSL:
+            self.assertRaises(
+                ConfigurationError, validate_cert_reqs, 'ssl_cert_reqs', 3)
+            self.assertRaises(
+                ConfigurationError, validate_cert_reqs, 'ssl_cert_reqs', -1)
+            self.assertRaises(
+                ConfigurationError, validate_cert_reqs, 'ssl_cert_reqs', 'foo')
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', None), None)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', ssl.CERT_NONE),
+                ssl.CERT_NONE)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', ssl.CERT_OPTIONAL),
+                ssl.CERT_OPTIONAL)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', ssl.CERT_REQUIRED),
+                ssl.CERT_REQUIRED)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', 0), ssl.CERT_NONE)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', 1), ssl.CERT_OPTIONAL)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', 2), ssl.CERT_REQUIRED)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', 'CERT_NONE'),
+                ssl.CERT_NONE)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', 'CERT_OPTIONAL'),
+                ssl.CERT_OPTIONAL)
+            self.assertEqual(
+                validate_cert_reqs('ssl_cert_reqs', 'CERT_REQUIRED'),
+                ssl.CERT_REQUIRED)
+
 
 class TestSSL(unittest.TestCase):
 
@@ -234,9 +269,9 @@ class TestSSL(unittest.TestCase):
         # Expects the server to be running with the server.pem, ca.pem
         # and crl.pem provided in mongodb and the server tests eg:
         #
-        #   --sslPEMKeyFile=jstests/libs/server.pem
-        #   --sslCAFile=jstests/libs/ca.pem
-        #   --sslCRLFile=jstests/libs/crl.pem
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
         #
         # Also requires an /etc/hosts entry where "server" is resolvable
         if not CERT_SSL:
@@ -260,9 +295,9 @@ class TestSSL(unittest.TestCase):
         # Expects the server to be running with the server.pem, ca.pem
         # and crl.pem provided in mongodb and the server tests eg:
         #
-        #   --sslPEMKeyFile=jstests/libs/server.pem
-        #   --sslCAFile=jstests/libs/ca.pem
-        #   --sslCRLFile=jstests/libs/crl.pem
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
         #
         # Also requires an /etc/hosts entry where "server" is resolvable
         if not CERT_SSL:
@@ -286,9 +321,9 @@ class TestSSL(unittest.TestCase):
         # Expects the server to be running with the server.pem, ca.pem
         # and crl.pem provided in mongodb and the server tests eg:
         #
-        #   --sslPEMKeyFile=jstests/libs/server.pem
-        #   --sslCAFile=jstests/libs/ca.pem
-        #   --sslCRLFile=jstests/libs/crl.pem
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
         #
         # Also requires an /etc/hosts entry where "server" is resolvable
         if not CERT_SSL:
@@ -323,13 +358,39 @@ class TestSSL(unittest.TestCase):
         self.assertTrue(db.test.find_one()['ssl'])
         client.drop_database('pymongo_ssl_test')
 
+    def test_cert_ssl_uri_support(self):
+        # Expects the server to be running with the server.pem, ca.pem
+        # and crl.pem provided in mongodb and the server tests eg:
+        #
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
+        #
+        # Also requires an /etc/hosts entry where "server" is resolvable
+        if not CERT_SSL:
+            raise SkipTest("No mongod available over SSL with certs")
+
+        if not SERVER_IS_RESOLVABLE:
+            raise SkipTest("No hosts entry for 'server'. Cannot validate "
+                           "hostname in the certificate")
+
+        uri_fmt = ("mongodb://server/?ssl=true&ssl_certfile=%s&ssl_cert_reqs"
+                   "=%s&ssl_ca_certs=%s")
+        client = MongoClient(uri_fmt % (CLIENT_PEM, 'CERT_REQUIRED', CA_PEM))
+
+        db = client.pymongo_ssl_test
+        db.test.drop()
+        db.test.insert({'ssl': True})
+        self.assertTrue(db.test.find_one()['ssl'])
+        client.drop_database('pymongo_ssl_test')
+
     def test_cert_ssl_validation_optional(self):
         # Expects the server to be running with the server.pem, ca.pem
         # and crl.pem provided in mongodb and the server tests eg:
         #
-        #   --sslPEMKeyFile=jstests/libs/server.pem
-        #   --sslCAFile=jstests/libs/ca.pem
-        #   --sslCRLFile=jstests/libs/crl.pem
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
         #
         # Also requires an /etc/hosts entry where "server" is resolvable
         if not CERT_SSL:
@@ -369,9 +430,9 @@ class TestSSL(unittest.TestCase):
         # Expects the server to be running with the server.pem, ca.pem
         # and crl.pem provided in mongodb and the server tests eg:
         #
-        #   --sslPEMKeyFile=jstests/libs/server.pem
-        #   --sslCAFile=jstests/libs/ca.pem
-        #   --sslCRLFile=jstests/libs/crl.pem
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
         if not CERT_SSL:
             raise SkipTest("No mongod available over SSL with certs")
 
@@ -406,9 +467,9 @@ class TestSSL(unittest.TestCase):
         # and crl.pem provided in mongodb and the server tests as well as
         # --auth
         #
-        #   --sslPEMKeyFile=jstests/libs/server.pem
-        #   --sslCAFile=jstests/libs/ca.pem
-        #   --sslCRLFile=jstests/libs/crl.pem
+        #   --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
+        #   --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
+        #   --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
         #   --auth
         if not CERT_SSL:
             raise SkipTest("No mongod available over SSL with certs")

test/test_threads.py

@@ -18,14 +18,15 @@ import unittest
 import threading
 import traceback
 
-from nose.plugins.skip import SkipTest
-
-from test.utils import (joinall, remove_all_users,
-                        server_started_with_auth, RendezvousThread)
+from test import skip_restricted_localhost
+from test.utils import joinall, RendezvousThread
 from test.test_client import get_client
 from test.utils import get_pool
 from pymongo.pool import SocketInfo, _closed
-from pymongo.errors import AutoReconnect, OperationFailure
+from pymongo.errors import AutoReconnect
+
+
+setUpModule = skip_restricted_localhost
 
 
 class AutoAuthenticateThreads(threading.Thread):
@@ -299,87 +300,9 @@ class BaseTestThreads(object):
             self.assertTrue(t.passed, "%s threw exception" % t)
 
 
-class BaseTestThreadsAuth(object):
-    """
-    Base test class for TestThreadsAuth and TestThreadsAuthReplicaSet. (This is
-    not itself a unittest.TestCase, otherwise it'd be run twice -- once when
-    nose imports this module, and once when nose imports
-    test_threads_replica_set_connection.py, which imports this module.)
-    """
-    def _get_client(self):
-        """
-        Intended for overriding in TestThreadsAuthReplicaSet. This method
-        returns a MongoClient here, and a MongoReplicaSetClient in
-        test_threads_replica_set_connection.py.
-        """
-        # Regular test client
-        return get_client()
-
-    def setUp(self):
-        client = self._get_client()
-        if not server_started_with_auth(client):
-            raise SkipTest("Authentication is not enabled on server")
-        self.client = client
-        self.client.admin.add_user('admin-user', 'password',
-                                   roles=['clusterAdmin',
-                                          'dbAdminAnyDatabase',
-                                          'readWriteAnyDatabase',
-                                          'userAdminAnyDatabase'])
-        self.client.admin.authenticate("admin-user", "password")
-        self.client.auth_test.add_user("test-user", "password",
-                                       roles=['readWrite'])
-
-    def tearDown(self):
-        # Remove auth users from databases
-        self.client.admin.authenticate("admin-user", "password")
-        remove_all_users(self.client.auth_test)
-        self.client.drop_database('auth_test')
-        remove_all_users(self.client.admin)
-        # Clear client reference so that RSC's monitor thread
-        # dies.
-        self.client = None
-
-    def test_auto_auth_login(self):
-        client = self._get_client()
-        self.assertRaises(OperationFailure, client.auth_test.test.find_one)
-
-        # Admin auth
-        client = self._get_client()
-        client.admin.authenticate("admin-user", "password")
-
-        nthreads = 10
-        threads = []
-        for _ in xrange(nthreads):
-            t = AutoAuthenticateThreads(client.auth_test.test, 100)
-            t.start()
-            threads.append(t)
-
-        joinall(threads)
-
-        for t in threads:
-            self.assertTrue(t.success)
-
-        # Database-specific auth
-        client = self._get_client()
-        client.auth_test.authenticate("test-user", "password")
-
-        threads = []
-        for _ in xrange(nthreads):
-            t = AutoAuthenticateThreads(client.auth_test.test, 100)
-            t.start()
-            threads.append(t)
-
-        joinall(threads)
-
-        for t in threads:
-            self.assertTrue(t.success)
-
 class TestThreads(BaseTestThreads, unittest.TestCase):
     pass
 
-class TestThreadsAuth(BaseTestThreadsAuth, unittest.TestCase):
-    pass
-
 
 if __name__ == "__main__":
     unittest.main()

test/test_threads_replica_set_client.py

@@ -16,10 +16,12 @@
 
 import unittest
 
-from pymongo.mongo_replica_set_client import MongoReplicaSetClient
+from test import skip_restricted_localhost
+from test.test_threads import BaseTestThreads
+from test.test_replica_set_client import TestReplicaSetClientBase
 
-from test.test_threads import BaseTestThreads, BaseTestThreadsAuth
-from test.test_replica_set_client import TestReplicaSetClientBase, pair
+
+setUpModule = skip_restricted_localhost
 
 
 class TestThreadsReplicaSet(TestReplicaSetClientBase, BaseTestThreads):
@@ -39,31 +41,6 @@ class TestThreadsReplicaSet(TestReplicaSetClientBase, BaseTestThreads):
         return TestReplicaSetClientBase._get_client(self, **kwargs)
 
 
-class TestThreadsAuthReplicaSet(TestReplicaSetClientBase, BaseTestThreadsAuth):
-
-    def setUp(self):
-        """
-        Prepare to test all the same things that TestThreads tests, but do it
-        with a replica-set client
-        """
-        TestReplicaSetClientBase.setUp(self)
-        BaseTestThreadsAuth.setUp(self)
-
-    def tearDown(self):
-        TestReplicaSetClientBase.tearDown(self)
-        BaseTestThreadsAuth.tearDown(self)
-
-    def _get_client(self):
-        """
-        Override TestThreadsAuth, so its tests run on a MongoReplicaSetClient
-        instead of a regular MongoClient.
-        """
-        return MongoReplicaSetClient(pair, replicaSet=self.name)
-
-
 if __name__ == "__main__":
-    suite = unittest.TestSuite([
-        unittest.makeSuite(TestThreadsReplicaSet),
-        unittest.makeSuite(TestThreadsAuthReplicaSet)
-    ])
+    suite = unittest.makeSuite(TestThreadsReplicaSet)
     unittest.TextTestRunner(verbosity=2).run(suite)

test/test_uri_parser.py

@@ -131,6 +131,10 @@ class TestURI(unittest.TestCase):
                          split_options('authMechanism=GSSAPI'))
         self.assertEqual({'authmechanism': 'MONGODB-CR'},
                          split_options('authMechanism=MONGODB-CR'))
+        self.assertEqual({'authmechanism': 'SCRAM-SHA-1'},
+                         split_options('authMechanism=SCRAM-SHA-1'))
+        self.assertRaises(ConfigurationError,
+                          split_options, 'authMechanism=foo')
         self.assertEqual({'authsource': 'foobar'}, split_options('authSource=foobar'))
         # maxPoolSize isn't yet a documented URI option.
         self.assertRaises(ConfigurationError, split_options, 'maxpoolsize=50')

test/utils.py

@@ -15,14 +15,18 @@
 """Utilities for testing pymongo
 """
 
+import gc
 import os
 import struct
 import sys
 import threading
+import time
 
 from nose.plugins.skip import SkipTest
+
+from bson.son import SON
 from pymongo import MongoClient, MongoReplicaSetClient
-from pymongo.errors import AutoReconnect
+from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure
 from pymongo.pool import NO_REQUEST, NO_SOCKET_YET, SocketInfo
 from test import host, port, version
 
@@ -88,10 +92,8 @@ def server_started_with_auth(client):
         # MongoDB >= 2.6
         if 'security' in parsed:
             security = parsed['security']
-            # >= rc3
             if 'authorization' in security:
                 return security['authorization'] == 'enabled'
-            # < rc3
             return security.get('auth', False) or bool(security.get('keyFile'))
         return parsed.get('auth', False) or bool(parsed.get('keyFile'))
     # Legacy
@@ -453,7 +455,7 @@ def lazy_client_trial(reset, target, test, get_client, use_greenlets):
     # Make concurrency bugs more likely to manifest.
     interval = None
     if not sys.platform.startswith('java'):
-        if sys.version_info >= (3, 2):
+        if hasattr(sys, 'getswitchinterval'):
             interval = sys.getswitchinterval()
             sys.setswitchinterval(1e-6)
         else:
@@ -472,7 +474,7 @@ def lazy_client_trial(reset, target, test, get_client, use_greenlets):
 
     finally:
         if not sys.platform.startswith('java'):
-            if sys.version_info >= (3, 2):
+            if hasattr(sys, 'setswitchinterval'):
                 sys.setswitchinterval(interval)
             else:
                 sys.setcheckinterval(interval)
@@ -489,9 +491,6 @@ class _TestLazyConnectMixin(object):
     """
     use_greenlets = False
 
-    NTRIALS = 5
-    NTHREADS = 10
-
     def test_insert(self):
         def reset(collection):
             collection.drop()
@@ -584,3 +583,164 @@ class _TestLazyConnectMixin(object):
             self.assertEqual(
                 ismaster['maxMessageSizeBytes'],
                 c.max_message_size)
+
+
+class _TestExhaustCursorMixin(object):
+    """Test that clients properly handle errors from exhaust cursors.
+
+    Inherit from this class and from unittest.TestCase, and override
+    _get_client(self, **kwargs).
+    """
+    def test_exhaust_query_server_error(self):
+        # When doing an exhaust query, the socket stays checked out on success
+        # but must be checked in on error to avoid semaphore leaks.
+        client = self._get_client(max_pool_size=1)
+        if is_mongos(client):
+            raise SkipTest("Can't use exhaust cursors with mongos")
+        if not version.at_least(client, (2, 2, 0)):
+            raise SkipTest("mongod < 2.2.0 closes exhaust socket on error")
+
+        collection = client.pymongo_test.test
+        pool = get_pool(client)
+
+        sock_info = one(pool.sockets)
+        # This will cause OperationFailure in all mongo versions since
+        # the value for $orderby must be a document.
+        cursor = collection.find(
+            SON([('$query', {}), ('$orderby', True)]), exhaust=True)
+        self.assertRaises(OperationFailure, cursor.next)
+        self.assertFalse(sock_info.closed)
+
+        # The semaphore was decremented despite the error.
+        self.assertTrue(pool._socket_semaphore.acquire(blocking=False))
+
+    def test_exhaust_query_network_error(self):
+        # When doing an exhaust query, the socket stays checked out on success
+        # but must be checked in on error to avoid semaphore leaks.
+        client = self._get_client(max_pool_size=1)
+        if is_mongos(client):
+            raise SkipTest("Can't use exhaust cursors with mongos")
+
+        collection = client.pymongo_test.test
+        pool = get_pool(client)
+        pool._check_interval_seconds = None  # Never check.
+
+        # Cause a network error.
+        sock_info = one(pool.sockets)
+        sock_info.sock.close()
+        cursor = collection.find(exhaust=True)
+        self.assertRaises(ConnectionFailure, cursor.next)
+        self.assertTrue(sock_info.closed)
+
+        # The semaphore was decremented despite the error.
+        self.assertTrue(pool._socket_semaphore.acquire(blocking=False))
+
+    def test_exhaust_getmore_network_error(self):
+        # When doing a getmore on an exhaust cursor, the socket stays checked
+        # out on success but must be checked in on error to avoid semaphore
+        # leaks.
+        client = self._get_client(max_pool_size=1)
+        if is_mongos(client):
+            raise SkipTest("Can't use exhaust cursors with mongos")
+
+        collection = client.pymongo_test.test
+        collection.remove()
+        collection.insert([{} for _ in range(200)])  # More than one batch.
+        pool = get_pool(client)
+        pool._check_interval_seconds = None  # Never check.
+
+        cursor = collection.find(exhaust=True)
+
+        # Initial query succeeds.
+        cursor.next()
+
+        # Cause a network error.
+        sock_info = cursor._Cursor__exhaust_mgr.sock
+        sock_info.sock.close()
+
+        # A getmore fails.
+        self.assertRaises(ConnectionFailure, list, cursor)
+        self.assertTrue(sock_info.closed)
+
+        # The semaphore was decremented despite the error.
+        self.assertTrue(pool._socket_semaphore.acquire(blocking=False))
+
+
+# Backport of WarningMessage from python 2.6, with fixed syntax for python 2.4.
+class WarningMessage(object):
+
+    """Holds the result of a single showwarning() call."""
+
+    _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
+                        "line")
+
+    def __init__(self, message, category,
+                 filename, lineno, file=None, line=None):
+        local_values = locals()
+        for attr in self._WARNING_DETAILS:
+            setattr(self, attr, local_values[attr])
+        self._category_name = None
+        if category:
+            self._category_name = category.__name__
+
+    def __str__(self):
+        return ("{message : %r, category : %r, filename : %r, lineno : %s, "
+                    "line : %r}" % (self.message, self._category_name,
+                                    self.filename, self.lineno, self.line))
+
+
+# Rough backport of warnings.catch_warnings from python 2.6,
+# with changes to support python 2.4.
+class CatchWarnings(object):
+    """A non-context manager version of warnings.catch_warnings.
+
+    The 'record' argument specifies whether warnings should be captured by a
+    custom implementation of warnings.showwarning() and be appended to a list
+    accessed through the `log` property. The objects appended to the list are
+    arguments whose attributes mirror the arguments to showwarning().
+
+    The 'module' argument is to specify an alternative module to the module
+    named 'warnings' and imported under that name. This argument is only useful
+    when testing the warnings module itself.
+    """
+
+    def __init__(self, record=False, module=None):
+        self._record = record
+        if module is None:
+            self._module = sys.modules['warnings']
+        else:
+            self._module = module
+
+        # No __enter__ so do that work here
+        self._filters = self._module.filters
+        self._module.filters = self._filters[:]
+        self._showwarning = self._module.showwarning
+        self._log = []
+        if self._record:
+            def showwarning(*args, **kwargs):
+                self._log.append(WarningMessage(*args, **kwargs))
+            self._module.showwarning = showwarning
+
+    @property
+    def log(self):
+        """A list of any warnings recorded when using record=True."""
+        return self._log
+
+    def __repr__(self):
+        args = []
+        if self._record:
+            args.append("record=True")
+        if self._module is not sys.modules['warnings']:
+            args.append("module=%r" % self._module)
+        name = type(self).__name__
+        return "%s(%s)" % (name, ", ".join(args))
+
+    def exit(self):
+        """Revert changes to the warnings module."""
+        self._module.filters = self._filters
+        self._module.showwarning = self._showwarning
+
+
+def catch_warnings(record=False, module=None):
+    """Helper for use with CatchWarnings."""
+    return CatchWarnings(record, module)