BUMP 2.9.5

This commit also cleans up the Evergreen config
file and fixes a test for MongoDB 3.5+.

.evergreen/install-dependencies.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+# Copy PyMongo's test certificates over driver-evergreen-tools'
+cp ${PROJECT_DIRECTORY}/test/certificates/* ${DRIVERS_TOOLS}/.evergreen/x509gen/
+
+# Replace MongoOrchestration's client certificate.
+cp ${PROJECT_DIRECTORY}/test/certificates/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem
+
+if [ -w /etc/hosts ]; then
+  SUDO=""
+else
+  SUDO="sudo"
+fi
+
+# Add 'server' and 'hostname_not_in_cert' as a hostnames
+echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts
+echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts

.evergreen/run-enterprise-auth-tests.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Don't trace to avoid secrets showing up in the logs
+set -o errexit
+
+echo "Running enterprise authentication tests"
+
+export JAVA_HOME=/opt/java/jdk8
+
+PYTHON_VERSION=$(${PYTHON_BINARY} -c 'import sys; sys.stdout.write(str(sys.version_info[0]))')
+PLATFORM="$(${PYTHON_BINARY} -c 'import platform, sys; sys.stdout.write(platform.system())')"
+
+export DB_USER="bob"
+export DB_PASSWORD="pwd123"
+
+# There is no kerberos package for Jython, but we do want to test PLAIN.
+if [ ${PLATFORM} != "Java" ]; then
+    # PyMongo 2.x doesn't support GSSAPI on Windows.
+    if [ "Windows_NT" != "$OS" ]; then
+        echo "Writing keytab"
+        echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab
+        echo "Running kinit"
+        kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL}
+        echo "Setting GSSAPI variables"
+        export GSSAPI_HOST=${SASL_HOST}
+        export GSSAPI_PORT=${SASL_PORT}
+        export GSSAPI_PRINCIPAL=${PRINCIPAL}
+    fi
+    EXTRA_ARGS=""
+else
+    # Keep Jython 2.5 from running out of memory.
+    EXTRA_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
+fi
+
+# Set verbose test output flag.
+if [ "$PYTHON_VERSION" = "3" ]; then
+    # With Python 3, the tests do not accept a "--verbosity=2" flag.
+    TEST_VERBOSITY="-v"
+else
+    # With Python 2, the tests accepts a "-v" flag but only "--verbosity=2"
+    # causes the verbose output we want.
+    TEST_VERBOSITY="--verbosity=2"
+fi
+
+echo "Running tests"
+${PYTHON_BINARY} setup.py clean
+${PYTHON_BINARY} $EXTRA_ARGS setup.py nosetests $TEST_VERBOSITY

.evergreen/run-mod-wsgi-tests.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -o xtrace
+set -o errexit
+
+APACHE=$(command -v apache2 || command -v /usr/lib/apache2/mpm-prefork/apache2) || true
+if [ -z "$APACHE" ]; then
+    echo "Could not find apache2 binary"
+    exit 1
+fi
+
+PYTHON_VERSION=$(${PYTHON_BINARY} -c "import sys; sys.stdout.write('.'.join(str(val) for val in sys.version_info[:2]))")
+
+if [ $MOD_WSGI_VERSION = "2.8" ] && [ $PYTHON_VERSION = "2.7" ]; then
+    # mod_wsgi 2.8 segfaults when built against the toolchain Python 2.7. Build
+    # against the system Python 2.7 instead.
+    git clone https://github.com/GrahamDumpleton/mod_wsgi.git
+    cd mod_wsgi
+    git checkout tags/2.8
+    ./configure
+    make
+    export MOD_WSGI_SO=$(pwd)/.libs/mod_wsgi.so
+    cd ..
+else
+    export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_version/$MOD_WSGI_VERSION/mod_wsgi.so
+    export PYTHONHOME=/opt/python/$PYTHON_VERSION
+fi
+
+cd ..
+$APACHE -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache22ubuntu1204.conf
+trap "$APACHE -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache22ubuntu1204.conf" EXIT HUP
+
+set +e
+wget -t 1 -T 10 -O - "http://localhost:8080${PROJECT_DIRECTORY}"
+STATUS=$?
+set -e
+
+# Debug
+cat error_log
+
+if [ $STATUS != 0 ]; then
+    exit $STATUS
+fi
+
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel http://localhost:8080${PROJECT_DIRECTORY}
+
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial http://localhost:8080${PROJECT_DIRECTORY}
+

.evergreen/run-tests.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+# Supported/used environment variables:
+#       AUTH                    Set to enable authentication. Defaults to "noauth"
+#       SSL                     Set to enable SSL. Defaults to "nossl"
+#       PYTHON_BINARY           The Python version to use. Defaults to whatever is available
+#       C_EXTENSIONS            Pass --no_ext to setup.py, or not.
+
+
+AUTH=${AUTH:-noauth}
+SSL=${SSL:-nossl}
+PYTHON_BINARY=${PYTHON_BINARY:-}
+C_EXTENSIONS=${C_EXTENSIONS:-}
+
+export JAVA_HOME=/opt/java/jdk8
+
+if [ "$AUTH" != "noauth" ]; then
+    export DB_USER="bob"
+    export DB_PASSWORD="pwd123"
+fi
+
+if [ -z "$PYTHON_BINARY" ]; then
+    PYTHON=$(command -v python || command -v python3) || true
+    if [ -z "$PYTHON" ]; then
+        echo "Cannot test without python or python3 installed!"
+        exit 1
+    fi
+else
+    PYTHON="$PYTHON_BINARY"
+fi
+
+PYTHON_VERSION=$($PYTHON -c 'import sys; sys.stdout.write(str(sys.version_info[0]))')
+PLATFORM=$($PYTHON -c 'import platform, sys; sys.stdout.write(platform.system())')
+
+if [ "$SSL" = "ssl" ]; then
+    if [ "$PYTHON_VERSION" = "3" ]; then
+        # We cannot pass arguments to the "nosetests" command with Python 3
+        # because of the hacks in setup.py to work around nose/2to3 usage.
+        # Instead, use the "test" command directly to run only test/test_ssl.py.
+        # Unfortunately, this does not produce XML output.
+        TEST_CMD="test --test-suite test.test_ssl"
+    else
+        # With Python 2 we use the nosetests command
+        # Run only test/test_ssl.py and produces nosetests.xml output.
+        TEST_CMD="nosetests --tests test/test_ssl.py"
+    fi
+else
+    # Run all the tests and produces nosetests.xml output.
+    TEST_CMD="nosetests"
+fi
+
+# Set verbose test output flag.
+if [ "$PYTHON_VERSION" = "3" ]; then
+    # With Python 3, the tests do not accept a "--verbosity=2" flag.
+    TEST_VERBOSITY="-v"
+else
+    # With Python 2, the tests accepts a "-v" flag but only "--verbosity=2"
+    # causes the verbose output we want.
+    TEST_VERBOSITY="--verbosity=2"
+fi
+
+if [ "$PLATFORM" = "Java" ]; then
+    # Keep Jython 2.5 from running out of memory.
+    EXTRA_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
+else
+    EXTRA_ARGS=""
+fi
+
+echo "Running $AUTH tests over $SSL with python $PYTHON"
+$PYTHON -c 'import sys; print(sys.version)'
+
+# Run the tests, and store the results in Evergreen compatible XUnit XML
+# files in the xunit-results/ directory.
+
+$PYTHON setup.py clean
+$PYTHON $EXTRA_ARGS setup.py $C_EXTENSIONS $TEST_CMD $TEST_VERBOSITY

.travis.yml

@@ -1,12 +1,15 @@
 language: python
 
 python:
-  - 2.5
   - 2.6
   - 2.7
   - 3.2
   - 3.3
+  - 3.4
+  - 3.5
+  - 3.6
   - pypy
+  - pypy3
 
 services:
   - mongodb
@@ -14,5 +17,7 @@ services:
 script: python setup.py test
 
 install:
-    #Temporary solution for Travis CI mutiprocessing issue #155
+    # Work around https://github.com/travis-ci/travis-ci/issues/943
     - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm
+    # Work around https://github.com/travis-ci/travis-ci/issues/5485
+    - travis_retry pip install setuptools==29.0.1

README.rst

@@ -5,6 +5,10 @@ PyMongo
 :Author: Mike Dirolf
 :Maintainer: Bernie Hackett <bernie@mongodb.com>
 
+**PyMongo 2.x is in maintenance mode. Support for new MongoDB
+features ended with the release of MongoDB 3.0 and PyMongo 2.8. Users are
+strongly encouraged to upgrade to PyMongo 3.x.**
+
 About
 =====
 
@@ -38,6 +42,24 @@ case in our issue management tool, JIRA:
 Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the
 Core Server (i.e. SERVER) project are **public**.
 
+How To Ask For Help
+-------------------
+
+Please include all of the following information when opening an issue:
+
+- Detailed steps to reproduce the problem, including full traceback, if possible.
+- The exact python version used, with patch level::
+
+  $ python -c "import sys; print(sys.version)"
+
+- The exact version of PyMongo used, with patch level::
+
+  $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())"
+
+- The operating system and version (e.g. Windows 7, OSX 10.8, ...)
+- Web framework or asynchronous network library used, if any, with version (e.g.
+  Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...)
+
 Security Vulnerabilities
 ------------------------
 
@@ -54,6 +76,10 @@ should be able to do **easy_install pymongo** to install
 PyMongo. Otherwise you can download the project source and do **python
 setup.py install** to install.
 
+Do **not** install the "bson" package. PyMongo comes with its own bson package;
+doing "easy_install bson" installs a third-party package that is incompatible
+with PyMongo.
+
 Dependencies
 ============
 
@@ -88,7 +114,7 @@ Here's a basic example (for more see the *examples* section of the docs):
   >>> db.my_collection.find_one()
   {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')}
   >>> for item in db.my_collection.find():
-  ...     print item["x"]
+  ...     print(item["x"])
   ...
   10
   8
@@ -96,7 +122,7 @@ Here's a basic example (for more see the *examples* section of the docs):
   >>> db.my_collection.create_index("x")
   u'x_1'
   >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING):
-  ...     print item["x"]
+  ...     print(item["x"])
   ...
   8
   10

bson/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ import sys
 from bson.binary import (Binary, OLD_UUID_SUBTYPE,
                          JAVA_LEGACY, CSHARP_LEGACY)
 from bson.code import Code
+from bson.codec_options import CodecOptions, DEFAULT_CODEC_OPTIONS
 from bson.dbref import DBRef
 from bson.errors import (InvalidBSON,
                          InvalidDocument,
@@ -89,8 +90,18 @@ BSONLON = b("\x12") # 64bit int
 BSONMIN = b("\xFF") # Min key
 BSONMAX = b("\x7F") # Max key
 
+_CODEC_OPTIONS_TYPE_ERROR = TypeError(
+    "codec_options must be an instance of bson.codec_options.CodecOptions")
 
-def _get_int(data, position, as_class=None,
+
+def _raise_unknown_type(element_type, element_name):
+    """Unknown type helper."""
+    raise InvalidBSON("Detected unknown BSON type %r for fieldname %r. Are "
+                      "you using the latest driver version?" % (
+                          element_type, element_name))
+
+
+def _get_int(data, position, name, as_class=None,
              tz_aware=False, uuid_subtype=OLD_UUID_SUBTYPE,
              compile_re=True, unsigned=False):
     format = unsigned and "I" or "i"
@@ -134,13 +145,15 @@ def _make_c_string(string, check_null=False):
                                     "UTF-8: %r" % string)
 
 
-def _get_number(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_number(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     num = struct.unpack("<d", data[position:position + 8])[0]
     position += 8
     return num, position
 
 
-def _get_string(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_string(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     length = struct.unpack("<i", data[position:position + 4])[0]
     if length <= 0 or (len(data) - position - 4) < length:
         raise InvalidBSON("invalid string length")
@@ -150,7 +163,8 @@ def _get_string(data, position, as_class, tz_aware, uuid_subtype, compile_re):
     return _get_c_string(data, position, length - 1)
 
 
-def _get_object(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_object(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     obj_size = struct.unpack("<i", data[position:position + 4])[0]
     if data[position + obj_size - 1:position + obj_size] != ZERO:
         raise InvalidBSON("bad eoo")
@@ -165,26 +179,43 @@ def _get_object(data, position, as_class, tz_aware, uuid_subtype, compile_re):
     return object, position
 
 
-def _get_array(data, position, as_class, tz_aware, uuid_subtype, compile_re):
-    obj, position = _get_object(data, position,
-                                as_class, tz_aware, uuid_subtype, compile_re)
+def _get_array(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
+    size = struct.unpack("<i", data[position:position + 4])[0]
+    end = position + size - 1
+    if data[end:end + 1] != ZERO:
+        raise InvalidBSON("bad eoo")
+
+    position += 4
+    end -= 1
     result = []
-    i = 0
-    while True:
+
+    # Avoid doing global and attibute lookups in the loop.
+    append = result.append
+    index = data.index
+    getter = _element_getter
+
+    while position < end:
+        element_type = data[position:position + 1]
+        # Just skip the keys.
+        position = index(ZERO, position) + 1
         try:
-            result.append(obj[str(i)])
-            i += 1
+            value, position = getter[element_type](
+                data, position, name,
+                as_class, tz_aware, uuid_subtype, compile_re)
         except KeyError:
-            break
-    return result, position
+            _raise_unknown_type(element_type, name)
+        append(value)
+    return result, position + 1
 
 
-def _get_binary(data, position, as_class, tz_aware, uuid_subtype, compile_re):
-    length, position = _get_int(data, position)
+def _get_binary(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
+    length, position = _get_int(data, position, name)
     subtype = ord(data[position:position + 1])
     position += 1
     if subtype == 2:
-        length2, position = _get_int(data, position)
+        length2, position = _get_int(data, position, name)
         if length2 != length - 4:
             raise InvalidBSON("invalid binary (st 2) - lengths don't match!")
         length = length2
@@ -210,20 +241,22 @@ def _get_binary(data, position, as_class, tz_aware, uuid_subtype, compile_re):
     return value, position
 
 
-def _get_oid(data, position, as_class=None,
+def _get_oid(data, position, name, as_class=None,
              tz_aware=False, uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True):
     value = ObjectId(data[position:position + 12])
     position += 12
     return value, position
 
 
-def _get_boolean(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_boolean(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     value = data[position:position + 1] == ONE
     position += 1
     return value, position
 
 
-def _get_date(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_date(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     millis = struct.unpack("<q", data[position:position + 8])[0]
     diff = millis % 1000
     seconds = (millis - diff) / 1000
@@ -235,27 +268,30 @@ def _get_date(data, position, as_class, tz_aware, uuid_subtype, compile_re):
     return dt.replace(microsecond=diff * 1000), position
 
 
-def _get_code(data, position, as_class, tz_aware, uuid_subtype, compile_re):
-    code, position = _get_string(data, position,
+def _get_code(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
+    code, position = _get_string(data, position, name,
                                  as_class, tz_aware, uuid_subtype, compile_re)
     return Code(code), position
 
 
 def _get_code_w_scope(
-        data, position, as_class, tz_aware, uuid_subtype, compile_re):
-    _, position = _get_int(data, position)
-    code, position = _get_string(data, position,
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
+    _, position = _get_int(data, position, name)
+    code, position = _get_string(data, position, name,
                                  as_class, tz_aware, uuid_subtype, compile_re)
-    scope, position = _get_object(data, position,
+    scope, position = _get_object(data, position, name,
                                   as_class, tz_aware, uuid_subtype, compile_re)
     return Code(code, scope), position
 
 
-def _get_null(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_null(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     return None, position
 
 
-def _get_regex(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_regex(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     pattern, position = _get_c_string(data, position)
     bson_flags, position = _get_c_string(data, position)
     bson_re = Regex(pattern, bson_flags)
@@ -265,21 +301,23 @@ def _get_regex(data, position, as_class, tz_aware, uuid_subtype, compile_re):
         return bson_re, position
 
 
-def _get_ref(data, position, as_class, tz_aware, uuid_subtype, compile_re):
-    collection, position = _get_string(data, position, as_class, tz_aware,
-                                       uuid_subtype, compile_re)
-    oid, position = _get_oid(data, position)
+def _get_ref(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
+    collection, position = _get_string(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re)
+    oid, position = _get_oid(data, position, name)
     return DBRef(collection, oid), position
 
 
 def _get_timestamp(
-        data, position, as_class, tz_aware, uuid_subtype, compile_re):
-    inc, position = _get_int(data, position, unsigned=True)
-    timestamp, position = _get_int(data, position, unsigned=True)
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
+    inc, position = _get_int(data, position, name, unsigned=True)
+    timestamp, position = _get_int(data, position, name, unsigned=True)
     return Timestamp(timestamp, inc), position
 
 
-def _get_long(data, position, as_class, tz_aware, uuid_subtype, compile_re):
+def _get_long(
+        data, position, name, as_class, tz_aware, uuid_subtype, compile_re):
     # Have to cast to long; on 32-bit unpack may return an int.
     # 2to3 will change long to int. That's fine since long doesn't
     # exist in python3.
@@ -307,8 +345,8 @@ _element_getter = {
     BSONINT: _get_int,  # number_int
     BSONTIM: _get_timestamp,
     BSONLON: _get_long, # Same as _get_int after 2to3 runs.
-    BSONMIN: lambda u, v, w, x, y, z: (MinKey(), v),
-    BSONMAX: lambda u, v, w, x, y, z: (MaxKey(), v)}
+    BSONMIN: lambda t, u, v, w, x, y, z: (MinKey(), u),
+    BSONMAX: lambda t, u, v, w, x, y, z: (MaxKey(), u)}
 
 
 def _element_to_dict(
@@ -316,8 +354,12 @@ def _element_to_dict(
     element_type = data[position:position + 1]
     position += 1
     element_name, position = _get_c_string(data, position)
-    value, position = _element_getter[element_type](
-        data, position, as_class, tz_aware, uuid_subtype, compile_re)
+    try:
+        func = _element_getter[element_type]
+    except KeyError:
+        _raise_unknown_type(element_type, element_name)
+    value, position = func(data, position, element_name,
+                           as_class, tz_aware, uuid_subtype, compile_re)
 
     return element_name, value, position
 
@@ -333,7 +375,10 @@ def _elements_to_dict(data, as_class, tz_aware, uuid_subtype, compile_re):
     return result
 
 def _bson_to_dict(data, as_class, tz_aware, uuid_subtype, compile_re):
-    obj_size = struct.unpack("<i", data[:4])[0]
+    try:
+        obj_size = struct.unpack("<i", data[:4])[0]
+    except struct.error, e:
+        raise InvalidBSON(str(e))
     length = len(data)
     if length < obj_size:
         raise InvalidBSON("objsize too large")
@@ -493,9 +538,9 @@ if _use_c:
     _dict_to_bson = _cbson._dict_to_bson
 
 
-
 def decode_all(data, as_class=dict,
-               tz_aware=True, uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True):
+               tz_aware=True, uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
+               codec_options=None):
     """Decode BSON data to multiple documents.
 
     `data` must be a string of concatenated, valid, BSON-encoded
@@ -507,6 +552,8 @@ def decode_all(data, as_class=dict,
         documents
       - `tz_aware` (optional): if ``True``, return timezone-aware
         :class:`~datetime.datetime` instances
+      - `uuid_subtype` (optional): The BSON representation to use for UUIDs.
+        See the :mod:`bson.binary` module for all options.
       - `compile_re` (optional): if ``False``, don't attempt to compile
         BSON regular expressions into Python regular expressions. Return
         instances of :class:`~bson.regex.Regex` instead. Can avoid
@@ -517,6 +564,12 @@ def decode_all(data, as_class=dict,
        Added `compile_re` option.
     .. versionadded:: 1.9
     """
+    if codec_options is not None:
+        if not isinstance(codec_options, CodecOptions):
+            raise _CODEC_OPTIONS_TYPE_ERROR
+        as_class = codec_options.document_class
+        tz_aware = codec_options.tz_aware
+        uuid_subtype = codec_options.uuid_representation
     docs = []
     position = 0
     end = len(data) - 1
@@ -542,6 +595,96 @@ if _use_c:
     decode_all = _cbson.decode_all
 
 
+def decode_iter(data, as_class=dict, tz_aware=True,
+                uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
+                codec_options=None):
+    """Decode BSON data to multiple documents as a generator.
+
+    Works similarly to the decode_all function, but yields one document at a
+    time.
+
+    `data` must be a string of concatenated, valid, BSON-encoded
+    documents.
+
+    :Parameters:
+      - `data`: BSON data
+      - `as_class` (optional): the class to use for the resulting
+        documents
+      - `tz_aware` (optional): if ``True``, return timezone-aware
+        :class:`~datetime.datetime` instances
+      - `uuid_subtype` (optional): The BSON representation to use for UUIDs.
+        See the :mod:`bson.binary` module for all options.
+      - `compile_re` (optional): if ``False``, don't attempt to compile
+        BSON regular expressions into Python regular expressions. Return
+        instances of
+        :class:`~bson.regex.Regex` instead. Can avoid
+        :exc:`~bson.errors.InvalidBSON` errors when receiving
+        Python-incompatible regular expressions, for example from
+        ``currentOp``
+
+    .. versionadded:: 2.8
+    """
+    if codec_options is not None:
+        if not isinstance(codec_options, CodecOptions):
+            raise _CODEC_OPTIONS_TYPE_ERROR
+        as_class = codec_options.document_class
+        tz_aware = codec_options.tz_aware
+        uuid_subtype = codec_options.uuid_representation
+    position = 0
+    end = len(data) - 1
+    while position < end:
+        obj_size = struct.unpack("<i", data[position:position + 4])[0]
+        elements = data[position:position + obj_size]
+        position += obj_size
+        yield _bson_to_dict(elements, as_class,
+                            tz_aware, uuid_subtype, compile_re)[0]
+
+
+def decode_file_iter(file_obj, as_class=dict, tz_aware=True,
+                     uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
+                     codec_options=None):
+    """Decode bson data from a file to multiple documents as a generator.
+
+    Works similarly to the decode_all function, but reads from the file object
+    in chunks and parses bson in chunks, yielding one document at a time.
+
+    :Parameters:
+      - `file_obj`: A file object containing BSON data.
+      - `as_class` (optional): the class to use for the resulting
+        documents
+      - `tz_aware` (optional): if ``True``, return timezone-aware
+        :class:`~datetime.datetime` instances
+      - `uuid_subtype` (optional): The BSON representation to use for UUIDs.
+        See the :mod:`bson.binary` module for all options.
+      - `compile_re` (optional): if ``False``, don't attempt to compile
+        BSON regular expressions into Python regular expressions. Return
+        instances of
+        :class:`~bson.regex.Regex` instead. Can avoid
+        :exc:`~bson.errors.InvalidBSON` errors when receiving
+        Python-incompatible regular expressions, for example from
+        ``currentOp``
+
+    .. versionadded:: 2.8
+    """
+    if codec_options is not None:
+        if not isinstance(codec_options, CodecOptions):
+            raise _CODEC_OPTIONS_TYPE_ERROR
+        as_class = codec_options.document_class
+        tz_aware = codec_options.tz_aware
+        uuid_subtype = codec_options.uuid_representation
+    while True:
+        # Read size of next object.
+        size_data = file_obj.read(4)
+        if len(size_data) == 0:
+            break  # Finished with file normaly.
+        elif len(size_data) != 4:
+            raise InvalidBSON("cut off in middle of objsize")
+        obj_size = struct.unpack("<i", size_data)[0] - 4
+        elements = size_data + file_obj.read(obj_size)
+        yield _bson_to_dict(elements, as_class,
+                            tz_aware, uuid_subtype, compile_re)[0]
+
+
 def is_valid(bson):
     """Check that the given string represents valid :class:`BSON` data.
 
@@ -568,7 +711,8 @@ class BSON(binary_type):
     """
 
     @classmethod
-    def encode(cls, document, check_keys=False, uuid_subtype=OLD_UUID_SUBTYPE):
+    def encode(cls, document, check_keys=False, uuid_subtype=OLD_UUID_SUBTYPE,
+               codec_options=None):
         """Encode a document to a new :class:`BSON` instance.
 
         A document can be any mapping type (like :class:`dict`).
@@ -584,13 +728,20 @@ class BSON(binary_type):
           - `check_keys` (optional): check if keys start with '$' or
             contain '.', raising :class:`~bson.errors.InvalidDocument` in
             either case
+          - `uuid_subtype` (optional): The BSON representation to use for
+            UUIDs. See the :mod:`bson.binary` module for all options.
 
         .. versionadded:: 1.9
         """
+        if codec_options is not None:
+            if not isinstance(codec_options, CodecOptions):
+                raise _CODEC_OPTIONS_TYPE_ERROR
+            uuid_subtype = codec_options.uuid_representation
         return cls(_dict_to_bson(document, check_keys, uuid_subtype))
 
     def decode(self, as_class=dict,
-               tz_aware=False, uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True):
+               tz_aware=False, uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
+               codec_options=None):
         """Decode this BSON data.
 
         The default type to use for the resultant document is
@@ -610,6 +761,8 @@ class BSON(binary_type):
             document
           - `tz_aware` (optional): if ``True``, return timezone-aware
             :class:`~datetime.datetime` instances
+          - `uuid_subtype` (optional): The BSON representation to use for
+            UUIDs. See the :mod:`bson.binary` module for all options.
           - `compile_re` (optional): if ``False``, don't attempt to compile
             BSON regular expressions into Python regular expressions. Return
             instances of
@@ -622,6 +775,12 @@ class BSON(binary_type):
            Added ``compile_re`` option.
         .. versionadded:: 1.9
         """
+        if codec_options is not None:
+            if not isinstance(codec_options, CodecOptions):
+                raise _CODEC_OPTIONS_TYPE_ERROR
+            as_class = codec_options.document_class
+            tz_aware = codec_options.tz_aware
+            uuid_subtype = codec_options.uuid_representation
         (document, _) = _bson_to_dict(
             self, as_class, tz_aware, uuid_subtype, compile_re)
 

bson/_cbsonmodule.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -554,6 +554,7 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer,
                                     unsigned char check_keys,
                                     unsigned char uuid_subtype) {
     struct module_state *state = GETSTATE(self);
+    PyObject* type_marker = NULL;
 
     /*
      * Don't use PyObject_IsInstance for our custom types. It causes
@@ -561,26 +562,32 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer,
      * have a _type_marker attribute, which we can switch on instead.
      */
     if (PyObject_HasAttrString(value, "_type_marker")) {
-        long type;
-        PyObject* type_marker = PyObject_GetAttrString(value, "_type_marker");
-        if (type_marker == NULL)
+        type_marker = PyObject_GetAttrString(value, "_type_marker");
+        if (type_marker == NULL) {
             return 0;
+        }
+    }
+    /*
+     * Python objects with broken __getattr__ implementations could return
+     * arbitrary types for a call to PyObject_GetAttrString. For example
+     * pymongo.database.Database returns a new Collection instance for
+     * __getattr__ calls with names that don't match an existing attribute
+     * or method. In some cases "value" could be a subtype of something
+     * we know how to serialize. Make a best effort to encode these types.
+     */
 #if PY_MAJOR_VERSION >= 3
-        type = PyLong_AsLong(type_marker);
+    if (type_marker && PyLong_CheckExact(type_marker)) {
+        long type = PyLong_AsLong(type_marker);
 #else
-        type = PyInt_AsLong(type_marker);
+    if (type_marker && PyInt_CheckExact(type_marker)) {
+        long type = PyInt_AsLong(type_marker);
 #endif
         Py_DECREF(type_marker);
         /* 
          * Py(Long|Int)_AsLong returns -1 for error but -1 is a valid value
          * so we call PyErr_Occurred to differentiate.
-         *
-         * One potential reason for an error is the user passing an invalid
-         * type that overrides __getattr__ (e.g. pymongo.collection.Collection)
          */
         if (type == -1 && PyErr_Occurred()) {
-            PyErr_Clear();
-            _set_cannot_encode(value);
             return 0;
         }
         switch (type) {
@@ -792,6 +799,8 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer,
                 return 1;
             }
         }
+    } else {
+        Py_XDECREF(type_marker);
     }
 
     /* No _type_marker attibute or not one of our types. */
@@ -1369,6 +1378,9 @@ int write_dict(PyObject* self, buffer_t buffer,
         Py_DECREF(key);
     }
     Py_DECREF(iter);
+    if (PyErr_Occurred()) {
+        return 0;
+    }
 
     /* write null byte and fill in length */
     if (!buffer_write_bytes(buffer, &zero, 1)) {
@@ -1415,10 +1427,10 @@ static PyObject* _cbson_dict_to_bson(PyObject* self, PyObject* args) {
     return result;
 }
 
-static PyObject* get_value(PyObject* self, const char* buffer, unsigned* position,
-                           unsigned char type, unsigned max, PyObject* as_class,
-                           unsigned char tz_aware, unsigned char uuid_subtype,
-                           unsigned char compile_re) {
+static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer,
+                           unsigned* position, unsigned char type, unsigned max,
+                           PyObject* as_class, unsigned char tz_aware,
+                           unsigned char uuid_subtype, unsigned char compile_re) {
     struct module_state *state = GETSTATE(self);
 
     PyObject* value = NULL;
@@ -1562,7 +1574,7 @@ static PyObject* get_value(PyObject* self, const char* buffer, unsigned* positio
                     Py_DECREF(value);
                     goto invalid;
                 }
-                to_append = get_value(self, buffer, position, bson_type,
+                to_append = get_value(self, name, buffer, position, bson_type,
                                       max - (unsigned)key_size,
                                       as_class, tz_aware, uuid_subtype,
                                       compile_re);
@@ -1775,7 +1787,7 @@ static PyObject* get_value(PyObject* self, const char* buffer, unsigned* positio
                 Py_DECREF(args);
                 goto invalid;
             }
-            utc_type = _get_object(state->UTC, "bson.tz_util", "UTC");
+            utc_type = _get_object(state->UTC, "bson.tz_util", "utc");
             if (!utc_type || PyDict_SetItemString(kwargs, "tzinfo", utc_type) == -1) {
                 Py_DECREF(replace);
                 Py_DECREF(args);
@@ -2066,11 +2078,50 @@ static PyObject* get_value(PyObject* self, const char* buffer, unsigned* positio
         }
     default:
         {
-            PyObject* InvalidDocument = _error("InvalidDocument");
-            if (InvalidDocument) {
-                PyErr_SetString(InvalidDocument,
-                                "no c decoder for this type yet");
-                Py_DECREF(InvalidDocument);
+            PyObject* InvalidBSON = _error("InvalidBSON");
+            if (InvalidBSON) {
+#if PY_MAJOR_VERSION >= 3
+                PyObject* type_obj = PyBytes_FromFormat("%c", type);
+#else
+                PyObject* type_obj = PyString_FromFormat("%c", type);
+#endif
+                if (type_obj) {
+                    PyObject* type_repr = PyObject_Repr(type_obj);
+                    Py_DECREF(type_obj);
+                    if (type_repr) {
+                        PyObject* errmsg = NULL;
+#if PY_MAJOR_VERSION >= 3
+                        PyObject* left = PyUnicode_FromString(
+                            "Detected unknown BSON type ");
+                        if (left) {
+                            PyObject* lmsg = PyUnicode_Concat(left, type_repr);
+                            Py_DECREF(left);
+                            if (lmsg) {
+                                errmsg = PyUnicode_FromFormat(
+                                    "%U for fieldname '%U'. Are you using the "
+                                    "latest driver version?", lmsg, name);
+                                Py_DECREF(lmsg);
+                            }
+                        }
+#else
+                        PyObject* name_repr = PyObject_Repr(name);
+                        if (name_repr) {
+                            errmsg = PyString_FromFormat(
+                                "Detected unknown BSON type %s for fieldname %s."
+                                " Are you using the latest driver version?",
+                                PyString_AS_STRING(type_repr),
+                                PyString_AS_STRING(name_repr));
+                            Py_DECREF(name_repr);
+                        }
+#endif
+                        Py_DECREF(type_repr);
+                        if (errmsg) {
+                            PyErr_SetObject(InvalidBSON, errmsg);
+                            Py_DECREF(errmsg);
+                        }
+                    }
+                }
+                Py_DECREF(InvalidBSON);
             }
             goto invalid;
         }
@@ -2093,27 +2144,29 @@ static PyObject* get_value(PyObject* self, const char* buffer, unsigned* positio
          * Calling _error clears the error state, so fetch it first.
          */
         PyErr_Fetch(&etype, &evalue, &etrace);
-        InvalidBSON = _error("InvalidBSON");
-        if (InvalidBSON) {
-            if (!PyErr_GivenExceptionMatches(etype, InvalidBSON)) {
-                /*
-                 * Raise InvalidBSON(str(e)).
-                 */
-                Py_DECREF(etype);
-                etype = InvalidBSON;
+        if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) {
+            InvalidBSON = _error("InvalidBSON");
+            if (InvalidBSON) {
+                if (!PyErr_GivenExceptionMatches(etype, InvalidBSON)) {
+                    /*
+                     * Raise InvalidBSON(str(e)).
+                     */
+                    Py_DECREF(etype);
+                    etype = InvalidBSON;
 
-                if (evalue) {
-                    PyObject *msg = PyObject_Str(evalue);
-                    Py_DECREF(evalue);
-                    evalue = msg;
+                    if (evalue) {
+                        PyObject *msg = PyObject_Str(evalue);
+                        Py_DECREF(evalue);
+                        evalue = msg;
+                    }
+                    PyErr_NormalizeException(&etype, &evalue, &etrace);
+                } else {
+                    /*
+                     * The current exception matches InvalidBSON, so we don't
+                     * need this reference after all.
+                     */
+                    Py_DECREF(InvalidBSON);
                 }
-                PyErr_NormalizeException(&etype, &evalue, &etrace);
-            } else {
-                /*
-                 * The current exception matches InvalidBSON, so we don't need
-                 * this reference after all.
-                 */
-                Py_DECREF(InvalidBSON);
             }
         }
         /* Steals references to args. */
@@ -2159,7 +2212,7 @@ static PyObject* _elements_to_dict(PyObject* self, const char* string,
             return NULL;
         }
         position += (unsigned)name_length + 1;
-        value = get_value(self, string, &position, type,
+        value = get_value(self, name, string, &position, type,
                           max - position, as_class, tz_aware, uuid_subtype,
                           compile_re);
         if (!value) {
@@ -2290,6 +2343,7 @@ static PyObject* _cbson_bson_to_dict(PyObject* self, PyObject* args) {
 }
 
 static PyObject* _cbson_decode_all(PyObject* self, PyObject* args) {
+    PyObject* options = Py_None;
     int size;
     Py_ssize_t total_size;
     const char* string;
@@ -2302,11 +2356,19 @@ static PyObject* _cbson_decode_all(PyObject* self, PyObject* args) {
     unsigned char compile_re = 1;
 
     if (!PyArg_ParseTuple(
-            args, "O|Obbb",
-            &bson, &as_class, &tz_aware, &uuid_subtype, &compile_re)) {
+            args, "O|ObbbO",
+            &bson, &as_class, &tz_aware, &uuid_subtype, &compile_re,
+            &options)) {
         return NULL;
     }
 
+    if (options != Py_None) {
+        if (!PyArg_ParseTuple(options, "Obb",
+                              &as_class, &tz_aware, &uuid_subtype)) {
+            return NULL;
+        }
+    }
+
 #if PY_MAJOR_VERSION >= 3
     if (!PyBytes_Check(bson)) {
         PyErr_SetString(PyExc_TypeError, "argument to decode_all must be a bytes object");

bson/_cbsonmodule.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,12 +26,12 @@ typedef int Py_ssize_t;
 #define PY_SSIZE_T_MIN INT_MIN
 #endif
 
-#if defined(WIN32) || defined(_MSC_VER)
+#ifdef _MSC_VER
 /*
  * This macro is basically an implementation of asprintf for win32
  * We print to the provided buffer to get the string value as an int.
  */
-#if defined(_MSC_VER) && (_MSC_VER >= 1400)
+#if _MSC_VER >= 1400
 #define INT2STRING(buffer, i)                                       \
     _snprintf_s((buffer),                                           \
                  _scprintf("%d", (i)) + 1,                          \

bson/binary.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -67,29 +67,53 @@ change to this in a future release.
 .. versionadded:: 1.5
 """
 
-JAVA_LEGACY = 5
-"""Used with :attr:`pymongo.collection.Collection.uuid_subtype`
-to specify that UUIDs should be stored in the legacy byte order
-used by the Java driver.
+STANDARD = UUID_SUBTYPE
+"""The standard UUID representation.
 
-:class:`uuid.UUID` instances will automatically be encoded
-by :mod:`bson` using :data:`OLD_UUID_SUBTYPE`.
+:class:`uuid.UUID` instances will automatically be encoded to
+and decoded from BSON binary, using RFC-4122 byte order with
+binary subtype :data:`UUID_SUBTYPE`.
+
+.. versionadded:: 2.9
+"""
+
+PYTHON_LEGACY = OLD_UUID_SUBTYPE
+"""The Python legacy UUID representation.
+
+:class:`uuid.UUID` instances will automatically be encoded to
+and decoded from BSON binary, using RFC-4122 byte order with
+binary subtype :data:`OLD_UUID_SUBTYPE`.
+
+.. versionadded:: 2.9
+"""
+
+JAVA_LEGACY = 5
+"""The Java legacy UUID representation.
+
+:class:`uuid.UUID` instances will automatically be encoded to
+and decoded from BSON binary, using the Java driver's legacy
+byte order with binary subtype :data:`OLD_UUID_SUBTYPE`.
 
 .. versionadded:: 2.3
 """
 
 CSHARP_LEGACY = 6
-"""Used with :attr:`pymongo.collection.Collection.uuid_subtype`
-to specify that UUIDs should be stored in the legacy byte order
-used by the C# driver.
+"""The C#/.net legacy UUID representation.
 
-:class:`uuid.UUID` instances will automatically be encoded
-by :mod:`bson` using :data:`OLD_UUID_SUBTYPE`.
+:class:`uuid.UUID` instances will automatically be encoded to
+and decoded from BSON binary, using the C# driver's legacy
+byte order and binary subtype :data:`OLD_UUID_SUBTYPE`.
 
 .. versionadded:: 2.3
 """
 
 ALL_UUID_SUBTYPES = (OLD_UUID_SUBTYPE, UUID_SUBTYPE, JAVA_LEGACY, CSHARP_LEGACY)
+ALL_UUID_REPRESENTATIONS = (STANDARD, PYTHON_LEGACY, JAVA_LEGACY, CSHARP_LEGACY)
+UUID_REPRESENTATION_NAMES = {
+    PYTHON_LEGACY: 'PYTHON_LEGACY',
+    STANDARD: 'STANDARD',
+    JAVA_LEGACY: 'JAVA_LEGACY',
+    CSHARP_LEGACY: 'CSHARP_LEGACY'}
 
 MD5_SUBTYPE = 5
 """BSON binary subtype for an MD5 hash.
@@ -163,6 +187,9 @@ class Binary(binary_type):
         # subclass of str...
         return False
 
+    def __hash__(self):
+        return super(Binary, self).__hash__() ^ hash(self.__subtype)
+
     def __ne__(self, other):
         return not self == other
 

bson/buffer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

bson/buffer.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

bson/code.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -76,5 +76,7 @@ class Code(str):
             return (self.__scope, str(self)) == (other.__scope, str(other))
         return False
 
+    __hash__ = None
+
     def __ne__(self, other):
         return not self == other

bson/codec_options.py

@@ -0,0 +1,77 @@
+# Copyright 2014-2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tools for specifying BSON codec options."""
+
+from bson.binary import (ALL_UUID_REPRESENTATIONS,
+                         PYTHON_LEGACY,
+                         UUID_REPRESENTATION_NAMES)
+
+
+class CodecOptions(tuple):
+    """Encapsulates BSON options used in CRUD operations.
+
+    :Parameters:
+      - `document_class`: BSON documents returned in queries will be decoded
+        to an instance of this class.
+      - `tz_aware`: If ``True``, BSON datetimes will be decoded to timezone
+        aware instances of :class:`~datetime.datetime`. Otherwise they will be
+        naive. Defaults to ``False``.
+      - `uuid_representation`: The BSON representation to use when encoding
+        and decoding instances of :class:`~uuid.UUID`. Defaults to
+        :data:`~bson.binary.PYTHON_LEGACY`.
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, document_class=dict,
+                tz_aware=False, uuid_representation=PYTHON_LEGACY):
+        if not isinstance(tz_aware, bool):
+            raise TypeError("tz_aware must be True or False")
+        if uuid_representation not in ALL_UUID_REPRESENTATIONS:
+            raise ValueError("uuid_representation must be a value "
+                             "from bson.binary.ALL_UUID_REPRESENTATIONS")
+
+        return tuple.__new__(
+            cls, (document_class, tz_aware, uuid_representation))
+
+    def __repr__(self):
+        if self.document_class is dict:
+            document_class_repr = 'dict'
+        else:
+            document_class_repr = repr(self.document_class)
+
+        uuid_rep_repr = UUID_REPRESENTATION_NAMES.get(self.uuid_representation,
+                                                      self.uuid_representation)
+
+        return (
+            'CodecOptions(document_class=%s, tz_aware=%r, uuid_representation='
+            '%s)' % (document_class_repr, self.tz_aware, uuid_rep_repr))
+
+    def __getnewargs__(self):
+        return tuple(self)
+
+    @property
+    def document_class(self):
+        return self[0]
+
+    @property
+    def tz_aware(self):
+        return self[1]
+
+    @property
+    def uuid_representation(self):
+        return self[2]
+
+DEFAULT_CODEC_OPTIONS = CodecOptions()

bson/dbref.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

bson/encoding_helpers.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

bson/encoding_helpers.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

bson/errors.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

bson/json_util.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@ Example usage (serialization):
    >>> dumps([{'foo': [1, 2]},
    ...        {'bar': {'hello': 'world'}},
    ...        {'code': Code("function x() { return 1; }")},
-   ...        {'bin': Binary("\x01\x02\x03\x04")}])
+   ...        {'bin': Binary(b"\x01\x02\x03\x04")}])
    '[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }", "$scope": {}}}, {"bin": {"$binary": "AQIDBA==", "$type": "00"}}]'
 
 Example usage (deserialization):
@@ -47,6 +47,16 @@ It won't handle :class:`~bson.binary.Binary` and :class:`~bson.code.Code`
 instances (as they are extended strings you can't provide custom defaults),
 but it will be faster as there is less recursion.
 
+.. versionchanged:: 2.8
+   The output format for :class:`~bson.timestamp.Timestamp` has changed from
+   '{"t": <int>, "i": <int>}' to '{"$timestamp": {"t": <int>, "i": <int>}}'.
+   This new format will be decoded to an instance of
+   :class:`~bson.timestamp.Timestamp`. The old format will continue to be
+   decoded to a python dict as before. Encoding to the old format is no longer
+   supported as it was never correct and loses type information.
+   Added support for $numberLong and $undefined - new in MongoDB 2.6 - and
+   parsing $date in ISO-8601 format.
+
 .. versionchanged:: 2.7
    Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
    instances. (But not in Python 2.4.)
@@ -76,6 +86,7 @@ import base64
 import calendar
 import datetime
 import re
+import time
 
 json_lib = True
 try:
@@ -96,6 +107,7 @@ from bson.min_key import MinKey
 from bson.objectid import ObjectId
 from bson.regex import Regex
 from bson.timestamp import Timestamp
+from bson.tz_util import utc
 
 from bson.py3compat import PY3, binary_type, string_types
 
@@ -166,7 +178,39 @@ def object_hook(dct, compile_re=True):
     if "$ref" in dct:
         return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None))
     if "$date" in dct:
-        secs = float(dct["$date"]) / 1000.0
+        dtm = dct["$date"]
+        # mongoexport 2.6 and newer
+        if isinstance(dtm, basestring):
+            # datetime.datetime.strptime is new in python 2.5
+            naive = datetime.datetime(
+                *(time.strptime(dtm[:19], "%Y-%m-%dT%H:%M:%S")[0:6]))
+            # The %f format is new in python 2.6
+            micros = int(dtm[20:23]) * 1000
+            aware = naive.replace(microsecond=micros, tzinfo=utc)
+            offset = dtm[23:]
+            if not offset or offset == 'Z':
+                # UTC
+                return aware
+            else:
+                if len(offset) == 5:
+                    # Offset from mongoexport is in format (+|-)HHMM
+                    secs = (int(offset[1:3]) * 3600 + int(offset[3:]) * 60)
+                elif ':' in offset and len(offset) == 6:
+                    # RFC-3339 format (+|-)HH:MM
+                    hours, minutes = offset[1:].split(':')
+                    secs = (int(hours) * 3600 + int(minutes) * 60)
+                else:
+                    # Not RFC-3339 compliant or mongoexport output.
+                    raise ValueError("invalid format for offset")
+                if offset[0] == "-":
+                    secs *= -1
+                return aware - datetime.timedelta(seconds=secs)
+        # mongoexport 2.6 and newer, time before the epoch (SERVER-15275)
+        elif isinstance(dtm, dict):
+            secs = float(dtm["$numberLong"]) / 1000.0
+        # mongoexport before 2.6
+        else:
+            secs = float(dtm) / 1000.0
         return EPOCH_AWARE + datetime.timedelta(seconds=secs)
     if "$regex" in dct:
         flags = 0
@@ -193,6 +237,15 @@ def object_hook(dct, compile_re=True):
         return Code(dct["$code"], dct.get("$scope"))
     if bson.has_uuid() and "$uuid" in dct:
         return bson.uuid.UUID(dct["$uuid"])
+    if "$undefined" in dct:
+        return None
+    if "$numberLong" in dct:
+        # 2to3 will change this to int. PyMongo 3.0 supports
+        # a new type, Int64, to avoid round trip issues.
+        return long(dct["$numberLong"])
+    if "$timestamp" in dct:
+        tsp = dct["$timestamp"]
+        return Timestamp(tsp["t"], tsp["i"])
     return dct
 
 
@@ -240,7 +293,7 @@ def default(obj):
     if isinstance(obj, MaxKey):
         return {"$maxKey": 1}
     if isinstance(obj, Timestamp):
-        return SON([("t", obj.time), ("i", obj.inc)])
+        return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])}
     if isinstance(obj, Code):
         return SON([('$code', str(obj)), ('$scope', obj.scope)])
     if isinstance(obj, Binary):

bson/max_key.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2014 MongoDB, Inc.
+# Copyright 2010-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,6 +28,9 @@ class MaxKey(object):
     def __eq__(self, other):
         return isinstance(other, MaxKey)
 
+    def __hash__(self):
+        return hash(self._type_marker)
+
     def __ne__(self, other):
         return not self == other
 

bson/min_key.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2014 MongoDB, Inc.
+# Copyright 2010-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,6 +28,9 @@ class MinKey(object):
     def __eq__(self, other):
         return isinstance(other, MinKey)
 
+    def __hash__(self):
+        return hash(self._type_marker)
+
     def __ne__(self, other):
         return not self == other
     

bson/objectid.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -54,6 +54,13 @@ def _machine_bytes():
     return machine_hash.digest()[0:3]
 
 
+def _raise_invalid_id(oid):
+    raise InvalidId(
+        "%r is not a valid ObjectId, it must be a 12-byte input"
+        " of type %r or a 24-character hex string" % (
+            oid, binary_type.__name__))
+
+
 class ObjectId(object):
     """A MongoDB ObjectId.
     """
@@ -70,19 +77,41 @@ class ObjectId(object):
     def __init__(self, oid=None):
         """Initialize a new ObjectId.
 
-        If `oid` is ``None``, create a new (unique) ObjectId. If `oid`
-        is an instance of (:class:`basestring` (:class:`str` or :class:`bytes`
-        in python 3), :class:`ObjectId`) validate it and use that.  Otherwise,
-        a :class:`TypeError` is raised. If `oid` is invalid,
-        :class:`~bson.errors.InvalidId` is raised.
+        An ObjectId is a 12-byte unique identifier consisting of:
+
+          - a 4-byte value representing the seconds since the Unix epoch,
+          - a 3-byte machine identifier,
+          - a 2-byte process id, and
+          - a 3-byte counter, starting with a random value.
+
+        By default, ``ObjectId()`` creates a new unique identifier. The
+        optional parameter `oid` can be an :class:`ObjectId`, or any 12
+        :class:`bytes` or, in Python 2, any 12-character :class:`str`.
+
+        For example, the 12 bytes b'foo-bar-quux' do not follow the ObjectId
+        specification but they are acceptable input::
+
+          >>> ObjectId(b'foo-bar-quux')
+          ObjectId('666f6f2d6261722d71757578')
+
+        `oid` can also be a :class:`unicode` or :class:`str` of 24 hex digits::
+
+          >>> ObjectId('0123456789ab0123456789ab')
+          ObjectId('0123456789ab0123456789ab')
+          >>>
+          >>> # A u-prefixed unicode literal:
+          >>> ObjectId(u'0123456789ab0123456789ab')
+          ObjectId('0123456789ab0123456789ab')
+
+        Raises :class:`~bson.errors.InvalidId` if `oid` is not 12 bytes nor
+        24 hex digits, or :class:`TypeError` if `oid` is not an accepted type.
 
         :Parameters:
-          - `oid` (optional): a valid ObjectId (12 byte binary or 24 character
-            hex string)
+          - `oid` (optional): a valid ObjectId.
 
         .. versionadded:: 1.2.1
            The `oid` parameter can be a ``unicode`` instance (that contains
-           only hexadecimal digits).
+           24 hexadecimal digits).
 
         .. mongodoc:: objectids
         """
@@ -140,6 +169,9 @@ class ObjectId(object):
 
         .. versionadded:: 2.3
         """
+        if not oid:
+            return False
+
         try:
             ObjectId(oid)
             return True
@@ -186,14 +218,14 @@ class ObjectId(object):
                 if isinstance(oid, binary_type):
                     self.__id = oid
                 else:
-                    raise InvalidId("%s is not a valid ObjectId" % oid)
+                    _raise_invalid_id(oid)
             elif len(oid) == 24:
                 try:
                     self.__id = bytes_from_hex(oid)
                 except (TypeError, ValueError):
-                    raise InvalidId("%s is not a valid ObjectId" % oid)
+                    _raise_invalid_id(oid)
             else:
-                raise InvalidId("%s is not a valid ObjectId" % oid)
+                _raise_invalid_id(oid)
         else:
             raise TypeError("id must be an instance of (%s, %s, ObjectId), "
                             "not %s" % (binary_type.__name__,

bson/py3compat.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

bson/regex.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2014 MongoDB, Inc.
+# Copyright 2013-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -104,6 +104,8 @@ class Regex(object):
         else:
             return NotImplemented
 
+    __hash__ = None
+
     def __ne__(self, other):
         return not self == other
 

bson/son.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -100,7 +100,7 @@ class SON(dict):
         return "SON([%s])" % ", ".join(result)
 
     def __setitem__(self, key, value):
-        if key not in self:
+        if key not in self.__keys:
             self.__keys.append(key)
         dict.__setitem__(self, key, value)
 
@@ -120,14 +120,11 @@ class SON(dict):
     # efficient.
     # second level definitions support higher levels
     def __iter__(self):
-        for k in self.keys():
+        for k in self.__keys:
             yield k
 
     def has_key(self, key):
-        return key in self.keys()
-
-    def __contains__(self, key):
-        return key in self.keys()
+        return key in self.__keys
 
     # third level takes advantage of second level definitions
     def iteritems(self):
@@ -149,8 +146,8 @@ class SON(dict):
         return [(key, self[key]) for key in self]
 
     def clear(self):
-        for key in self.keys():
-            del self[key]
+        self.__keys = []
+        super(SON, self).clear()
 
     def setdefault(self, key, default=None):
         try:
@@ -214,7 +211,7 @@ class SON(dict):
         return not self == other
 
     def __len__(self):
-        return len(self.keys())
+        return len(self.__keys)
 
     def to_dict(self):
         """Convert a SON document to a normal Python dictionary instance.
@@ -226,12 +223,12 @@ class SON(dict):
         def transform_value(value):
             if isinstance(value, list):
                 return [transform_value(v) for v in value]
-            if isinstance(value, SON):
-                value = dict(value)
-            if isinstance(value, dict):
-                for k, v in value.iteritems():
-                    value[k] = transform_value(v)
-            return value
+            elif isinstance(value, dict):
+                return dict([
+                    (k, transform_value(v))
+                    for k, v in value.iteritems()])
+            else:
+                return value
 
         return transform_value(dict(self))
 

bson/time64.c

@@ -43,12 +43,10 @@ gmtime64_r() is a 64-bit equivalent of gmtime_r().
     #define _CRT_SECURE_NO_WARNINGS
 #endif
 
-#include <assert.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
+/* Including Python.h fixes issues with interpreters built with -std=c99. */
+#include "Python.h"
+
 #include <time.h>
-#include <errno.h>
 #include "time64.h"
 #include "time64_limits.h"
 
@@ -112,7 +110,7 @@ static const int safe_years_low[SOLAR_CYCLE_LENGTH] = {
 #define CHEAT_YEARS 108
 
 #define IS_LEAP(n)      ((!(((n) + 1900) % 400) || (!(((n) + 1900) % 4) && (((n) + 1900) % 100))) != 0)
-#define WRAP(a,b,m)     ((a) = ((a) <  0  ) ? ((b)--, (a) + (m)) : (a))
+#define _TIME64_WRAP(a,b,m)     ((a) = ((a) <  0  ) ? ((b)--, (a) + (m)) : (a))
 
 #ifdef USE_SYSTEM_LOCALTIME
 #    define SHOULD_USE_SYSTEM_LOCALTIME(a)  (       \
@@ -543,6 +541,7 @@ struct TM *gmtime64_r (const Time64_T *in_time, struct TM *p)
 
     assert(p != NULL);
 
+#ifdef USE_SYSTEM_GMTIME
     /* Use the system gmtime() if time_t is small enough */
     if( SHOULD_USE_SYSTEM_GMTIME(*in_time) ) {
         time_t safe_time = (time_t)*in_time;
@@ -554,6 +553,7 @@ struct TM *gmtime64_r (const Time64_T *in_time, struct TM *p)
 
         return p;
     }
+#endif
 
 #ifdef HAS_TM_TM_GMTOFF
     p->tm_gmtoff = 0;
@@ -572,9 +572,9 @@ struct TM *gmtime64_r (const Time64_T *in_time, struct TM *p)
     time /= 24;
     v_tm_tday = time;
 
-    WRAP (v_tm_sec, v_tm_min, 60);
-    WRAP (v_tm_min, v_tm_hour, 60);
-    WRAP (v_tm_hour, v_tm_tday, 24);
+    _TIME64_WRAP (v_tm_sec, v_tm_min, 60);
+    _TIME64_WRAP (v_tm_min, v_tm_hour, 60);
+    _TIME64_WRAP (v_tm_hour, v_tm_tday, 24);
 
     v_tm_wday = (int)((v_tm_tday + 4) % 7);
     if (v_tm_wday < 0)
@@ -668,6 +668,7 @@ struct TM *localtime64_r (const Time64_T *time, struct TM *local_tm)
 
     assert(local_tm != NULL);
 
+#ifdef USE_SYSTEM_LOCALTIME
     /* Use the system localtime() if time_t is small enough */
     if( SHOULD_USE_SYSTEM_LOCALTIME(*time) ) {
         safe_time = (time_t)*time;
@@ -681,6 +682,7 @@ struct TM *localtime64_r (const Time64_T *time, struct TM *local_tm)
 
         return local_tm;
     }
+#endif
 
     if( gmtime64_r(time, &gm_tm) == NULL ) {
         TIME64_TRACE1("gmtime64_r returned null for %lld\n", *time);

bson/timestamp.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2014 MongoDB, Inc.
+# Copyright 2010-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ from bson.tz_util import utc
 
 UPPERBOUND = 4294967296
 
+
 class Timestamp(object):
     """MongoDB internal timestamps used in the opLog.
     """
@@ -83,6 +84,9 @@ class Timestamp(object):
         else:
             return NotImplemented
 
+    def __hash__(self):
+        return hash(self.time) ^ hash(self.inc)
+
     def __ne__(self, other):
         return not self == other
 

bson/tz_util.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2014 MongoDB, Inc.
+# Copyright 2010-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

doc/api/bson/binary.rst

@@ -9,6 +9,8 @@
    .. autodata:: OLD_BINARY_SUBTYPE
    .. autodata:: OLD_UUID_SUBTYPE
    .. autodata:: UUID_SUBTYPE
+   .. autodata:: STANDARD
+   .. autodata:: PYTHON_LEGACY
    .. autodata:: JAVA_LEGACY
    .. autodata:: CSHARP_LEGACY
    .. autodata:: MD5_SUBTYPE

doc/api/bson/codec_options.rst

@@ -0,0 +1,6 @@
+:mod:`codec_options` -- Tools for specifying BSON codec options
+===============================================================
+
+.. automodule:: bson.codec_options
+   :synopsis: Tools for specifying BSON codec options.
+   :members:

doc/api/bson/index.rst

@@ -11,14 +11,15 @@ Sub-modules:
    :maxdepth: 2
 
    binary
-   regex
    code
+   codec_options
    dbref
    errors
    json_util
    max_key
    min_key
    objectid
+   regex
    son
    timestamp
    tz_util

doc/api/pymongo/collection.rst

@@ -10,6 +10,7 @@
    .. autodata:: pymongo.GEOHAYSTACK
    .. autodata:: pymongo.GEOSPHERE
    .. autodata:: pymongo.HASHED
+   .. autodata:: pymongo.TEXT
 
    .. autoclass:: pymongo.collection.Collection(database, name[, create=False[, **kwargs]]])
 
@@ -23,11 +24,13 @@
       .. autoattribute:: full_name
       .. autoattribute:: name
       .. autoattribute:: database
+      .. autoattribute:: codec_options
       .. autoattribute:: read_preference
       .. autoattribute:: tag_sets
       .. autoattribute:: secondary_acceptable_latency_ms
       .. autoattribute:: write_concern
       .. autoattribute:: uuid_subtype
+      .. automethod:: with_options
       .. automethod:: insert(doc_or_docs[, manipulate=True[, safe=None[, check_keys=True[, continue_on_error=False[, **kwargs]]]]])
       .. automethod:: save(to_save[, manipulate=True[, safe=None[, check_keys=True[, **kwargs]]]])
       .. automethod:: update(spec, document[, upsert=False[, manipulate=False[, safe=None[, multi=False[, check_keys=True[, **kwargs]]]]]])
@@ -35,8 +38,12 @@
       .. automethod:: initialize_unordered_bulk_op
       .. automethod:: initialize_ordered_bulk_op
       .. automethod:: drop
-      .. automethod:: find([spec=None[, fields=None[, skip=0[, limit=0[, timeout=True[, snapshot=False[, tailable=False[, sort=None[, max_scan=None[, as_class=None[, slave_okay=False[, await_data=False[, partial=False[, manipulate=True[, read_preference=ReadPreference.PRIMARY[, exhaust=False, [compile_re=True, [,**kwargs]]]]]]]]]]]]]]]]]])
+      .. automethod:: find([spec=None[, fields=None[, skip=0[, limit=0[, timeout=True[, snapshot=False[, tailable=False[, sort=None[, max_scan=None[, as_class=None[, slave_okay=False[, await_data=False[, partial=False[, manipulate=True[, read_preference=ReadPreference.PRIMARY[, tag_sets=[{}][, secondary_acceptable_latency_ms=None[, exhaust=False[, compile_re=True[, oplog_replay=False[, modifiers=None[, network_timeout=None[, filter=None[, projection=None[, no_cursor_timeout=None[, allow_partial_results=None[, cursor_type=None]]]]]]]]]]]]]]]]]]]]]]]]]]])
       .. automethod:: find_one([spec_or_id=None[, *args[, **kwargs]]])
+      .. automethod:: find_one_and_delete
+      .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, **kwargs)
+      .. automethod:: find_one_and_update(filter, update, projection=None, sort=None, return_document=ReturnDocument.BEFORE, **kwargs)
+      .. automethod:: bulk_write
       .. automethod:: parallel_scan
       .. automethod:: count
       .. automethod:: create_index
@@ -53,6 +60,13 @@
       .. automethod:: map_reduce
       .. automethod:: inline_map_reduce
       .. automethod:: find_and_modify
+      .. automethod:: insert_one
+      .. automethod:: insert_many
+      .. automethod:: replace_one
+      .. automethod:: update_one
+      .. automethod:: update_many
+      .. automethod:: delete_one
+      .. automethod:: delete_many
       .. autoattribute:: slave_okay
       .. autoattribute:: safe
       .. automethod:: get_lasterror_options

doc/api/pymongo/connection.rst

@@ -6,9 +6,7 @@
 
    .. autoclass:: pymongo.connection.Connection([host='localhost'[, port=27017[, max_pool_size=None[, network_timeout=None[, document_class=dict[, tz_aware=False[, **kwargs]]]]]]])
 
-      .. automethod:: disconnect
       .. automethod:: close
-      .. automethod:: alive
 
       .. describe:: c[db_name] || c.db_name
 
@@ -29,25 +27,31 @@
       .. autoattribute:: max_message_size
       .. autoattribute:: min_wire_version
       .. autoattribute:: max_wire_version
+      .. autoattribute:: codec_options
       .. autoattribute:: read_preference
       .. autoattribute:: tag_sets
       .. autoattribute:: secondary_acceptable_latency_ms
       .. autoattribute:: write_concern
-      .. autoattribute:: slave_okay
-      .. autoattribute:: safe
       .. autoattribute:: is_locked
       .. automethod:: database_names
       .. automethod:: drop_database
       .. automethod:: copy_database(from_name, to_name[, from_host=None[, username=None[, password=None]]])
       .. automethod:: get_default_database
+      .. automethod:: get_database
       .. automethod:: server_info
       .. automethod:: start_request
+      .. automethod:: in_request
       .. automethod:: end_request
       .. automethod:: close_cursor
       .. automethod:: kill_cursors
       .. automethod:: set_cursor_manager
       .. automethod:: fsync
       .. automethod:: unlock
+      .. automethod:: disconnect
+      .. automethod:: alive
+      .. autoattribute:: uuid_subtype
+      .. autoattribute:: slave_okay
+      .. autoattribute:: safe
       .. automethod:: get_lasterror_options
       .. automethod:: set_lasterror_options
       .. automethod:: unset_lasterror_options

doc/api/pymongo/cursor.rst

@@ -4,7 +4,14 @@
 .. automodule:: pymongo.cursor
    :synopsis: Tools for iterating over MongoDB query results
 
-   .. autoclass:: pymongo.cursor.Cursor(collection, spec=None, fields=None, skip=0, limit=0, timeout=True, snapshot=False, tailable=False, sort=None, max_scan=None, as_class=None, slave_okay=False, await_data=False, partial=False, manipulate=True, read_preference=ReadPreference.PRIMARY, tag_sets=[{}], secondary_acceptable_latency_ms=None, exhaust=False, network_timeout=None)
+   .. autoclass:: pymongo.cursor.CursorType
+
+      .. autoattribute:: NON_TAILABLE
+      .. autoattribute:: TAILABLE
+      .. autoattribute:: TAILABLE_AWAIT
+      .. autoattribute:: EXHAUST
+
+   .. autoclass:: pymongo.cursor.Cursor(collection, spec=None, fields=None, skip=0, limit=0, timeout=True, snapshot=False, tailable=False, sort=None, max_scan=None, as_class=None, slave_okay=False, await_data=False, partial=False, manipulate=True, read_preference=ReadPreference.PRIMARY, tag_sets=[{}], secondary_acceptable_latency_ms=None, exhaust=False, compile_re=True, oplog_replay=False, modifiers=None, network_timeout=None, filter=None, projection=None, no_cursor_timeout=None, allow_partial_results=None, cursor_type=None)
       :members:
 
       .. describe:: c[index]

doc/api/pymongo/database.rst

@@ -23,6 +23,7 @@
          .. note::  Use dictionary style access if `collection_name` is an
             attribute of the :class:`Database` class eg: db[`collection_name`].
 
+      .. autoattribute:: codec_options
       .. autoattribute:: read_preference
       .. autoattribute:: tag_sets
       .. autoattribute:: secondary_acceptable_latency_ms

doc/api/pymongo/index.rst

@@ -13,7 +13,10 @@
 
       Alias for :class:`pymongo.mongo_replica_set_client.MongoReplicaSetClient`.
 
-   .. autoclass:: pymongo.read_preferences.ReadPreference
+   .. data:: ReadPreference
+
+      Alias for :class:`pymongo.read_preferences.ReadPreference`
+
    .. autofunction:: has_c
    .. data:: MIN_SUPPORTED_WIRE_VERSION
 
@@ -39,8 +42,12 @@ Sub-modules:
    message
    mongo_client
    mongo_replica_set_client
+   operations
    pool
+   read_preferences
+   results
    replica_set_connection
    son_manipulator
    cursor_manager
    uri_parser
+   write_concern

doc/api/pymongo/mongo_client.rst

@@ -6,9 +6,7 @@
 
    .. autoclass:: pymongo.mongo_client.MongoClient([host='localhost'[, port=27017[, max_pool_size=100[, document_class=dict[, tz_aware=False[, **kwargs]]]]]])
 
-      .. automethod:: disconnect
       .. automethod:: close
-      .. automethod:: alive
 
       .. describe:: c[db_name] || c.db_name
 
@@ -16,6 +14,7 @@
 
          Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used.
 
+      .. autoattribute:: address
       .. autoattribute:: host
       .. autoattribute:: port
       .. autoattribute:: is_primary
@@ -23,27 +22,39 @@
       .. autoattribute:: max_pool_size
       .. autoattribute:: nodes
       .. autoattribute:: auto_start_request
+      .. autoattribute:: use_greenlets
       .. autoattribute:: document_class
       .. autoattribute:: tz_aware
       .. autoattribute:: max_bson_size
       .. autoattribute:: max_message_size
       .. autoattribute:: min_wire_version
       .. autoattribute:: max_wire_version
+      .. autoattribute:: codec_options
       .. autoattribute:: read_preference
       .. autoattribute:: tag_sets
       .. autoattribute:: secondary_acceptable_latency_ms
+      .. autoattribute:: local_threshold_ms
       .. autoattribute:: write_concern
-      .. autoattribute:: uuid_subtype
       .. autoattribute:: is_locked
       .. automethod:: database_names
       .. automethod:: drop_database
       .. automethod:: copy_database(from_name, to_name[, from_host=None[, username=None[, password=None]]])
       .. automethod:: get_default_database
+      .. automethod:: get_database
       .. automethod:: server_info
       .. automethod:: start_request
+      .. automethod:: in_request
       .. automethod:: end_request
       .. automethod:: close_cursor
       .. automethod:: kill_cursors
       .. automethod:: set_cursor_manager
       .. automethod:: fsync
       .. automethod:: unlock
+      .. automethod:: disconnect
+      .. automethod:: alive
+      .. autoattribute:: uuid_subtype
+      .. autoattribute:: slave_okay
+      .. autoattribute:: safe
+      .. automethod:: get_lasterror_options
+      .. automethod:: set_lasterror_options
+      .. automethod:: unset_lasterror_options

doc/api/pymongo/mongo_replica_set_client.rst

@@ -8,7 +8,6 @@
 
       .. automethod:: disconnect
       .. automethod:: close
-      .. automethod:: alive
 
       .. describe:: c[db_name] || c.db_name
 
@@ -18,6 +17,7 @@
 
       .. autoattribute:: seeds
       .. autoattribute:: hosts
+      .. autoattribute:: address
       .. autoattribute:: primary
       .. autoattribute:: secondaries
       .. autoattribute:: arbiters
@@ -30,13 +30,26 @@
       .. autoattribute:: min_wire_version
       .. autoattribute:: max_wire_version
       .. autoattribute:: auto_start_request
+      .. autoattribute:: use_greenlets
+      .. autoattribute:: codec_options
       .. autoattribute:: read_preference
       .. autoattribute:: tag_sets
       .. autoattribute:: secondary_acceptable_latency_ms
+      .. autoattribute:: local_threshold_ms
       .. autoattribute:: write_concern
-      .. autoattribute:: uuid_subtype
       .. automethod:: database_names
       .. automethod:: drop_database
       .. automethod:: copy_database(from_name, to_name[, from_host=None[, username=None[, password=None]]])
       .. automethod:: get_default_database
+      .. automethod:: get_database
       .. automethod:: close_cursor
+      .. automethod:: start_request
+      .. automethod:: in_request
+      .. automethod:: end_request
+      .. automethod:: alive
+      .. autoattribute:: uuid_subtype
+      .. autoattribute:: slave_okay
+      .. autoattribute:: safe
+      .. automethod:: get_lasterror_options
+      .. automethod:: set_lasterror_options
+      .. automethod:: unset_lasterror_options

doc/api/pymongo/operations.rst

@@ -0,0 +1,6 @@
+:mod:`operations` -- Operation class definitions
+================================================
+
+.. automodule:: pymongo.operations
+   :synopsis: Operation class definitions
+   :members:

doc/api/pymongo/read_preferences.rst

@@ -0,0 +1,19 @@
+:mod:`read_preferences` -- Utilities for choosing which member of a replica set to read from.
+=============================================================================================
+
+.. automodule:: pymongo.read_preferences
+   :synopsis: Utilities for choosing which member of a replica set to read from.
+
+   .. autoclass:: pymongo.read_preferences.Primary
+      :inherited-members:
+   .. autoclass:: pymongo.read_preferences.PrimaryPreferred
+      :inherited-members:
+   .. autoclass:: pymongo.read_preferences.Secondary
+      :inherited-members:
+   .. autoclass:: pymongo.read_preferences.SecondaryPreferred
+      :inherited-members:
+   .. autoclass:: pymongo.read_preferences.Nearest
+      :inherited-members:
+
+   .. autoclass:: ReadPreference
+

doc/api/pymongo/replica_set_connection.rst

@@ -8,7 +8,6 @@
 
       .. automethod:: disconnect
       .. automethod:: close
-      .. automethod:: alive
 
       .. describe:: c[db_name] || c.db_name
 
@@ -30,16 +29,21 @@
       .. autoattribute:: min_wire_version
       .. autoattribute:: max_wire_version
       .. autoattribute:: auto_start_request
+      .. autoattribute:: codec_options
       .. autoattribute:: read_preference
       .. autoattribute:: tag_sets
       .. autoattribute:: secondary_acceptable_latency_ms
       .. autoattribute:: write_concern
-      .. autoattribute:: safe
       .. automethod:: database_names
       .. automethod:: drop_database
       .. automethod:: copy_database(from_name, to_name[, from_host=None[, username=None[, password=None]]])
       .. automethod:: get_default_database
+      .. automethod:: get_database
       .. automethod:: close_cursor
+      .. automethod:: alive
+      .. autoattribute:: uuid_subtype
+      .. autoattribute:: slave_okay
+      .. autoattribute:: safe
       .. automethod:: get_lasterror_options
       .. automethod:: set_lasterror_options
       .. automethod:: unset_lasterror_options

doc/api/pymongo/results.rst

@@ -0,0 +1,7 @@
+:mod:`results` -- Result class definitions
+==========================================
+
+.. automodule:: pymongo.results
+   :synopsis: Result class definitions
+   :members:
+   :inherited-members:

doc/api/pymongo/write_concern.rst

@@ -0,0 +1,6 @@
+:mod:`write_concern` -- Tools for specifying write concern
+==========================================================
+
+.. automodule:: pymongo.write_concern
+   :synopsis: Tools for specifying write concern.
+   :members:

doc/atlas.rst

@@ -0,0 +1,35 @@
+Using PyMongo 2.x with MongoDB Atlas
+====================================
+
+`Atlas <https://www.mongodb.com/cloud>`_ is MongoDB, Inc's hosted MongoDB as a
+service offering. The following steps are required to securely connect to Atlas
+with PyMongo 2.x.
+
+.. warning:: These directions **MUST** be followed carefully to ensure a secure
+  connection is used.
+
+First, install `certifi <https://pypi.python.org/pypi/certifi>`_::
+
+  $ python -m pip install certifi
+
+To connect to Atlas, pass the connection string provided by Atlas to
+:class:`~pymongo.mongo_client.MongoClient`. You **MUST** provide all of these
+options to make a secure connection::
+
+  >>> import certifi
+  >>> import ssl
+  >>> from pymongo import MongoClient
+  >>> client = MongoClient(<Atlas connection string>,
+  ...                      ssl_cert_reqs=ssl.CERT_REQUIRED,
+  ...                      ssl_ca_certs=certifi.where())
+
+Connections to Atlas using
+:class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` require the
+same options::
+
+  >>> import certifi
+  >>> import ssl
+  >>> from pymongo import MongoReplicaSetClient
+  >>> client = MongoReplicaSetClient(<Atlas connection string>,
+  ...                                ssl_cert_reqs=ssl.CERT_REQUIRED,
+  ...                                ssl_ca_certs=certifi.where())

doc/changelog.rst

@@ -1,6 +1,286 @@
 Changelog
 =========
 
+.. warning:: PyMongo 2.x is in maintenance mode. Support for new MongoDB
+  features ended with the release of MongoDB 3.0 and PyMongo 2.8. Users are
+  strongly encouraged to upgrade to PyMongo 3.x. See the
+  :doc:`/migrate-to-pymongo3` for details.
+
+Changes in Version 2.9.5
+------------------------
+
+Version 2.9.5 works around ssl module deprecations in Python 3.6, and expected
+future ssl module deprecations. It also fixes bugs found since the release of
+2.9.4.
+
+- Use ssl.SSLContext and ssl.PROTOCOL_TLS_CLIENT when available.
+- Fixed a C extensions build issue when the interpreter was built with -std=c99
+- Fixed various build issues with MinGW32.
+- Fixed a write concern bug in :meth:`~pymongo.database.Database.add_user` and
+  :meth:`~pymongo.database.Database.remove_user` when connected to MongoDB 3.2+
+- Fixed various test failures related to changes in gevent, MongoDB, and our CI
+  test environment.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.9.5 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.9.5 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/17605
+
+Changes in Version 2.9.4
+------------------------
+
+Version 2.9.4 fixes issues reported since the release of 2.9.3.
+
+- Fixed __repr__ for closed instances of :class:`~pymongo.mongo_client.MongoClient`.
+- Fixed :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` handling of
+  uuidRepresentation.
+- Fixed building and testing the documentation with python 3.x.
+- New documentation for :doc:`examples/tls` and :doc:`atlas`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.9.4 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.9.4 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/16885
+
+Changes in Version 2.9.3
+------------------------
+
+Version 2.9.3 fixes a few issues reported since the release of 2.9.2 including
+thread safety issues in :meth:`~pymongo.collection.Collection.ensure_index`,
+:meth:`~pymongo.collection.Collection.drop_index`, and
+:meth:`~pymongo.collection.Collection.drop_indexes`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.9.3 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.9.3 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/16539
+
+Changes in Version 2.9.2
+------------------------
+
+Version 2.9.2 restores Python 3.1 support, which was broken in PyMongo 2.8. It
+improves an error message when decoding BSON as well as fixes a couple other
+issues including :meth:`~pymongo.collection.Collection.aggregate` ignoring
+:attr:`~pymongo.collection.Collection.codec_options` and
+:meth:`~pymongo.database.Database.command` raising a superfluous
+`DeprecationWarning`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.9.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.9.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/16303
+
+Changes in Version 2.9.1
+------------------------
+
+Version 2.9.1 fixes two interrupt handling issues in the C extensions and
+adapts a test case for a behavior change in MongoDB 3.2.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.9.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.9.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/16208
+
+Changes in Version 2.9
+----------------------
+
+Version 2.9 provides an upgrade path to PyMongo 3.x. Most of the API changes
+from PyMongo 3.0 have been backported in a backward compatible way, allowing
+applications to be written against PyMongo >= 2.9, rather then PyMongo 2.x or
+PyMongo 3.x. See the :doc:`/migrate-to-pymongo3` for detailed examples.
+
+.. note:: There are a number of new deprecations in this release for features
+  that were removed in PyMongo 3.0.
+
+  :class:`~pymongo.mongo_client.MongoClient`:
+    - :attr:`~pymongo.mongo_client.MongoClient.host`
+    - :attr:`~pymongo.mongo_client.MongoClient.port`
+    - :attr:`~pymongo.mongo_client.MongoClient.use_greenlets`
+    - :attr:`~pymongo.mongo_client.MongoClient.document_class`
+    - :attr:`~pymongo.mongo_client.MongoClient.tz_aware`
+    - :attr:`~pymongo.mongo_client.MongoClient.secondary_acceptable_latency_ms`
+    - :attr:`~pymongo.mongo_client.MongoClient.tag_sets`
+    - :attr:`~pymongo.mongo_client.MongoClient.uuid_subtype`
+    - :meth:`~pymongo.mongo_client.MongoClient.disconnect`
+    - :meth:`~pymongo.mongo_client.MongoClient.alive`
+
+  :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`:
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.use_greenlets`
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.document_class`
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.tz_aware`
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.secondary_acceptable_latency_ms`
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.tag_sets`
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.uuid_subtype`
+    - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.alive`
+
+  :class:`~pymongo.database.Database`:
+    - :attr:`~pymongo.database.Database.secondary_acceptable_latency_ms`
+    - :attr:`~pymongo.database.Database.tag_sets`
+    - :attr:`~pymongo.database.Database.uuid_subtype`
+
+  :class:`~pymongo.collection.Collection`:
+    - :attr:`~pymongo.collection.Collection.secondary_acceptable_latency_ms`
+    - :attr:`~pymongo.collection.Collection.tag_sets`
+    - :attr:`~pymongo.collection.Collection.uuid_subtype`
+
+.. warning::
+  In previous versions of PyMongo, changing the value of
+  :attr:`~pymongo.mongo_client.MongoClient.document_class` changed
+  the behavior of all existing instances of
+  :class:`~pymongo.collection.Collection`::
+
+    >>> coll = client.test.test
+    >>> coll.find_one()
+    {u'_id': ObjectId('5579dc7cfba5220cc14d9a18')}
+    >>> from bson.son import SON
+    >>> client.document_class = SON
+    >>> coll.find_one()
+    SON([(u'_id', ObjectId('5579dc7cfba5220cc14d9a18'))])
+
+  The document_class setting is now configurable at the client,
+  database, collection, and per-operation level. This required breaking
+  the existing behavior. To change the document class per operation in a
+  forward compatible way use
+  :meth:`~pymongo.collection.Collection.with_options`::
+
+    >>> coll.find_one()
+    {u'_id': ObjectId('5579dc7cfba5220cc14d9a18')}
+    >>> from bson.codec_options import CodecOptions
+    >>> coll.with_options(CodecOptions(SON)).find_one()
+    SON([(u'_id', ObjectId('5579dc7cfba5220cc14d9a18'))])
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.9 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.9 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14795
+
+
+Changes in Version 2.8.1
+------------------------
+
+Version 2.8.1 fixes a number of issues reported since the release of PyMongo
+2.8. It is a recommended upgrade for all users of PyMongo 2.x.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.8.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.8.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/15324
+
+Changes in Version 2.8
+----------------------
+
+Version 2.8 is a major release that provides full support for MongoDB 3.0 and
+fixes a number of bugs.
+
+Special thanks to Don Mitchell, Ximing, Can Zhang, Sergey Azovskov, and Heewa
+Barfchin for their contributions to this release.
+
+Highlights include:
+
+- Support for the SCRAM-SHA-1 authentication mechanism (new in MongoDB 3.0).
+- JSON decoder support for the new $numberLong and $undefined types.
+- JSON decoder support for the $date type as an ISO-8601 string.
+- Support passing an index name to :meth:`~pymongo.cursor.Cursor.hint`.
+- The :meth:`~pymongo.cursor.Cursor.count` method will use a hint if one
+  has been provided through :meth:`~pymongo.cursor.Cursor.hint`.
+- A new socketKeepAlive option for the connection pool.
+- New generator based BSON decode functions, :func:`~bson.decode_iter`
+  and :func:`~bson.decode_file_iter`.
+- Internal changes to support alternative storage engines like wiredtiger.
+
+.. note:: There are a number of deprecations in this release for features that
+  will be removed in PyMongo 3.0. These include:
+
+  :class:`~pymongo.mongo_client.MongoClient`:
+    - :attr:`~pymongo.mongo_client.MongoClient.auto_start_request`
+    - :meth:`~pymongo.mongo_client.MongoClient.start_request`
+    - :meth:`~pymongo.mongo_client.MongoClient.in_request`
+    - :meth:`~pymongo.mongo_client.MongoClient.end_request`
+    - :meth:`~pymongo.mongo_client.MongoClient.copy_database`
+
+  :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`:
+    - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.auto_start_request`
+    - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.start_request`
+    - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.in_request`
+    - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.end_request`
+    - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.copy_database`
+
+  :class:`~pymongo.database.Database`:
+    - :meth:`~pymongo.database.Database.error`
+    - :meth:`~pymongo.database.Database.last_status`
+    - :meth:`~pymongo.database.Database.previous_error`
+    - :meth:`~pymongo.database.Database.reset_error_history`
+
+  :class:`~pymongo.master_slave_connection.MasterSlaveConnection`
+
+  The JSON format for :class:`~bson.timestamp.Timestamp` has changed from
+  '{"t": <int>, "i": <int>}' to '{"$timestamp": {"t": <int>, "i": <int>}}'.
+  This new format will be decoded to an instance of
+  :class:`~bson.timestamp.Timestamp`. The old format will continue to be
+  decoded to a python dict as before. Encoding to the old format is no
+  longer supported as it was never correct and loses type information.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.8 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.8 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14223
+
+Changes in Version 2.7.2
+------------------------
+
+Version 2.7.2 includes fixes for upsert reporting in the bulk API for MongoDB
+versions previous to 2.6, a regression in how son manipulators are applied in
+:meth:`~pymongo.collection.Collection.insert`, a few obscure connection pool
+semaphore leaks, and a few other minor issues. See the list of issues resolved
+for full details.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.7.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.7.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14005
+
+Changes in Version 2.7.1
+------------------------
+
+Version 2.7.1 fixes a number of issues reported since the release of 2.7,
+most importantly a fix for creating indexes and manipulating users through
+mongos versions older than 2.4.0.
+
+Issues Resolved
+...............
+
+See the `PyMongo 2.7.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 2.7.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/13823
+
 Changes in Version 2.7
 ----------------------
 

doc/compatibility-policy.rst

@@ -0,0 +1,61 @@
+Compatibility Policy
+====================
+
+Semantic Versioning
+-------------------
+
+PyMongo's version numbers follow `semantic versioning`_: each version number
+is structured "major.minor.patch". Patch releases fix bugs, minor releases
+add features (and may fix bugs), and major releases include API changes that
+break backwards compatibility (and may add features and fix bugs).
+
+Deprecation
+-----------
+
+Before we remove a feature in a major release, PyMongo's maintainers make an
+effort to release at least one minor version that *deprecates* it. We add
+"**DEPRECATED**" to the feature's documentation, and update the code to raise a
+`DeprecationWarning`_. You can ensure your code is future-proof by running
+your code with the latest PyMongo release and looking for DeprecationWarnings.
+
+Starting with Python 2.7, the interpreter silences DeprecationWarnings by
+default. For example, the following code uses the deprecated ``slave_okay``
+option but does not raise any warning:
+
+.. code-block:: python
+
+    # "slave_okay.py"
+    from pymongo import MongoClient
+
+    client = MongoClient(slave_okay=True)
+
+To print deprecation warnings to stderr, run python with "-Wd"::
+
+  $ python -Wd slave_okay.py
+  slave_okay.py:4: DeprecationWarning: slave_okay is deprecated. Please use read_preference instead.
+    client = MongoClient(slave_okay=True)
+
+You can turn warnings into exceptions with "python -We"::
+
+  $ python -We slave_okay.py
+  Traceback (most recent call last):
+    File "slave_okay.py", line 4, in <module>
+      client = MongoClient(slave_okay=True)
+    File "/Users/emptysquare/.virtualenvs/official/mongo-python-driver/pymongo/mongo_client.py", line 373, in __init__
+      stacklevel=2)
+  DeprecationWarning: slave_okay is deprecated. Please use read_preference instead.
+
+If your own code's test suite passes with "python -We" then it uses no
+deprecated PyMongo features.
+
+.. seealso:: The Python documentation on `the warnings module`_,
+   and `the -W command line option`_.
+
+.. _semantic versioning: http://semver.org/
+
+.. _DeprecationWarning:
+  https://docs.python.org/2/library/exceptions.html#exceptions.DeprecationWarning
+
+.. _the warnings module: https://docs.python.org/2/library/warnings.html
+
+.. _the -W command line option: https://docs.python.org/2/using/cmdline.html#cmdoption-W

doc/conf.py

@@ -4,8 +4,17 @@
 #
 # This file is execfile()d with the current directory set to its containing dir.
 
-import sys, os
-sys.path[0:0] = [os.path.abspath('..')]
+import os
+import sys
+
+_path = os.path.abspath('..')
+sys.path[0:0] = [_path]
+if sys.version_info[0] >= 3:
+    import glob
+    ver = '.'.join(map(str, sys.version_info[:2]))
+    _path = glob.glob(
+        os.path.join(os.path.abspath('..'), 'build', 'lib*' + ver))[0]
+    sys.path[0:0] = [_path]
 
 import pymongo
 
@@ -27,7 +36,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'PyMongo'
-copyright = u'2008 - 2014, MongoDB, Inc.'
+copyright = u'2008 - 2015, MongoDB, Inc.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -65,9 +74,9 @@ pygments_style = 'sphinx'
 # -- Options for extensions ----------------------------------------------------
 autoclass_content = 'init'
 
-doctest_path = os.path.abspath('..')
+doctest_path = [_path]
 
-doctest_test_doctest_blocks = False
+doctest_test_doctest_blocks = ''
 
 doctest_global_setup = """
 from pymongo.mongo_client import MongoClient

doc/contributors.rst

@@ -69,3 +69,10 @@ The following is a list of people who have contributed to
 - Yuchen Ying (yegle)
 - Kyle Erf (3rf)
 - Luke Lovett (lovett89)
+- Jaroslav Semančík (girogiro)
+- Don Mitchell (dmitchell)
+- Ximing (armnotstrong)
+- Can Zhang (cannium)
+- Sergey Azovskov (last-g)
+- Heewa Barfchin (heewa)
+- Len Buckens (buckensl)

doc/examples/aggregation.rst

@@ -51,15 +51,20 @@ eg "$sort":
     PyMongo version **>= 2.3**.
 
 .. doctest::
+  :options: +NORMALIZE_WHITESPACE
 
   >>> from bson.son import SON
-  >>> db.things.aggregate([
+  >>> import pprint
+  >>> pprint.pprint(db.things.aggregate([
   ...         {"$unwind": "$tags"},
   ...         {"$group": {"_id": "$tags", "count": {"$sum": 1}}},
   ...         {"$sort": SON([("count", -1), ("_id", -1)])}
-  ...     ])
+  ...     ]))
   ...
-  {u'ok': 1.0, u'result': [{u'count': 3, u'_id': u'cat'}, {u'count': 2, u'_id': u'dog'}, {u'count': 1, u'_id': u'mouse'}]}
+  {u'ok': 1.0,
+   u'result': [{u'_id': u'cat', u'count': 3},
+               {u'_id': u'dog', u'count': 2},
+               {u'_id': u'mouse', u'count': 1}]...}
 
 
 As well as simple aggregations the aggregation framework provides projection
@@ -74,7 +79,7 @@ Map/Reduce
 ----------
 
 Another option for aggregation is to use the map reduce framework.  Here we
-will define **map** and **reduce** functions to also count he number of
+will define **map** and **reduce** functions to also count the number of
 occurrences for each tag in the ``tags`` array, across the entire collection.
 
 Our **map** function just emits a single `(key, 1)` pair for each tag in
@@ -115,7 +120,7 @@ iterate over the result collection:
 
   >>> result = db.things.map_reduce(mapper, reducer, "myresults")
   >>> for doc in result.find():
-  ...   print doc
+  ...   pprint.pprint(doc)
   ...
   {u'_id': u'cat', u'value': 3.0}
   {u'_id': u'dog', u'value': 2.0}
@@ -132,8 +137,12 @@ response to the map/reduce command, rather than just the result collection:
 
 .. doctest::
 
-  >>> db.things.map_reduce(mapper, reducer, "myresults", full_response=True)
-  {u'counts': {u'input': 4, u'reduce': 2, u'emit': 6, u'output': 3}, u'timeMillis': ..., u'ok': ..., u'result': u'...'}
+  >>> pprint.pprint(
+  ...     db.things.map_reduce(mapper, reducer, "myresults", full_response=True))
+  {u'counts': {u'emit': 6, u'input': 4, u'output': 3, u'reduce': 2},
+   u'ok': ...,
+   u'result': u'...',
+   u'timeMillis': ...}
 
 All of the optional map/reduce parameters are also supported, simply pass them
 as keyword arguments. In this example we use the `query` parameter to limit the
@@ -141,9 +150,10 @@ documents that will be mapped over:
 
 .. doctest::
 
-  >>> result = db.things.map_reduce(mapper, reducer, "myresults", query={"x": {"$lt": 2}})
-  >>> for doc in result.find():
-  ...   print doc
+  >>> results = db.things.map_reduce(
+  ...     mapper, reducer, "myresults", query={"x": {"$lt": 2}})
+  >>> for doc in results.find():
+  ...   pprint.pprint(doc)
   ...
   {u'_id': u'cat', u'value': 1.0}
   {u'_id': u'dog', u'value': 1.0}
@@ -155,8 +165,16 @@ result collection:
 .. doctest::
 
   >>> from bson.son import SON
-  >>> db.things.map_reduce(mapper, reducer, out=SON([("replace", "results"), ("db", "outdb")]), full_response=True)
-  {u'counts': {u'input': 4, u'reduce': 2, u'emit': 6, u'output': 3}, u'timeMillis': ..., u'ok': ..., u'result': {u'db': ..., u'collection': ...}}
+  >>> pprint.pprint(
+  ...     db.things.map_reduce(
+  ...         mapper,
+  ...         reducer,
+  ...         out=SON([("replace", "results"), ("db", "outdb")]),
+  ...         full_response=True))
+  {u'counts': {u'emit': 6, u'input': 4, u'output': 3, u'reduce': 2},
+   u'ok': ...,
+   u'result': {u'collection': ..., u'db': ...},
+   u'timeMillis': ...}
 
 .. seealso:: The full list of options for MongoDB's `map reduce engine <http://www.mongodb.org/display/DOCS/MapReduce>`_
 
@@ -171,20 +189,20 @@ reduce function.
 .. note:: Doesn't work with sharded MongoDB configurations, use aggregation or
           map/reduce instead of group().
 
-Here we are doing a simple group and count of the occurrences ``x`` values:
+Here we are doing a simple group and count of the occurrences of ``x`` values:
 
 .. doctest::
 
+  >>> from bson.code import Code
   >>> reducer = Code("""
   ...                function(obj, prev){
   ...                  prev.count++;
   ...                }
   ...                """)
   ...
-  >>> from bson.son import SON
   >>> results = db.things.group(key={"x":1}, condition={}, initial={"count": 0}, reduce=reducer)
   >>> for doc in results:
-  ...   print doc
+  ...   pprint.pprint(doc)
   {u'count': 1.0, u'x': 1.0}
   {u'count': 2.0, u'x': 2.0}
   {u'count': 1.0, u'x': 3.0}

doc/examples/authentication.rst

@@ -5,24 +5,69 @@ MongoDB supports several different authentication mechanisms. These examples
 cover all authentication methods currently supported by PyMongo, documenting
 Python module and MongoDB version dependencies.
 
-MONGODB-CR
-----------
-MONGODB-CR is the default authentication mechanism supported by a MongoDB
-cluster configured for authentication. Authentication is per-database and
-credentials can be specified through the MongoDB URI or passed to the
-:meth:`~pymongo.database.Database.authenticate` method::
+Support For Special Characters In Usernames And Passwords
+---------------------------------------------------------
+
+If your username or password contains special characters (e.g. '/', ' ',
+or '@') you must ``%xx`` escape them for use in the MongoDB URI. PyMongo
+uses :meth:`~urllib.unquote_plus` to decode them. For example::
+
+  >>> import urllib
+  >>> password = urllib.quote_plus('pass/word')
+  >>> password
+  'pass%2Fword'
+  >>> MongoClient('mongodb://user:' + password + '@127.0.0.1')
+  MongoClient('127.0.0.1', 27017)
+
+SCRAM-SHA-1 (RFC 5802)
+----------------------
+.. versionadded:: 2.8
+
+SCRAM-SHA-1 is the default authentication mechanism supported by a cluster
+configured for authentication with MongoDB 3.0 or later. Authentication is
+per-database and credentials can be specified through the MongoDB URI or
+passed to the :meth:`~pymongo.database.Database.authenticate` method::
 
   >>> from pymongo import MongoClient
   >>> client = MongoClient('example.com')
-  >>> client.the_database.authenticate('user', 'password')
+  >>> client.the_database.authenticate('user', 'password', mechanism='SCRAM-SHA-1')
   True
   >>>
-  >>> uri = "mongodb://user:password@example.com/the_database"
+  >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=SCRAM-SHA-1"
   >>> client = MongoClient(uri)
-  >>>
 
-When using MongoDB's delegated authentication features, a separate
-authentication source can be specified (using PyMongo 2.5 or newer)::
+For best performance install `backports.pbkdf2`_, especially on Python older
+than 2.7.8, or on Python 3 before Python 3.4.
+
+.. _backports.pbkdf2: https://pypi.python.org/pypi/backports.pbkdf2/
+
+MONGODB-CR
+----------
+
+Before MongoDB 3.0 the default authentication mechanism was MONGODB-CR,
+the "MongoDB Challenge-Response" protocol::
+
+  >>> from pymongo import MongoClient
+  >>> client = MongoClient('example.com')
+  >>> client.the_database.authenticate('user', 'password', mechanism='MONGODB-CR')
+  True
+  >>>
+  >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=MONGODB-CR"
+  >>> client = MongoClient(uri)
+
+Default Authentication Mechanism
+--------------------------------
+
+If no mechanism is specified, PyMongo automatically uses MONGODB-CR when
+connected to a pre-3.0 version of MongoDB, and SCRAM-SHA-1 when connected to
+a recent version.
+
+Delegated Authentication
+------------------------
+.. versionadded: 2.5
+
+In MongoDB 2.4.x a separate authentication source can be specified.
+This feature was introduced in MongoDB 2.4 and removed in 2.6::
 
   >>> from pymongo import MongoClient
   >>> client = MongoClient('example.com')
@@ -113,15 +158,15 @@ or using :meth:`~pymongo.database.Database.authenticate`::
   True
 
 The default service name used by MongoDB and PyMongo is `mongodb`. You can
-specify a custom service name with the ``gssapiServiceName`` option::
+specify a custom service name with the ``authMechanismProperties`` option::
 
   >>> from pymongo import MongoClient
-  >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&gssapiServiceName=myservicename"
+  >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&authMechanismProperties=SERVICE_NAME:myservicename"
   >>> client = MongoClient(uri)
   >>>
   >>> client = MongoClient('example.com')
   >>> db = client.test
-  >>> db.authenticate('mongodbuser@EXAMPLE.COM', mechanism='GSSAPI', gssapiServiceName='myservicename')
+  >>> db.authenticate('mongodbuser@EXAMPLE.COM', mechanism='GSSAPI', authMechanismProperties='SERVICE_NAME:myservicename')
   True
 
 .. note::
@@ -177,4 +222,3 @@ the SASL PLAIN mechanism::
   ...                      ssl_cert_reqs=ssl.CERT_REQUIRED,
   ...                      ssl_ca_certs='/path/to/ca.pem')
   >>>
-

doc/examples/bulk.rst

@@ -27,7 +27,7 @@ bulk insert operations.
 
   >>> import pymongo
   >>> db = pymongo.MongoClient().bulk_example
-  >>> db.test.insert(({'i': i} for i in xrange(10000)))
+  >>> db.test.insert(({'i': i} for i in range(10000)))
   [...]
   >>> db.test.count()
   10000
@@ -58,6 +58,7 @@ order provided for serial execution. The return value is a document
 describing the type and count of operations performed.
 
 .. doctest::
+  :options: +NORMALIZE_WHITESPACE
 
   >>> from pprint import pprint
   >>>
@@ -81,7 +82,6 @@ describing the type and count of operations performed.
    'upserted': [{u'_id': 4, u'index': 5}],
    'writeConcernErrors': [],
    'writeErrors': []}
-  >>>
 
 .. warning:: ``nModified`` is only reported by MongoDB 2.6 and later. When
   connected to an earlier server version, or in certain mixed version sharding
@@ -96,6 +96,7 @@ occurred and details about the failure - including the operation that caused
 the failure.
 
 .. doctest::
+  :options: +NORMALIZE_WHITESPACE
 
   >>> from pymongo.errors import BulkWriteError
   >>> bulk = db.test.initialize_ordered_bulk_op()
@@ -117,10 +118,9 @@ the failure.
    'upserted': [],
    'writeConcernErrors': [],
    'writeErrors': [{u'code': 11000,
-                    u'errmsg': u'insertDocument :: caused by :: 11000 E11000 duplicate key error index: bulk_example.test.$_id_  dup key: { : 4 }',
+                    u'errmsg': u'...E11000...duplicate key error...',
                     u'index': 1,
                     u'op': {'_id': 4}}]}
-  >>>
 
 .. _unordered_bulk:
 
@@ -136,6 +136,7 @@ constraint on _id. Since we are doing unordered execution the second
 and fourth operations succeed.
 
 .. doctest::
+  :options: +NORMALIZE_WHITESPACE
 
   >>> bulk = db.test.initialize_unordered_bulk_op()
   >>> bulk.insert({'_id': 1})
@@ -155,14 +156,13 @@ and fourth operations succeed.
    'upserted': [],
    'writeConcernErrors': [],
    'writeErrors': [{u'code': 11000,
-                    u'errmsg': u'insertDocument :: caused by :: 11000 E11000 duplicate key error index: bulk_example.test.$_id_  dup key: { : 1 }',
+                    u'errmsg': u'...E11000...duplicate key error...',
                     u'index': 0,
                     u'op': {'_id': 1}},
                    {u'code': 11000,
-                    u'errmsg': u'insertDocument :: caused by :: 11000 E11000 duplicate key error index: bulk_example.test.$_id_  dup key: { : 3 }',
+                    u'errmsg': u'...E11000...duplicate key error...',
                     u'index': 2,
                     u'op': {'_id': 3}}]}
-  >>>
 
 Write Concern
 .............
@@ -175,6 +175,7 @@ errors (e.g. wtimeout) will be reported after all operations are attempted,
 regardless of execution order.
 
 .. doctest::
+  :options: +NORMALIZE_WHITESPACE
 
   >>> bulk = db.test.initialize_ordered_bulk_op()
   >>> bulk.insert({'a': 0})
@@ -182,7 +183,7 @@ regardless of execution order.
   >>> bulk.insert({'a': 2})
   >>> bulk.insert({'a': 3})
   >>> try:
-  ...     bulk.execute({'w': 4, 'wtimeout': 1})
+  ...     bulk.execute({'w': 3, 'wtimeout': 1})
   ... except BulkWriteError as bwe:
   ...     pprint(bwe.details)
   ... 
@@ -192,9 +193,7 @@ regardless of execution order.
    'nRemoved': 0,
    'nUpserted': 0,
    'upserted': [],
-   'writeConcernErrors': [{u'code': 64,
+   'writeConcernErrors': [{u'code': 64...
                            u'errInfo': {u'wtimeout': True},
                            u'errmsg': u'waiting for replication timed out'}],
    'writeErrors': []}
-  >>>
-

doc/examples/copydb.rst

@@ -0,0 +1,71 @@
+Copying a Database
+==================
+
+Raw command
+-----------
+
+To copy a database within a single mongod process, or between mongod
+servers, simply connect to the target mongod and use the
+:meth:`~pymongo.database.Database.command` method::
+
+  >>> from pymongo import MongoClient
+  >>> client = MongoClient('target.example.com')
+  >>> client.admin.command('copydb',
+                           fromdb='source_db_name',
+                           todb='target_db_name')
+
+To copy from a different mongod server that is not password-protected::
+
+  >>> client.admin.command('copydb',
+                           fromdb='source_db_name',
+                           todb='target_db_name',
+                           fromhost='source.example.com')
+
+If the target server is password-protected, authenticate to the "admin"
+database first::
+
+  >>> client.admin.authenticate('administrator', 'pwd')
+  True
+  >>> client.admin.command('copydb',
+                           fromdb='source_db_name',
+                           todb='target_db_name',
+                           fromhost='source.example.com')
+
+See the :doc:`authentication examples </examples/authentication>`.
+
+``copy_database`` method
+------------------------
+
+The current version of PyMongo provides a helper method,
+:meth:`~pymongo.mongo_client.MongoClient.copy_database`, to copy a database
+from a password-protected mongod server to the target server.
+This method is deprecated and will be removed in PyMongo 3.0.
+Use the `copyDatabase function in the mongo shell`_ instead.
+
+Until the method is removed from PyMongo, you can copy a database from a
+password-protected server like so::
+
+  >>> client = MongoClient('target.example.com')
+  >>> client.copy_database(from_name='source_db_name',
+                           to_name='target_db_name',
+                           from_host='source.example.com',
+                           username='jesse',
+                           password='pwd',
+                           mechanism='SCRAM-SHA-1')
+
+Provide the username and password of a user who is authorized to read the
+source database on the source host. Again, if the target database is also
+password-protected, authenticate to the "admin" database first.
+
+The mechanism can be "MONGODB-CR" or "SCRAM-SHA-1". Use SCRAM-SHA-1 if the
+target and source hosts are both MongoDB 2.8 or later, otherwise use
+MONGODB-CR.
+
+If no mechanism is specified, PyMongo tries to use MONGODB-CR when
+connected to a pre-2.8 version of MongoDB, and SCRAM-SHA-1 when connected to
+a recent version. However, since PyMongo cannot determine the MongoDB
+version of the **source** host, it is better if you specify a mechanism
+yourself.
+
+.. _copyDatabase function in the mongo shell:
+   http://docs.mongodb.org/manual/reference/method/db.copyDatabase/

doc/examples/custom_type.rst

@@ -68,10 +68,12 @@ use them with PyMongo:
 
 .. doctest::
 
+  >>> import pprint
   >>> db.test.insert({"custom": encode_custom(Custom(5))})
   ObjectId('...')
-  >>> db.test.find_one()
-  {u'_id': ObjectId('...'), u'custom': {u'x': 5, u'_type': u'custom'}}
+  >>> pprint.pprint(db.test.find_one())
+  {u'_id': ObjectId('...'),
+   u'custom': {u'_type': u'custom', u'x': 5}}
   >>> decode_custom(db.test.find_one()["custom"])
   <Custom object at ...>
   >>> decode_custom(db.test.find_one()["custom"]).x()
@@ -122,8 +124,9 @@ After doing so we can save and restore :class:`Custom` instances seamlessly:
   {...}
   >>> db.test.insert({"custom": Custom(5)})
   ObjectId('...')
-  >>> db.test.find_one()
-  {u'_id': ObjectId('...'), u'custom': <Custom object at ...>}
+  >>> pprint.pprint(db.test.find_one())
+  {u'_id': ObjectId('...'),
+   u'custom': <Custom object at ...>}
   >>> db.test.find_one()["custom"].x()
   5
 
@@ -139,8 +142,9 @@ This allows us to see what was actually saved to the database:
 
 .. doctest::
 
-  >>> db.test.find_one()
-  {u'_id': ObjectId('...'), u'custom': {u'x': 5, u'_type': u'custom'}}
+  >>> pprint.pprint(db.test.find_one())
+  {u'_id': ObjectId('...'),
+   u'custom': {u'_type': u'custom', u'x': 5}}
 
 which is the same format that we encode to with our
 :meth:`encode_custom` method!
@@ -163,7 +167,7 @@ from :class:`~bson.binary.Binary` instances:
 
   >>> from bson.binary import Binary
   >>> def to_binary(custom):
-  ...   return Binary(str(custom.x()), 128)
+  ...   return Binary(str(custom.x()).encode(), 128)
   ...
   >>> def from_binary(binary):
   ...   return Custom(int(binary))
@@ -209,8 +213,9 @@ seamlessly:
 
   >>> db.test.insert({"custom": Custom(5)})
   ObjectId('...')
-  >>> db.test.find_one()
-  {u'_id': ObjectId('...'), u'custom': <Custom object at ...>}
+  >>> pprint.pprint(db.test.find_one())
+  {u'_id': ObjectId('...'),
+   u'custom': <Custom object at ...>}
   >>> db.test.find_one()["custom"].x()
   5
 
@@ -222,5 +227,5 @@ clearing out the manipulators and repeating our
 .. doctest::
 
   >>> db = client.custom_type_example
-  >>> db.test.find_one()
+  >>> pprint.pprint(db.test.find_one())
   {u'_id': ObjectId('...'), u'custom': Binary('5', 128)}

doc/examples/geo.rst

@@ -32,18 +32,15 @@ Inserting Places
 
 Locations in MongoDB are represented using either embedded documents
 or lists where the first two elements are coordinates. Here, we'll
-insert a couple of example locations:
+insert a few example locations:
 
 .. doctest::
 
-  >>> db.places.insert({"loc": [2, 5]})
-  ObjectId('...')
-  >>> db.places.insert({"loc": [30, 5]})
-  ObjectId('...')
-  >>> db.places.insert({"loc": [1, 2]})
-  ObjectId('...')
-  >>> db.places.insert({"loc": [4, 4]})
-  ObjectId('...')
+  >>> result = db.places.insert([
+  ...     {"loc": [2, 5]},
+  ...     {"loc": [30, 5]},
+  ...     {"loc": [1, 2]},
+  ...     {"loc": [4, 4]}])
 
 Querying
 --------
@@ -52,50 +49,53 @@ Using the geospatial index we can find documents near another point:
 
 .. doctest::
 
+  >>> import pprint
   >>> for doc in db.places.find({"loc": {"$near": [3, 6]}}).limit(3):
-  ...   repr(doc)
+  ...   pprint.pprint(doc)
   ...
-  "{u'loc': [2, 5], u'_id': ObjectId('...')}"
-  "{u'loc': [4, 4], u'_id': ObjectId('...')}"
-  "{u'loc': [1, 2], u'_id': ObjectId('...')}"
+  {u'_id': ObjectId('...'), u'loc': [2, 5]}
+  {u'_id': ObjectId('...'), u'loc': [4, 4]}
+  {u'_id': ObjectId('...'), u'loc': [1, 2]}
 
 The $maxDistance operator requires the use of :class:`~bson.son.SON`:
 
 .. doctest::
 
   >>> from bson.son import SON
-  >>> for doc in db.places.find({"loc": SON([("$near", [3, 6]), ("$maxDistance", 100)])}).limit(3):
-  ...   repr(doc)
+  >>> query = {"loc": SON([("$near", [3, 6]), ("$maxDistance", 100)])}
+  >>> for doc in db.places.find(query).limit(3):
+  ...   pprint.pprint(doc)
   ...
-  "{u'loc': [2, 5], u'_id': ObjectId('...')}"
-  "{u'loc': [4, 4], u'_id': ObjectId('...')}"
-  "{u'loc': [1, 2], u'_id': ObjectId('...')}"
+  {u'_id': ObjectId('...'), u'loc': [2, 5]}
+  {u'_id': ObjectId('...'), u'loc': [4, 4]}
+  {u'_id': ObjectId('...'), u'loc': [1, 2]}
 
 It's also possible to query for all items within a given rectangle
 (specified by lower-left and upper-right coordinates):
 
 .. doctest::
 
-  >>> for doc in db.places.find({"loc": {"$within": {"$box": [[2, 2], [5, 6]]}}}):
-  ...   repr(doc)
+  >>> query = {"loc": {"$within": {"$box": [[2, 2], [5, 6]]}}}
+  >>> for doc in db.places.find(query).sort('_id'):
+  ...   pprint.pprint(doc)
   ...
-  "{u'loc': [4, 4], u'_id': ObjectId('...')}"
-  "{u'loc': [2, 5], u'_id': ObjectId('...')}"
+  {u'_id': ObjectId('...'), u'loc': [2, 5]}
+  {u'_id': ObjectId('...'), u'loc': [4, 4]}
 
 Or circle (specified by center point and radius):
 
 .. doctest::
 
-  >>> for doc in db.places.find({"loc": {"$within": {"$center": [[0, 0], 6]}}}):
-  ...   repr(doc)
+  >>> query = {"loc": {"$within": {"$center": [[0, 0], 6]}}}
+  >>> for doc in db.places.find(query).sort('_id'):
+  ...   pprint.pprint(doc)
   ...
-  "{u'loc': [1, 2], u'_id': ObjectId('...')}"
-  "{u'loc': [4, 4], u'_id': ObjectId('...')}"
-  "{u'loc': [2, 5], u'_id': ObjectId('...')}"
+  {u'_id': ObjectId('...'), u'loc': [2, 5]}
+  {u'_id': ObjectId('...'), u'loc': [1, 2]}
+  {u'_id': ObjectId('...'), u'loc': [4, 4]}
 
 geoNear queries are also supported using :class:`~bson.son.SON`::
 
   >>> from bson.son import SON
   >>> db.command(SON([('geoNear', 'places'), ('near', [1, 2])]))
   {u'ok': 1.0, u'stats': ...}
-

doc/examples/gridfs.rst

@@ -42,7 +42,7 @@ interface (the :meth:`~gridfs.GridFS.put` and
 
 .. doctest::
 
-  >>> a = fs.put("hello world")
+  >>> a = fs.put(b"hello world")
 
 :meth:`~gridfs.GridFS.put` creates a new file in GridFS, and returns
 the value of the file document's ``"_id"`` key. Given that ``"_id"``

doc/examples/high_availability.rst

@@ -14,10 +14,6 @@ PyMongo makes working with `replica sets
 replica set and show how to handle both initialization and normal
 connections with PyMongo.
 
-.. note:: Replica sets require server version **>= 1.6.0**. Support
-   for connecting to replica sets also requires PyMongo version **>=
-   1.8.0**.
-
 .. mongodoc:: rs
 
 Starting a Replica Set
@@ -65,7 +61,7 @@ tell PyMongo that it's okay to connect to a slave/secondary::
 
   >>> from pymongo import MongoClient, ReadPreference
   >>> c = MongoClient("morton.local:27017",
-                      read_preference=ReadPreference.SECONDARY)
+                      readPreference="secondary")
 
 .. note:: We could have connected to any of the other nodes instead,
    but only the node we initiate from is allowed to contain any
@@ -128,10 +124,8 @@ connect to the replica set and perform a couple of basic operations::
 By checking the host and port, we can see that we're connected to
 *morton.local:27017*, which is the current primary::
 
-  >>> db.connection.host
-  'morton.local'
-  >>> db.connection.port
-  27017
+  >>> db.client.address
+  ('morton.local', 27017)
 
 Now let's bring down that node and see what happens when we run our
 query again::
@@ -156,10 +150,8 @@ the operation will succeed::
 
   >>> db.test.find_one()
   {u'x': 1, u'_id': ObjectId('...')}
-  >>> db.connection.host
-  'morton.local'
-  >>> db.connection.port
-  27018
+  >>> db.client.address
+  ('morton.local', 27018)
 
 MongoReplicaSetClient
 ~~~~~~~~~~~~~~~~~~~~~
@@ -180,90 +172,95 @@ Secondary Reads
 '''''''''''''''
 
 By default an instance of MongoReplicaSetClient will only send queries to
-the primary member of the replica set. To use secondaries for queries
-we have to change the :class:`~pymongo.read_preferences.ReadPreference`::
+the primary member of the replica set. To prefer secondaries for queries
+we have to change the read preference::
 
-  >>> db = MongoReplicaSetClient("morton.local:27017", replicaSet='foo').test
-  >>> from pymongo.read_preferences import ReadPreference
-  >>> db.read_preference = ReadPreference.SECONDARY_PREFERRED
+  >>> client = MongoReplicaSetClient(
+  ...     "morton.local:27017",
+  ...     replicaSet='foo',
+  ...     readPreference='secondaryPreferred')
 
 Now all queries will be sent to the secondary members of the set. If there are
 no secondary members the primary will be used as a fallback. If you have
 queries you would prefer to never send to the primary you can specify that
-using the ``SECONDARY`` read preference::
+using the ``secondary`` read preference::
 
-  >>> db.read_preference = ReadPreference.SECONDARY
+  >>> client = MongoReplicaSetClient(
+  ...     "morton.local:27017",
+  ...     replicaSet='foo',
+  ...     readPreference='secondary')
 
-Read preference can be set on a client, database, collection, or on a
-per-query basis, e.g.::
+Read preference can also be set on a database or collection, e.g.::
 
-  >>> db.collection.find_one(read_preference=ReadPreference.PRIMARY)
+  >>> from pymongo import ReadPreference
+  >>> db = client.get_database('test', read_preference=ReadPreference.PRIMARY)
+  >>> coll = db.get_collection(
+  ...     'testcoll', read_preference=ReadPreference.NEAREST)
 
-Reads are configured using three options: **read_preference**, **tag_sets**,
-and **secondary_acceptable_latency_ms**.
+Reads are configured using three options: **read preference**, **tag sets**,
+and **local threshold**.
 
-**read_preference**:
+**Read preference**:
 
 - ``PRIMARY``: Read from the primary. This is the default, and provides the
   strongest consistency. If no primary is available, raise
   :class:`~pymongo.errors.AutoReconnect`.
 
-- ``PRIMARY_PREFERRED``: Read from the primary if available, or if there is
-  none, read from a secondary matching your choice of ``tag_sets`` and
-  ``secondary_acceptable_latency_ms``.
+- ``PRIMARY_PREFERRED``: Read from the primary if available, otherwise read
+  from a secondary.
 
-- ``SECONDARY``: Read from a secondary matching your choice of ``tag_sets`` and
-  ``secondary_acceptable_latency_ms``. If no matching secondary is available,
+- ``SECONDARY``: Read from a secondary. If no matching secondary is available,
   raise :class:`~pymongo.errors.AutoReconnect`.
 
-- ``SECONDARY_PREFERRED``: Read from a secondary matching your choice of
-  ``tag_sets`` and ``secondary_acceptable_latency_ms`` if available, otherwise
-  from primary (regardless of the primary's tags and latency).
+- ``SECONDARY_PREFERRED``: Read from a secondary if available, otherwise from
+  the primary.
 
-- ``NEAREST``: Read from any member matching your choice of ``tag_sets`` and
-  ``secondary_acceptable_latency_ms``.
+- ``NEAREST``: Read from any available member.
 
-**tag_sets**:
+**Tag sets**:
 
 Replica-set members can be `tagged
 <http://www.mongodb.org/display/DOCS/Data+Center+Awareness>`_ according to any
 criteria you choose. By default, MongoReplicaSetClient ignores tags when
-choosing a member to read from, but it can be configured with the ``tag_sets``
-parameter. ``tag_sets`` must be a list of dictionaries, each dict providing tag
+choosing a member to read from, but it can be configured with the
+``readPreferenceTags`` option or the ``tag_sets`` read preference option.
+``tag_sets`` must be a list of dictionaries, each dict providing tag
 values that the replica set member must match. MongoReplicaSetClient tries each
 set of tags in turn until it finds a set of tags with at least one matching
 member. For example, to prefer reads from the New York data center, but fall
 back to the San Francisco data center, tag your replica set members according
-to their location and create a MongoReplicaSetClient like so:
+to their location and create a Database like so::
 
-  >>> rsc = MongoReplicaSetClient(
-  ...     "morton.local:27017",
-  ...     replicaSet='foo'
-  ...     read_preference=ReadPreference.SECONDARY,
-  ...     tag_sets=[{'dc': 'ny'}, {'dc': 'sf'}]
-  ... )
+  >>> from pymongo.read_preferences import Secondary
+  >>> db = client.get_database(
+  ...     'test',
+  ...     read_preference=Secondary(tag_sets=[{'dc': 'ny'}, {'dc': 'sf'}]))
+  >>>
 
 MongoReplicaSetClient tries to find secondaries in New York, then San Francisco,
 and raises :class:`~pymongo.errors.AutoReconnect` if none are available. As an
 additional fallback, specify a final, empty tag set, ``{}``, which means "read
 from any member that matches the mode, ignoring tags."
 
-**secondary_acceptable_latency_ms**:
+**Local threshold**:
 
-If multiple members match the mode and tag sets, MongoReplicaSetClient reads
-from among the nearest members, chosen according to ping time. By default,
-only members whose ping times are within 15 milliseconds of the nearest
-are used for queries. You can choose to distribute reads among members with
-higher latencies by setting ``secondary_acceptable_latency_ms`` to a larger
-number. In that case, MongoReplicaSetClient distributes reads among matching
-members within ``secondary_acceptable_latency_ms`` of the closest member's
-ping time.
+If multiple members match the read preference and tag sets, PyMongo reads from
+among the nearest members, chosen according to ping time. By default, only
+members whose ping times are within 15 milliseconds of the nearest are used for
+queries. You can choose to distribute reads among members with higher latencies
+by setting ``localThresholdMS`` to a larger number::
 
-.. note:: ``secondary_acceptable_latency_ms`` is ignored when talking to a
+  >>> client = pymongo.MongoReplicaSetClient(
+  ...     replicaSet='repl0',
+  ...     readPreference='secondaryPreferred',
+  ...     localThresholdMS=35)
+
+.. note:: ``localThresholdMS`` is ignored when talking to a
   replica set *through* a mongos. The equivalent is the localThreshold_ command
   line option.
 
-.. _localThreshold: http://docs.mongodb.org/manual/reference/mongos/#cmdoption-mongos--localThreshold
+.. _localThreshold:
+  http://docs.mongodb.org/manual/reference/mongos/#cmdoption-mongos--localThreshold
 
 Health Monitoring
 '''''''''''''''''
@@ -307,11 +304,9 @@ Each member of the seed list passed to MongoClient must be a mongos. By checking
 the host, port, and is_mongos attributes we can see that we're connected to
 *morton.local:30001*, a mongos::
 
-  >>> db.connection.host
-  'morton.local'
-  >>> db.connection.port
-  30001
-  >>> db.connection.is_mongos
+  >>> db.client.address
+  ('morton.local', 30001)
+  >>> db.client.is_mongos
   True
 
 Now let's shut down that mongos instance and see what happens when we run our
@@ -335,9 +330,7 @@ operation will succeed::
 
   >>> db.test.find_one()
   {u'x': 1, u'_id': ObjectId('...')}
-  >>> db.connection.host
-  'morton.local'
-  >>> db.connection.port
-  30002
-  >>> db.connection.is_mongos
+  >>> db.client.address
+  ('morton.local', 30002)
+  >>> db.client.is_mongos
   True

doc/examples/index.rst

@@ -18,6 +18,7 @@ MongoDB, you can start it like so:
 
    aggregation
    authentication
+   copydb
    bulk
    custom_type
    geo
@@ -26,3 +27,5 @@ MongoDB, you can start it like so:
    high_availability
    mod_wsgi
    requests
+   tailable
+   tls

doc/examples/requests.rst

@@ -1,11 +1,19 @@
 Requests
 ========
 
-PyMongo supports the idea of a *request*: a series of operations executed with
-a single socket, which are guaranteed to be processed on the server in the same
-order as they ran on the client.
+The ``start_request`` method of :class:`~pymongo.mongo_client.MongoClient`
+and :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` is now
+**deprecated** and will be removed in PyMongo 3.0.
 
-Requests are not usually necessary with PyMongo.
+PyMongo versions previous to 3.0 support the idea of a *request*: a series of
+operations executed with a single socket. This feature intended to make
+read-your-writes consistency more likely, even with unacknowledged writes.
+(That is, operations with write concern ``w=0``.)
+
+However, mongos 2.6 doesn't support socket pinning by default, and `mongos 3.0
+doesn't support it at all`_, so requests provide no benefit with sharding.
+
+In any case, requests are no longer necessary with PyMongo.
 By default, the methods :meth:`~pymongo.collection.Collection.insert`,
 :meth:`~pymongo.collection.Collection.update`,
 :meth:`~pymongo.collection.Collection.save`, and
@@ -14,97 +22,7 @@ acknowledgment from the server, so ordered execution is already guaranteed. You
 can be certain the next :meth:`~pymongo.collection.Collection.find` or
 :meth:`~pymongo.collection.Collection.count`, for example, is executed on the
 server after the writes complete. This is called "read-your-writes
-consistency."
+consistency." If your application requires this consistency, do not override
+the default write concern with ``w=0``.
 
-An example of when a request is necessary is if a series of documents are
-inserted with ``w=0`` for performance reasons, and you want to query those
-documents immediately afterward: With ``w=0`` the writes can queue up at the
-server and might not be immediately visible in query results. Wrapping the
-inserts and queries within
-:meth:`~pymongo.mongo_client.MongoClient.start_request` and
-:meth:`~pymongo.mongo_client.MongoClient.end_request` forces a query to be on
-the same socket as the inserts, so the query won't execute until the inserts
-are complete on the server side.
-
-Example
--------
-
-Let's consider a collection of web-analytics counters. We want to count the
-number of page views our site has served for each combination of browser,
-region, and OS, and then show the user the number of page views from his or her
-region, *including* the user's own visit. We have three ways to do so reliably:
-
-1. Simply update the counters with an acknowledged write (the default), and
-then ``find`` all counters for the visitor's region. This will ensure that the
-``update`` completes before the ``find`` begins, but it comes with a performance
-penalty that may be unacceptable for analytics.
-
-2. Create the :class:`~pymongo.mongo_client.MongoClient` with ``w=0`` and
-``auto_start_request=True`` to do unacknowledged writes and ensure each thread
-gets its own socket.
-
-3. Explicitly call :meth:`~pymongo.mongo_client.MongoClient.start_request`,
-then do the unacknowledged updates and the queries within the request. This
-third method looks like:
-
-.. testsetup::
-
-  from pymongo import MongoClient
-  client = MongoClient()
-  counts = client.requests_example.counts
-  counts.drop()
-  region, browser, os = 'US', 'Firefox', 'Mac OS X'
-
-.. doctest::
-
-  >>> client = MongoClient()
-  >>> counts = client.requests_example.counts
-  >>> region, browser, os = 'US', 'Firefox', 'Mac OS X'
-  >>> request = client.start_request()
-  >>> try:
-  ...   counts.update(
-  ...     {'region': region, 'browser': browser, 'os': os},
-  ...     {'$inc': {'n': 1 }},
-  ...     upsert=True,
-  ...     w=0) # unacknowledged write
-  ...
-  ...   # This always runs after update has completed:
-  ...   count = sum([p['n'] for p in counts.find({'region': region})])
-  ... finally:
-  ...   request.end()
-  >>> print count
-  1
-
-Requests can also be used as context managers, with the `with statement
-<http://docs.python.org/reference/compound_stmts.html#index-15>`_, which makes
-the previous example more terse:
-
-.. doctest::
-
-  >>> client.in_request()
-  False
-  >>> with client.start_request():
-  ...   # MongoClient is now in request
-  ...   counts.update(
-  ...     {'region': region, 'browser': browser, 'os': os},
-  ...     {'$inc': {'n': 1 }},
-  ...     upsert=True,
-  ...     safe=False)
-  ...   print sum([p['n'] for p in counts.find({'region': region})])
-  2
-  >>> client.in_request() # request automatically ended
-  False
-
-Requests And ``max_pool_size``
-------------------------------
-
-A thread in a request retains exclusive access to a socket until its request
-ends or the thread dies; thus, applications in which more than 100 threads are
-in requests at once should disable the ``max_pool_size`` option::
-
-    client = MongoClient(host, port, max_pool_size=None)
-
-Failure to increase or disable ``max_pool_size`` in such an application can
-leave threads forever waiting for sockets.
-
-See :ref:`connection-pooling`
+.. _mongos 3.0 doesn't support it at all: https://jira.mongodb.org/browse/SERVER-12273

doc/examples/tailable.rst

@@ -0,0 +1,41 @@
+Tailable Cursors
+================
+
+By default, MongoDB will automatically close a cursor when the client has
+exhausted all results in the cursor. However, for `capped collections
+<https://docs.mongodb.org/manual/core/capped-collections/>`_ you may
+use a `tailable cursor
+<https://docs.mongodb.org/manual/reference/glossary/#term-tailable-cursor>`_
+that remains open after the client exhausts the results in the initial cursor.
+
+The following is a basic example of using a tailable cursor to tail the oplog
+of a replica set member::
+
+  import time
+
+  import pymongo
+
+  client = pymongo.MongoClient()
+  oplog = client.local.oplog.rs
+  first = oplog.find().sort('$natural', pymongo.ASCENDING).limit(-1).next()
+  print(first)
+  ts = first['ts']
+
+  while True:
+      # The tailable and await_data options make this a tailable cursor.
+      cursor = oplog.find({'ts': {'$gt': ts}},
+                          tailable=True,
+                          await_data=True)
+      # Enable the OplogReplay cursor option. This enables an optimization
+      # to quickly find the 'ts' value we're looking for. It can only be used
+      # when querying the oplog.
+      cursor.add_option(8)
+      while cursor.alive:
+          for doc in cursor:
+              ts = doc['ts']
+              print(doc)
+          # We end up here if the find() returned no documents or if the
+          # tailable cursor timed out (no new documents were added to the
+          # collection for more than 1 second).
+          time.sleep(1)
+

doc/examples/tls.rst

@@ -0,0 +1,60 @@
+TLS/SSL and PyMongo 2.x
+=======================
+
+PyMongo supports connecting to MongoDB over TLS/SSL. This guide covers the
+configuration options supported by PyMongo. See `the server documentation
+<http://docs.mongodb.org/manual/tutorial/configure-ssl/>`_ to configure
+MongoDB.
+
+To make a secure TLS connection create
+:class:`~pymongo.mongo_client.MongoClient`
+(or :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`)
+with the following options::
+
+  >>> import ssl
+  >>> client = pymongo.MongoClient('example.com',
+  ...                              ssl=True,
+  ...                              ssl_cert_reqs=ssl.CERT_REQUIRED,
+  ...                              ssl_ca_certs='/path/to/ca.pem')
+
+Or, in the URI::
+
+  >>> uri = 'mongodb://example.com/?ssl=true&ssl_cert_reqs=CERT_REQUIRED&ssl_ca_certs=/path/to/ca.pem'
+  >>> client = pymongo.MongoClient(uri)
+
+To verify server certificates signed by a well known certificate authority, use
+`certifi <https://pypi.python.org/pypi/certifi>`_::
+
+  >>> import certifi
+  >>> import ssl
+  >>> client = pymongo.MongoClient('example.com',
+  ...                              ssl=True,
+  ...                              ssl_cert_reqs=ssl.CERT_REQUIRED,
+  ...                              ssl_ca_certs=certifi.where())
+  >>>
+  >>> uri = 'mongodb://example.com/?ssl=true&ssl_cert_reqs=CERT_REQUIRED&ssl_ca_certs=%s' % (certifi.where(),)
+  >>> client = pymongo.MongoClient(uri)
+
+Client certificates
+...................
+
+PyMongo can be configured to present a client certificate using the
+`ssl_certfile` option::
+
+  >>> client = pymongo.MongoClient('example.com',
+  ...                              ssl=True,
+  ...                              ssl_cert_reqs=ssl.CERT_REQUIRED,
+  ...                              ssl_ca_certs='/path/to/ca.pem',
+  ...                              ssl_certfile='/path/to/client.pem')
+
+If the private key for the client certificate is stored in a separate file use
+the `ssl_keyfile` option::
+
+  >>> client = pymongo.MongoClient('example.com',
+  ...                              ssl=True,
+  ...                              ssl_cert_reqs=ssl.CERT_REQUIRED,
+  ...                              ssl_ca_certs='/path/to/ca.pem',
+  ...                              ssl_certfile='/path/to/client.pem',
+  ...                              ssl_keyfile='/path/to/key.pem')
+
+These options can also be passed as part of the MongoDB URI.

doc/faq.rst

@@ -95,8 +95,98 @@ To use MongoDB with `Tornado <http://www.tornadoweb.org/>`_ see the
 `Motor <https://github.com/mongodb/motor>`_ project.
 
 For `Twisted <http://twistedmatrix.com/>`_, see `TxMongo
-<http://github.com/fiorix/mongo-async-python-driver>`_. Compared to PyMongo,
-TxMongo is less stable, lacks features, and is less actively maintained.
+<https://github.com/twisted/txmongo>`_. Its stated mission is to keep feature
+parity with PyMongo.
+
+Key order in subdocuments -- why does my query work in the shell but not PyMongo?
+---------------------------------------------------------------------------------
+
+.. testsetup:: key-order
+
+  from bson.son import SON
+  from pymongo.mongo_client import MongoClient
+
+  collection = MongoClient().test.collection
+  collection.drop()
+  collection.insert({'_id': 1.0,
+                     'subdocument': SON([('b', 1.0), ('a', 1.0)])})
+
+The key-value pairs in a BSON document can have any order (except that ``_id``
+is always first). The mongo shell preserves key order when reading and writing
+data. Observe that "b" comes before "a" when we create the document and when it
+is displayed:
+
+.. code-block:: javascript
+
+  > // mongo shell.
+  > db.collection.insert( { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } )
+  WriteResult({ "nInserted" : 1 })
+  > db.collection.find()
+  { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } }
+
+PyMongo represents BSON documents as Python dicts by default, and the order
+of keys in dicts is not defined. That is, a dict declared with the "a" key
+first is the same, to Python, as one with "b" first::
+
+  >>> print({'a': 1.0, 'b': 1.0})
+  {'a': 1.0, 'b': 1.0}
+  >>> print({'b': 1.0, 'a': 1.0})
+  {'a': 1.0, 'b': 1.0}
+
+Therefore, Python dicts are not guaranteed to show keys in the order they are
+stored in BSON. Here, "a" is shown before "b"::
+
+  >>> print(collection.find_one())
+  {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}}
+
+To preserve order when reading BSON, use the :class:`~bson.son.SON` class,
+which is a dict that remembers its key order. Now, documents and subdocuments
+in query results are represented with :class:`~bson.son.SON` objects::
+
+  >>> from bson.son import SON
+  >>> print(collection.find_one(as_class=SON))
+  SON([(u'_id', 1.0), (u'subdocument', SON([(u'b', 1.0), (u'a', 1.0)]))])
+
+The subdocument's actual storage layout is now visible: "b" is before "a".
+
+Because a dict's key order is not defined, you cannot predict how it will be
+serialized **to** BSON. But MongoDB considers subdocuments equal only if their
+keys have the same order. So if you use a dict to query on a subdocument it may
+not match::
+
+  >>> collection.find_one({'subdocument': {'a': 1.0, 'b': 1.0}}) is None
+  True
+
+Swapping the key order in your query makes no difference::
+
+  >>> collection.find_one({'subdocument': {'b': 1.0, 'a': 1.0}}) is None
+  True
+
+... because, as we saw above, Python considers the two dicts the same.
+
+There are two solutions. First, you can match the subdocument field-by-field::
+
+  >>> collection.find_one({'subdocument.a': 1.0,
+  ...                      'subdocument.b': 1.0})
+  {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}}
+
+The query matches any subdocument with an "a" of 1.0 and a "b" of 1.0,
+regardless of the order you specify them in Python or the order they are stored
+in BSON. Additionally, this query now matches subdocuments with additional
+keys besides "a" and "b", whereas the previous query required an exact match.
+
+The second solution is to use a :class:`~bson.son.SON` to specify the key order::
+
+  >>> query = {'subdocument': SON([('b', 1.0), ('a', 1.0)])}
+  >>> collection.find_one(query)
+  {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}}
+
+The key order you use when you create a :class:`~bson.son.SON` is preserved
+when it is serialized to BSON and used as a query. Thus you can create a
+subdocument that exactly matches the subdocument in the collection.
+
+.. seealso:: `MongoDB Manual entry on subdocument matching
+   <http://docs.mongodb.org/manual/tutorial/query-documents/#embedded-documents>`_.
 
 What does *CursorNotFound* cursor id not valid at server mean?
 --------------------------------------------------------------

doc/index.rst

@@ -1,6 +1,11 @@
 PyMongo |release| Documentation
 ===============================
 
+.. warning:: PyMongo 2.x is in maintenance mode. Support for new MongoDB
+  features ended with the release of MongoDB 3.0 and PyMongo 2.8. Users are
+  strongly encouraged to upgrade to PyMongo 3.x. See the
+  :doc:`/migrate-to-pymongo3` for details.
+
 Overview
 --------
 **PyMongo** is a Python distribution containing tools for working with
@@ -19,12 +24,25 @@ everything you need to know to use **PyMongo**.
 :doc:`examples/index`
   Examples of how to perform specific tasks.
 
+:doc:`atlas`
+  Using PyMongo 2.x with MongoDB Atlas.
+
+:doc:`examples/tls`
+  Using PyMongo 2.x with TLS / SSL.
+
 :doc:`faq`
   Some questions that come up often.
 
+:doc:`migrate-to-pymongo3`
+  A PyMongo 2.x to 3.x migration guide.
+
 :doc:`python3`
   Frequently asked questions about python 3 support.
 
+:doc:`compatibility-policy`
+  Explanation of deprecations, and how to keep pace with changes in PyMongo's
+  API.
+
 :doc:`api/index`
   The complete API documentation, organized by module.
 
@@ -79,13 +97,15 @@ Indices and tables
 .. toctree::
    :hidden:
 
+   atlas
    installation
    tutorial
    examples/index
    faq
+   compatibility-policy
    api/index
    tools
    contributors
    changelog
    python3
-
+   migrate-to-pymongo3

doc/installation.rst

@@ -5,6 +5,10 @@ Installing / Upgrading
 **PyMongo** is in the `Python Package Index
 <http://pypi.python.org/pypi/pymongo/>`_.
 
+.. warning:: **Do not install the "bson" package.** PyMongo comes with its own
+   bson package; doing "pip install bson" or "easy_install bson" installs a
+   third-party package that is incompatible with PyMongo.
+
 Microsoft Windows
 -----------------
 
@@ -21,7 +25,7 @@ to install pymongo on platforms other than Windows::
 
 To get a specific version of pymongo::
 
-  $ pip install pymongo==2.6.3
+  $ pip install pymongo==2.8.1
 
 To upgrade using pip::
 
@@ -188,7 +192,7 @@ PyMongo source directory::
   $ python setup.py bdist_egg
 
 The egg package can be found in the dist/ subdirectory. The file name will
-resemble “pymongo-2.6.3-py2.7-linux-x86_64.egg” but may have a different name
+resemble “pymongo-2.8.1-py2.7-linux-x86_64.egg” but may have a different name
 depending on your platform and the version of python you use to compile.
 
 .. warning::
@@ -201,7 +205,7 @@ depending on your platform and the version of python you use to compile.
 Copy this file to the target system and issue the following command to install the
 package::
 
-  $ sudo easy_install pymongo-2.6.3-py2.7-linux-x86_64.egg
+  $ sudo easy_install pymongo-2.8.1-py2.7-linux-x86_64.egg
 
 Installing a release candidate
 ------------------------------
@@ -212,9 +216,9 @@ but can be found on the
 `github tags page <https://github.com/mongodb/mongo-python-driver/tags>`_.
 They can be installed by passing the full URL for the tag to pip::
 
-  $ pip install https://github.com/mongodb/mongo-python-driver/archive/2.7rc1.tar.gz
+  $ pip install https://github.com/mongodb/mongo-python-driver/archive/2.9rc0.tar.gz
 
 or easy_install::
 
-  $ easy_install https://github.com/mongodb/mongo-python-driver/archive/2.7rc1.tar.gz
+  $ easy_install https://github.com/mongodb/mongo-python-driver/archive/2.9rc0.tar.gz
 

doc/migrate-to-pymongo3.rst

@@ -0,0 +1,560 @@
+PyMongo 3 Migration Guide
+=========================
+
+.. contents::
+
+.. testsetup::
+
+  from pymongo import MongoClient, ReadPreference
+  client = MongoClient()
+  collection = client.my_database.my_collection
+
+PyMongo 3 is a partial rewrite bringing a large number of improvements. It
+also brings a number of backward breaking changes. This guide provides a
+roadmap for migrating an existing application from PyMongo 2.x to 3.x or
+writing libraries that will work with both PyMongo 2.x and 3.x.
+
+PyMongo 2.9
+-----------
+
+The first step in any successful migration involves upgrading to, or
+requiring, at least PyMongo 2.9. If your project has a
+requirements.txt file, add the line "pymongo >= 2.9, < 3.0" until you have
+completely migrated to PyMongo 3. Most of the key new
+methods and options from PyMongo 3.0 are backported in PyMongo 2.9 making
+migration much easier.
+
+Enable Deprecation Warnings
+---------------------------
+
+Starting with PyMongo 2.9, :exc:`DeprecationWarning` is raised by most methods
+removed in PyMongo 3.0. Make sure you enable runtime warnings to see
+where deprecated functions and methods are being used in your application::
+
+  python -Wd <your application>
+
+Warnings can also be changed to errors::
+
+  python -Wd -Werror <your application>
+
+.. note:: Not all deprecated features raise :exc:`DeprecationWarning` when
+  used. For example, the :meth:`~pymongo.collection.Collection.find` options
+  renamed in PyMongo 3.0 do not raise :exc:`DeprecationWarning` when used in
+  PyMongo 2.x. See also `Removed features with no migration path`_.
+
+CRUD API
+--------
+
+Changes to find() and find_one()
+................................
+
+"spec" renamed "filter"
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The `spec` option has been renamed to `filter`. Code like this::
+
+  >>> cursor = collection.find(spec={"a": 1})
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> cursor = collection.find(filter={"a": 1})
+
+or this with any version of PyMongo:
+
+.. doctest::
+
+  >>> cursor = collection.find({"a": 1})
+
+"fields" renamed "projection"
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `fields` option has been renamed to `projection`. Code like this::
+
+  >>> cursor = collection.find({"a": 1}, fields={"_id": False})
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> cursor = collection.find({"a": 1}, projection={"_id": False})
+
+or this with any version of PyMongo:
+
+.. doctest::
+
+  >>> cursor = collection.find({"a": 1}, {"_id": False})
+
+"partial" renamed "allow_partial_results"
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `partial` option has been renamed to `allow_partial_results`. Code like
+this::
+
+  >>> cursor = collection.find({"a": 1}, partial=True)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> cursor = collection.find({"a": 1}, allow_partial_results=True)
+
+"timeout" replaced by "no_cursor_timeout"
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `timeout` option has been replaced by `no_cursor_timeout`. Code like this::
+
+  >>> cursor = collection.find({"a": 1}, timeout=False)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> cursor = collection.find({"a": 1}, no_cursor_timeout=True)
+
+"snapshot" and "max_scan" replaced by "modifiers"
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `snapshot` and `max_scan` options have been removed. They can now be set,
+along with other $ query modifiers, through the `modifiers` option. Code like
+this::
+
+  >>> cursor = collection.find({"a": 1}, snapshot=True)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> cursor = collection.find({"a": 1}, modifiers={"$snapshot": True})
+
+or with any version of PyMongo:
+
+.. doctest::
+
+  >>> cursor = collection.find({"$query": {"a": 1}, "$snapshot": True})
+
+"network_timeout" is removed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `network_timeout` option has been removed. This option was always the
+wrong solution for timing out long running queries and should never be used
+in production. Starting with **MongoDB 2.6** you can use the $maxTimeMS query
+modifier. Code like this::
+
+  # Set a 5 second select() timeout.
+  >>> cursor = collection.find({"a": 1}, network_timeout=5)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  # Set a 5 second (5000 millisecond) server side query timeout.
+  >>> cursor = collection.find({"a": 1}, modifiers={"$maxTimeMS": 5000})
+
+or with any version of PyMongo:
+
+.. doctest::
+
+  >>> cursor = collection.find({"$query": {"a": 1}, "$maxTimeMS": 5000})
+
+.. seealso:: `$maxTimeMS
+  <http://docs.mongodb.org/manual/reference/operator/meta/maxTimeMS/>`_
+
+Tailable cursors
+~~~~~~~~~~~~~~~~
+
+The `tailable` and `await_data` options have been replaced by `cursor_type`.
+Code like this::
+
+  >>> cursor = collection.find({"a": 1}, tailable=True)
+  >>> cursor = collection.find({"a": 1}, tailable=True, await_data=True)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from pymongo import CursorType
+  >>> cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE)
+  >>> cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE_AWAIT)
+
+Other removed options
+~~~~~~~~~~~~~~~~~~~~~
+
+The `slave_okay`, `read_preference`, `tag_sets`,
+and `secondary_acceptable_latency_ms` options have been removed. See the `Read
+Preferences`_ section for solutions.
+
+The aggregate method always returns a cursor
+............................................
+
+PyMongo 2.6 added an option to return an iterable cursor from
+:meth:`~pymongo.collection.Collection.aggregate`. In PyMongo 3
+:meth:`~pymongo.collection.Collection.aggregate` always returns a cursor. Use
+the `cursor` option for consistent behavior with PyMongo 2.9 and later:
+
+.. doctest::
+
+  >>> for result in collection.aggregate([], cursor={}):
+  ...     pass
+
+Read Preferences
+----------------
+
+The "slave_okay" option is removed
+..................................
+
+The `slave_okay` option is removed from PyMongo's API. The
+secondaryPreferred read preference provides the same behavior.
+Code like this::
+
+  >>> client = MongoClient(slave_okay=True)
+
+can be changed to this with PyMongo 2.9 or newer:
+
+.. doctest::
+
+  >>> client = MongoClient(readPreference="secondaryPreferred")
+
+The "read_preference" attribute is immutable
+............................................
+
+Code like this::
+
+  >>> from pymongo import ReadPreference
+  >>> db = client.my_database
+  >>> db.read_preference = ReadPreference.SECONDARY
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> db = client.get_database("my_database",
+  ...                          read_preference=ReadPreference.SECONDARY)
+
+Code like this::
+
+  >>> cursor = collection.find({"a": 1},
+  ...                          read_preference=ReadPreference.SECONDARY)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> coll2 = collection.with_options(read_preference=ReadPreference.SECONDARY)
+  >>> cursor = coll2.find({"a": 1})
+
+.. seealso:: :meth:`~pymongo.database.Database.get_collection`
+
+The "tag_sets" option and attribute are removed
+...............................................
+
+The `tag_sets` MongoClient option is removed. The `read_preference`
+option can be used instead. Code like this::
+
+  >>> client = MongoClient(
+  ...     read_preference=ReadPreference.SECONDARY,
+  ...     tag_sets=[{"dc": "ny"}, {"dc": "sf"}])
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from pymongo.read_preferences import Secondary
+  >>> client = MongoClient(read_preference=Secondary([{"dc": "ny"}]))
+
+To change the tags sets for a Database or Collection, code like this::
+
+  >>> db = client.my_database
+  >>> db.read_preference = ReadPreference.SECONDARY
+  >>> db.tag_sets = [{"dc": "ny"}]
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> db = client.get_database("my_database",
+  ...                          read_preference=Secondary([{"dc": "ny"}]))
+
+Code like this::
+
+  >>> cursor = collection.find(
+  ...     {"a": 1},
+  ...     read_preference=ReadPreference.SECONDARY,
+  ...     tag_sets=[{"dc": "ny"}])
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from pymongo.read_preferences import Secondary
+  >>> coll2 = collection.with_options(
+  ...     read_preference=Secondary([{"dc": "ny"}]))
+  >>> cursor = coll2.find({"a": 1})
+
+.. seealso:: :meth:`~pymongo.database.Database.get_collection`
+
+The "secondary_acceptable_latency_ms" option and attribute are removed
+......................................................................
+
+PyMongo 2.x supports `secondary_acceptable_latency_ms` as an option to methods
+throughout the driver, but mongos only supports a global latency option.
+PyMongo 3.x has changed to match the behavior of mongos, allowing migration
+from a single server, to a replica set, to a sharded cluster without a
+surprising change in server selection behavior. A new option,
+`localThresholdMS`, is available through MongoClient and should be used in
+place of `secondaryAcceptableLatencyMS`. Code like this::
+
+  >>> client = MongoClient(readPreference="nearest",
+  ...                      secondaryAcceptableLatencyMS=100)
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> client = MongoClient(readPreference="nearest",
+  ...                      localThresholdMS=100)
+
+Write Concern
+-------------
+
+The "safe" option is removed
+............................
+
+In PyMongo 3 the `safe` option is removed from the entire API.
+:class:`~pymongo.mongo_client.MongoClient` has always defaulted to acknowledged
+write operations and continues to do so in PyMongo 3.
+
+The "write_concern" attribute is immutable
+..........................................
+
+The `write_concern` attribute is immutable in PyMongo 3. Code like this::
+
+  >>> client = MongoClient()
+  >>> client.write_concern = {"w": "majority"}
+
+can be changed to this with any version of PyMongo:
+
+.. doctest::
+
+  >>> client = MongoClient(w="majority")
+
+Code like this::
+
+  >>> db = client.my_database
+  >>> db.write_concern = {"w": "majority"}
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from pymongo import WriteConcern
+  >>> db = client.get_database("my_database",
+  ...                          write_concern=WriteConcern(w="majority"))
+
+The new CRUD API write methods do not accept write concern options. Code like
+this::
+
+  >>> oid = collection.insert({"a": 2}, w="majority")
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from pymongo import WriteConcern
+  >>> coll2 = collection.with_options(
+  ...     write_concern=WriteConcern(w="majority"))
+  >>> oid = coll2.insert({"a": 2})
+
+.. seealso:: :meth:`~pymongo.database.Database.get_collection`
+
+Codec Options
+-------------
+
+The "document_class" attribute is removed
+.........................................
+
+Code like this::
+
+  >>> from bson.son import SON
+  >>> client = MongoClient()
+  >>> client.document_class = SON
+
+can be replaced by this in any version of PyMongo:
+
+.. doctest::
+
+  >>> from bson.son import SON
+  >>> client = MongoClient(document_class=SON)
+
+or to change the `document_class` for a :class:`~pymongo.database.Database`
+with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from bson.codec_options import CodecOptions
+  >>> from bson.son import SON
+  >>> db = client.get_database("my_database", CodecOptions(SON))
+
+.. seealso:: :meth:`~pymongo.database.Database.get_collection` and
+  :meth:`~pymongo.collection.Collection.with_options`
+
+The "uuid_subtype" option and attribute are removed
+...................................................
+
+Code like this::
+
+  >>> from bson.binary import JAVA_LEGACY
+  >>> db = client.my_database
+  >>> db.uuid_subtype = JAVA_LEGACY
+
+can be replaced by this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from bson.binary import JAVA_LEGACY
+  >>> from bson.codec_options import CodecOptions
+  >>> db = client.get_database("my_database",
+  ...                          CodecOptions(uuid_representation=JAVA_LEGACY))
+
+.. seealso:: :meth:`~pymongo.database.Database.get_collection` and
+  :meth:`~pymongo.collection.Collection.with_options`
+
+MongoClient
+-----------
+
+MongoClient connects asynchronously
+...................................
+
+In PyMongo 3, the :class:`~pymongo.mongo_client.MongoClient` constructor no
+longer blocks while connecting to the server or servers, and it no longer
+raises :exc:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor
+:exc:`~pymongo.errors.ConfigurationError` if the user’s credentials are wrong.
+Instead, the constructor returns immediately and launches the connection
+process on background threads. The `connect` option is added to control whether
+these threads are started immediately, or when the client is first used.
+
+For consistent behavior in PyMongo 2.x and PyMongo 3.x, code like this::
+
+  >>> from pymongo.errors import ConnectionFailure
+  >>> try:
+  ...     client = MongoClient()
+  ... except ConnectionFailure:
+  ...     print("Server not available")
+  >>>
+
+can be changed to this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from pymongo.errors import ConnectionFailure
+  >>> client = MongoClient(connect=False)
+  >>> try:
+  ...     result = client.admin.command("ismaster")
+  ... except ConnectionFailure:
+  ...     print("Server not available")
+  >>>
+
+Any operation can be used to determine if the server is available. We choose
+the "ismaster" command here because it is cheap and does not require auth, so
+it is a simple way to check whether the server is available.
+
+The max_pool_size parameter is removed
+......................................
+
+PyMongo 3 replaced the max_pool_size parameter with support for the MongoDB URI
+`maxPoolSize` option. Code like this::
+
+  >>> client = MongoClient(max_pool_size=10)
+
+can be replaced by this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> client = MongoClient(maxPoolSize=10)
+  >>> client = MongoClient("mongodb://localhost:27017/?maxPoolSize=10")
+
+The "disconnect" method is removed
+..................................
+
+Code like this::
+
+  >>> client.disconnect()
+
+can be replaced by this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> client.close()
+
+The host and port attributes are removed
+........................................
+
+Code like this::
+
+  >>> host = client.host
+  >>> port = client.port
+
+can be replaced by this with PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> address = client.address
+  >>> host, port = address or (None, None)
+
+BSON
+----
+
+"as_class", "tz_aware", and "uuid_subtype" are removed
+......................................................
+
+The `as_class`, `tz_aware`, and `uuid_subtype` parameters have been
+removed from the functions provided in :mod:`bson`. Code like this::
+
+  >>> from bson import BSON
+  >>> from bson.son import SON
+  >>> encoded = BSON.encode({"a": 1}, as_class=SON)
+
+can be replaced by this in PyMongo 2.9 or later:
+
+.. doctest::
+
+  >>> from bson import BSON
+  >>> from bson.codec_options import CodecOptions
+  >>> from bson.son import SON
+  >>> encoded = BSON.encode({"a": 1}, codec_options=CodecOptions(SON))
+
+Removed features with no migration path
+---------------------------------------
+
+MasterSlaveConnection is removed
+................................
+
+Master slave deployments are deprecated in MongoDB. Starting with MongoDB 3.0
+a replica set can have up to 50 members and that limit is likely to be
+removed in later releases. We recommend migrating to replica sets instead.
+
+Requests are removed
+....................
+
+The client methods `start_request`, `in_request`, and `end_request` are
+removed. Requests were designed to make read-your-writes consistency more
+likely with the w=0 write concern. Additionally, a thread in a request used the
+same member for all secondary reads in a replica set. To ensure
+read-your-writes consistency in PyMongo 3.0, do not override the default write
+concern with w=0, and do not override the default read preference of PRIMARY.
+
+The "compile_re" option is removed
+..................................
+
+In PyMongo 3 regular expressions are never compiled to Python match objects.
+
+The "use_greenlets" option is removed
+.....................................
+
+The `use_greenlets` option was meant to allow use of PyMongo with Gevent
+without the use of gevent.monkey.patch_threads(). This option caused a lot
+of confusion and made it difficult to support alternative asyncio libraries
+like Eventlet. Users of Gevent should use gevent.monkey.patch_all() instead.
+
+.. seealso:: :doc:`examples/gevent`

doc/mongo_extensions.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

doc/tools.rst

@@ -71,7 +71,7 @@ Minimongo
   and provides a number of additional features, including a simple
   document-oriented interface, connection pooling, index management, and
   collection & database naming helpers. The `source is on github
-  <http://github.com/slacy/minimongo>`_.
+  <https://github.com/MiniMongo/minimongo>`_.
 
 Manga
   `Manga <http://pypi.python.org/pypi/manga>`_ aims to be a simpler ORM-like
@@ -107,6 +107,9 @@ various Python frameworks and libraries.
   project to enable using MongoDB as a backend for `beaker's
   <http://beaker.groovie.org/>`_ caching / session system.
   `The source is on github <http://github.com/bwmcadams/mongodb_beaker>`_.
+* `Log4Mongo <https://github.com/log4mongo/log4mongo-python>`_ is a flexible
+  Python logging handler that can store logs in MongoDB using normal and capped
+  collections.
 * `MongoLog <http://github.com/puentesarrin/mongodb-log/>`_ is a Python logging
   handler that stores logs in MongoDB using a capped collection.
 * `c5t <http://bitbucket.org/percious/c5t/>`_ is a content-management system
@@ -132,6 +135,5 @@ These are alternatives to PyMongo.
 
 * `Motor <https://github.com/mongodb/motor>`_ is a full-featured, non-blocking
   MongoDB driver for Python Tornado applications.
-* `TxMongo <http://github.com/fiorix/mongo-async-python-driver>`_ is an
-  asynchronous Python driver for MongoDB, although it is not currently
-  recommended for production use.
+* `TxMongo <https://github.com/twisted/txmongo>`_ is an asynchronous Twisted
+  Python driver for MongoDB.

doc/tutorial.rst

@@ -144,8 +144,8 @@ of the collections in our database:
 
 .. doctest::
 
-  >>> db.collection_names()
-  [u'system.indexes', u'posts']
+  >>> db.collection_names(include_system_collections=False)
+  [u'posts']
 
 .. note:: The *system.indexes* collection is a special internal
    collection that was created automatically.
@@ -162,8 +162,13 @@ document from the posts collection:
 
 .. doctest::
 
-  >>> posts.find_one()
-  {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']}
+  >>> import pprint
+  >>> pprint.pprint(posts.find_one())
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'mongodb', u'python', u'pymongo'],
+   u'text': u'My first blog post!'}
 
 The result is a dictionary matching the one that we inserted previously.
 
@@ -176,8 +181,12 @@ our results to a document with author "Mike" we do:
 
 .. doctest::
 
-  >>> posts.find_one({"author": "Mike"})
-  {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']}
+  >>> pprint.pprint(posts.find_one({"author": "Mike"}))
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'mongodb', u'python', u'pymongo'],
+   u'text': u'My first blog post!'}
 
 If we try with a different author, like "Eliot", we'll get no result:
 
@@ -194,10 +203,14 @@ We can also find a post by its ``_id``, which in our example is an ObjectId:
 
 .. doctest::
 
-  >>> post_id
-  ObjectId(...)
-  >>> posts.find_one({"_id": post_id})
-  {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']}
+    >>> post_id
+    ObjectId(...)
+    >>> pprint.pprint(posts.find_one({"_id": post_id}))
+    {u'_id': ObjectId('...'),
+     u'author': u'Mike',
+     u'date': datetime.datetime(...),
+     u'tags': [u'mongodb', u'python', u'pymongo'],
+     u'text': u'My first blog post!'}
 
 Note that an ObjectId is not the same as its string representation:
 
@@ -282,11 +295,23 @@ document in the ``posts`` collection:
 .. doctest::
 
   >>> for post in posts.find():
-  ...   post
+  ...   pprint.pprint(post)
   ...
-  {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']}
-  {u'date': datetime.datetime(2009, 11, 12, 11, 14), u'text': u'Another post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'bulk', u'insert']}
-  {u'date': datetime.datetime(2009, 11, 10, 10, 45), u'text': u'and pretty easy too!', u'_id': ObjectId('...'), u'author': u'Eliot', u'title': u'MongoDB is fun'}
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'mongodb', u'python', u'pymongo'],
+   u'text': u'My first blog post!'}
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'bulk', u'insert'],
+   u'text': u'Another post!'}
+  {u'_id': ObjectId('...'),
+   u'author': u'Eliot',
+   u'date': datetime.datetime(...),
+   u'text': u'and pretty easy too!',
+   u'title': u'MongoDB is fun'}
 
 Just like we did with :meth:`~pymongo.collection.Collection.find_one`,
 we can pass a document to :meth:`~pymongo.collection.Collection.find`
@@ -296,10 +321,18 @@ author is "Mike":
 .. doctest::
 
   >>> for post in posts.find({"author": "Mike"}):
-  ...   post
+  ...   pprint.pprint(post)
   ...
-  {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']}
-  {u'date': datetime.datetime(2009, 11, 12, 11, 14), u'text': u'Another post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'bulk', u'insert']}
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'mongodb', u'python', u'pymongo'],
+   u'text': u'My first blog post!'}
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'bulk', u'insert'],
+   u'text': u'Another post!'}
 
 Counting
 --------
@@ -331,10 +364,18 @@ than a certain date, but also sort the results by author:
 
   >>> d = datetime.datetime(2009, 11, 12, 12)
   >>> for post in posts.find({"date": {"$lt": d}}).sort("author"):
-  ...   print post
+  ...   pprint.pprint(post)
   ...
-  {u'date': datetime.datetime(2009, 11, 10, 10, 45), u'text': u'and pretty easy too!', u'_id': ObjectId('...'), u'author': u'Eliot', u'title': u'MongoDB is fun'}
-  {u'date': datetime.datetime(2009, 11, 12, 11, 14), u'text': u'Another post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'bulk', u'insert']}
+  {u'_id': ObjectId('...'),
+   u'author': u'Eliot',
+   u'date': datetime.datetime(...),
+   u'text': u'and pretty easy too!',
+   u'title': u'MongoDB is fun'}
+  {u'_id': ObjectId('...'),
+   u'author': u'Mike',
+   u'date': datetime.datetime(...),
+   u'tags': [u'bulk', u'insert'],
+   u'text': u'Another post!'}
 
 Here we use the special ``"$lt"`` operator to do a range query, and
 also call :meth:`~pymongo.cursor.Cursor.sort` to sort the results
@@ -342,33 +383,46 @@ by author.
 
 Indexing
 --------
-To make the above query fast we can add a compound index on
-``"date"`` and ``"author"``. To start, lets use the
-:meth:`~pymongo.cursor.Cursor.explain` method to get some information
-about how the query is being performed without the index:
+
+Adding indexes can help accelerate certain queries and can also add additional
+functionality to querying and storing documents. In this example, we'll
+demonstrate how to create a `unique index
+<http://docs.mongodb.org/manual/core/index-unique/>`_ on a key that rejects
+documents whose value for that key already exists in the index.
+
+First, we'll need to create the index:
 
 .. doctest::
 
-  >>> posts.find({"date": {"$lt": d}}).sort("author").explain()["cursor"]
-  u'BasicCursor'
-  >>> posts.find({"date": {"$lt": d}}).sort("author").explain()["nscanned"]
-  3
+   >>> result = db.profiles.create_index([('user_id', pymongo.ASCENDING)],
+   ...                                   unique=True)
+   >>> sorted(list(db.profiles.index_information()))
+   [u'_id_', u'user_id_1']
 
-We can see that the query is using the *BasicCursor* and scanning over
-all 3 documents in the collection. Now let's add a compound index and
-look at the same information:
+Notice that we have two indexes now: one is the index on ``_id`` that MongoDB
+creates automatically, and the other is the index on ``user_id`` we just
+created.
+
+Now let's set up some user profiles:
 
 .. doctest::
 
-  >>> from pymongo import ASCENDING, DESCENDING
-  >>> posts.create_index([("date", DESCENDING), ("author", ASCENDING)])
-  u'date_-1_author_1'
-  >>> posts.find({"date": {"$lt": d}}).sort("author").explain()["cursor"]
-  u'BtreeCursor date_-1_author_1'
-  >>> posts.find({"date": {"$lt": d}}).sort("author").explain()["nscanned"]
-  2
+   >>> user_profiles = [
+   ...     {'user_id': 211, 'name': 'Luke'},
+   ...     {'user_id': 212, 'name': 'Ziltoid'}]
+   >>> result = db.profiles.insert_many(user_profiles)
 
-Now the query is using a *BtreeCursor* (the index) and only scanning
-over the 2 matching documents.
+The index prevents us from inserting a document whose ``user_id`` is already in
+the collection:
+
+.. doctest::
+  :options: +IGNORE_EXCEPTION_DETAIL
+
+   >>> new_profile = {'user_id': 213, 'name': 'Drew'}
+   >>> duplicate_profile = {'user_id': 212, 'name': 'Tommy'}
+   >>> result = db.profiles.insert_one(new_profile)  # This is fine.
+   >>> result = db.profiles.insert_one(duplicate_profile)
+   Traceback (most recent call last):
+   DuplicateKeyError: E11000 duplicate key error index: test_database.profiles.$user_id_1 dup key: { : 212 }
 
 .. seealso:: The MongoDB documentation on `indexes <http://www.mongodb.org/display/DOCS/Indexes>`_

gridfs/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -279,6 +279,33 @@ class GridFS(object):
             name for name in self.__files.distinct("filename")
             if name is not None]
 
+    def find_one(self, spec_or_id=None, *args, **kwargs):
+        """Get a single file from gridfs.
+
+        All arguments to :meth:`find` are also valid arguments for
+        :meth:`find_one`, although any `limit` argument will be
+        ignored. Returns a single :class:`~gridfs.grid_file.GridOut`,
+        or ``None`` if no matching file is found. For example::
+
+            file = fs.find_one({"filename": "lisa.txt"})
+
+        :Parameters:
+          - `spec_or_id` (optional): a dictionary specifying
+            the query to be performing OR any other type to be used  as
+            the value for a query for ``"_id"`` in the file collection.
+          - `*args` (optional): any additional positional arguments are
+            the same as the arguments to :meth:`find`.
+          - `**kwargs` (optional): any additional keyword arguments
+            are the same as the arguments to :meth:`find`.
+        """
+        if spec_or_id is not None and not isinstance(spec_or_id, dict):
+            spec_or_id = {"_id": spec_or_id}
+
+        for f in self.find(spec_or_id, *args, **kwargs):
+            return f
+
+        return None
+
     def find(self, *args, **kwargs):
         """Query GridFS for files.
 
@@ -333,6 +360,15 @@ class GridFS(object):
           - `compile_re` (optional): if ``False``, don't attempt to compile
             BSON regex objects into Python regexes. Return instances of
             :class:`~bson.regex.Regex` instead.
+          - `filter` (optional): a SON object specifying elements which
+            must be present for a document to be included in the
+            result set. Takes precedence over `spec`.
+          - `no_cursor_timeout` (optional): if False (the default), any
+            returned cursor is closed by the server after 10 minutes of
+            inactivity. If set to True, the returned cursor will never
+            time out on the server. Care should be taken to ensure that
+            cursors with no_cursor_timeout turned on are properly closed.
+            Takes precedence over `timeout`.
 
         Raises :class:`TypeError` if any of the arguments are of
         improper type. Returns an instance of

gridfs/errors.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

gridfs/grid_file.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,6 +30,7 @@ from pymongo import ASCENDING
 from pymongo.collection import Collection
 from pymongo.cursor import Cursor
 from pymongo.errors import DuplicateKeyError
+from pymongo.read_preferences import ReadPreference
 
 try:
     _SEEK_SET = os.SEEK_SET
@@ -110,7 +111,7 @@ class GridIn(object):
             for the file
 
           - ``"chunkSize"`` or ``"chunk_size"``: size of each of the
-            chunks, in bytes (default: 256 kb)
+            chunks, in bytes (default: 255 kb)
 
           - ``"encoding"``: encoding used for this file. In Python 2,
             any :class:`unicode` that is written to the file will be
@@ -255,10 +256,11 @@ class GridIn(object):
             # connection, can succeed out-of-order due to the writebackListener.
             # We mustn't call "filemd5" until all inserts are complete, which
             # we ensure by calling getLastError (and ignoring the result).
-            db.error()
+            db.command('getlasterror', read_preference=ReadPreference.PRIMARY)
 
             md5 = db.command(
-                "filemd5", self._id, root=self._coll.name)["md5"]
+                "filemd5", self._id, root=self._coll.name,
+                read_preference=ReadPreference.PRIMARY)["md5"]
 
             self._file["md5"] = md5
             self._file["length"] = self._position
@@ -439,19 +441,24 @@ class GridOut(object):
         """Reads a chunk at a time. If the current position is within a
         chunk the remainder of the chunk is returned.
         """
+        self._ensure_file()
         received = len(self.__buffer)
         chunk_data = EMPTY
+        chunk_size = int(self.chunk_size)
 
         if received > 0:
             chunk_data = self.__buffer
         elif self.__position < int(self.length):
-            chunk_number = int((received + self.__position) / self.chunk_size)
+            chunk_number = int((received + self.__position) / chunk_size)
             chunk = self.__chunks.find_one({"files_id": self._id,
                                             "n": chunk_number})
             if not chunk:
                 raise CorruptGridFile("no chunk #%d" % chunk_number)
 
-            chunk_data = chunk["data"][self.__position % self.chunk_size:]
+            chunk_data = chunk["data"][self.__position % chunk_size:]
+
+            if not chunk_data:
+                raise CorruptGridFile("truncated chunk")
 
         self.__position += len(chunk_data)
         self.__buffer = EMPTY
@@ -626,7 +633,8 @@ class GridOutCursor(Cursor):
     def __init__(self, collection, spec=None, skip=0, limit=0,
                  timeout=True, sort=None, max_scan=None,
                  read_preference=None, tag_sets=None,
-                 secondary_acceptable_latency_ms=None, compile_re=True):
+                 secondary_acceptable_latency_ms=None, compile_re=True,
+                 filter=None, no_cursor_timeout=None):
         """Create a new cursor, similar to the normal
         :class:`~pymongo.cursor.Cursor`.
 
@@ -646,11 +654,18 @@ class GridOutCursor(Cursor):
         latency = (secondary_acceptable_latency_ms
                    or collection.files.secondary_acceptable_latency_ms)
 
+        # If backported args are set, pass them to Cursor.
+        extra_args = {}
+        if filter is not None:
+            extra_args["filter"] = filter
+        if no_cursor_timeout is not None:
+            extra_args["no_cursor_timeout"] = no_cursor_timeout
+
         super(GridOutCursor, self).__init__(
             collection.files, spec, skip=skip, limit=limit, timeout=timeout,
             sort=sort, max_scan=max_scan, read_preference=read_preference,
             secondary_acceptable_latency_ms=latency, compile_re=compile_re,
-            tag_sets=tag_sets)
+            tag_sets=tag_sets, **extra_args)
 
     def next(self):
         """Get next GridOut object from cursor.

pymongo/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ GEO2D = "2d"
 
 .. note:: Geo-spatial indexing requires server version **>= 1.3.3**.
 
-.. _geospatial index: http://docs.mongodb.org/manual/core/geospatial-indexes/
+.. _geospatial index: http://docs.mongodb.org/manual/core/2d/
 """
 
 GEOHAYSTACK = "geoHaystack"
@@ -37,7 +37,7 @@ GEOHAYSTACK = "geoHaystack"
 
 .. note:: Geo-spatial indexing requires server version **>= 1.5.6**.
 
-.. _haystack index: http://docs.mongodb.org/manual/core/geospatial-indexes/#haystack-indexes
+.. _haystack index: http://docs.mongodb.org/manual/core/geohaystack/
 """
 
 GEOSPHERE = "2dsphere"
@@ -47,7 +47,7 @@ GEOSPHERE = "2dsphere"
 
 .. note:: 2dsphere indexing requires server version **>= 2.4.0**.
 
-.. _spherical geospatial index: http://docs.mongodb.org/manual/release-notes/2.4/#new-geospatial-indexes-with-geojson-and-improved-spherical-geometry
+.. _spherical geospatial index: http://docs.mongodb.org/manual/core/2dsphere/
 """
 
 HASHED = "hashed"
@@ -57,7 +57,17 @@ HASHED = "hashed"
 
 .. note:: hashed indexing requires server version **>= 2.4.0**.
 
-.. _hashed index: http://docs.mongodb.org/manual/release-notes/2.4/#new-hashed-index-and-sharding-with-a-hashed-shard-key
+.. _hashed index: http://docs.mongodb.org/manual/core/index-hashed/
+"""
+
+TEXT = "text"
+"""Index specifier for a `text index`_.
+
+.. versionadded:: 2.7.1
+
+.. note:: text search requires server version **>= 2.4.0**.
+
+.. _text index: http://docs.mongodb.org/manual/core/index-text/
 """
 
 OFF = 0
@@ -67,23 +77,32 @@ SLOW_ONLY = 1
 ALL = 2
 """Profile all operations."""
 
-version_tuple = (2, 7)
+version_tuple = (2, 9, 5)
 
 def get_version_string():
     if isinstance(version_tuple[-1], basestring):
         return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1]
     return '.'.join(map(str, version_tuple))
 
-version = get_version_string()
+__version__ = version = get_version_string()
 """Current version of PyMongo."""
 
+from pymongo.collection import ReturnDocument
 from pymongo.common import (MIN_SUPPORTED_WIRE_VERSION,
                             MAX_SUPPORTED_WIRE_VERSION)
 from pymongo.connection import Connection
+from pymongo.cursor import CursorType
 from pymongo.mongo_client import MongoClient
 from pymongo.mongo_replica_set_client import MongoReplicaSetClient
+from pymongo.operations import (InsertOne,
+                                DeleteOne,
+                                DeleteMany,
+                                UpdateOne,
+                                UpdateMany,
+                                ReplaceOne)
 from pymongo.replica_set_connection import ReplicaSetConnection
 from pymongo.read_preferences import ReadPreference
+from pymongo.write_concern import WriteConcern
 
 def has_c():
     """Is the C extension installed?

pymongo/_cmessagemodule.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2014 MongoDB, Inc.
+ * Copyright 2009-2015 MongoDB, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -181,16 +181,29 @@ static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) {
     buffer_t buffer;
     int length_location, message_length;
     PyObject* result;
+    PyObject* codec_options = Py_None;
+    PyObject* as_class;
+    unsigned char tz_aware;
 
-    if (!PyArg_ParseTuple(args, "et#ObbObb",
+    if (!PyArg_ParseTuple(args, "et#ObbObb|O",
                           "utf-8",
                           &collection_name,
                           &collection_name_length,
                           &docs, &check_keys, &safe,
                           &last_error_args,
-                          &continue_on_error, &uuid_subtype)) {
+                          &continue_on_error, &uuid_subtype,
+                          &codec_options)) {
         return NULL;
     }
+
+    if (codec_options != Py_None) {
+        if (!PyArg_ParseTuple(codec_options, "Obb",
+                              &as_class, &tz_aware, &uuid_subtype)) {
+            PyMem_Free(collection_name);
+            return NULL;
+        }
+    }
+
     if (continue_on_error) {
         options += 1;
     }
@@ -307,16 +320,28 @@ static PyObject* _cbson_update_message(PyObject* self, PyObject* args) {
     buffer_t buffer;
     int length_location, message_length;
     PyObject* result;
+    PyObject* codec_options = Py_None;
+    PyObject* as_class;
+    unsigned char tz_aware;
 
-    if (!PyArg_ParseTuple(args, "et#bbOObObb",
+    if (!PyArg_ParseTuple(args, "et#bbOObObb|O",
                           "utf-8",
                           &collection_name,
                           &collection_name_length,
                           &upsert, &multi, &spec, &doc, &safe,
-                          &last_error_args, &check_keys, &uuid_subtype)) {
+                          &last_error_args, &check_keys, &uuid_subtype,
+                          &codec_options)) {
         return NULL;
     }
 
+    if (codec_options != Py_None) {
+        if (!PyArg_ParseTuple(codec_options, "Obb",
+                              &as_class, &tz_aware, &uuid_subtype)) {
+            PyMem_Free(collection_name);
+            return NULL;
+        }
+    }
+
     options = 0;
     if (upsert) {
         options += 1;
@@ -410,16 +435,29 @@ static PyObject* _cbson_query_message(PyObject* self, PyObject* args) {
     buffer_t buffer;
     int length_location, message_length;
     PyObject* result;
+    PyObject* codec_options = Py_None;
+    PyObject* as_class;
+    unsigned char tz_aware;
 
-    if (!PyArg_ParseTuple(args, "Iet#iiO|Ob",
+    if (!PyArg_ParseTuple(args, "Iet#iiO|ObO",
                           &options,
                           "utf-8",
                           &collection_name,
                           &collection_name_length,
                           &num_to_skip, &num_to_return,
-                          &query, &field_selector, &uuid_subtype)) {
+                          &query, &field_selector, &uuid_subtype,
+                          &codec_options)) {
         return NULL;
     }
+
+    if (codec_options != Py_None) {
+        if (!PyArg_ParseTuple(codec_options, "Obb",
+                              &as_class, &tz_aware, &uuid_subtype)) {
+            PyMem_Free(collection_name);
+            return NULL;
+        }
+    }
+
     buffer = buffer_new();
     if (!buffer) {
         PyErr_NoMemory();
@@ -1112,8 +1150,10 @@ _cbson_do_batched_write_command(PyObject* self, PyObject* args) {
              */
             buffer_update_position(buffer, sub_doc_begin);
 
-            if (!buffer_write_bytes(buffer, "\x00\x00", 2))
+            if (!buffer_write_bytes(buffer, "\x00\x00", 2)) {
+                buffer_free(new_buffer);
                 goto cmditerfail;
+            }
 
             result = _send_write_command(client, buffer,
                                          lst_len_loc, cmd_len_loc, &errors);

pymongo/auth.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2014 MongoDB, Inc.
+# Copyright 2013-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,13 +15,19 @@
 """Authentication helpers."""
 
 import hmac
+import sys
+import warnings
 try:
     import hashlib
     _MD5 = hashlib.md5
+    _SHA1 = hashlib.sha1
+    _SHA1MOD = _SHA1
     _DMOD = _MD5
 except ImportError:  # for Python < 2.5
-    import md5
+    import md5, sha
     _MD5 = md5.new
+    _SHA1 = sha.new
+    _SHA1MOD = sha
     _DMOD = md5
 
 HAVE_KERBEROS = True
@@ -30,26 +36,274 @@ try:
 except ImportError:
     HAVE_KERBEROS = False
 
+from base64 import standard_b64decode, standard_b64encode
+from random import SystemRandom
+
 from bson.binary import Binary
-from bson.py3compat import b
+from bson.py3compat import b, PY3
 from bson.son import SON
 from pymongo.errors import ConfigurationError, OperationFailure
 
 
-MECHANISMS = frozenset(['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', 'PLAIN'])
+MECHANISMS = frozenset(
+    ['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', 'PLAIN', 'SCRAM-SHA-1', 'DEFAULT'])
 """The authentication mechanisms supported by PyMongo."""
 
 
 def _build_credentials_tuple(mech, source, user, passwd, extra):
     """Build and return a mechanism specific credentials tuple.
     """
+    user = unicode(user)
     if mech == 'GSSAPI':
-        gsn = extra.get('gssapiservicename', 'mongodb')
+        gsn = 'mongodb'
+        if "gssapiservicename" in extra:
+            gsn = extra.get('gssapiservicename')
+            msg = ('The gssapiServiceName option is deprecated. Use '
+                   '"authMechanismProperties=SERVICE_NAME:%s" instead.' % gsn)
+            warnings.warn(msg, DeprecationWarning, stacklevel=3)
+        # SERVICE_NAME overrides gssapiServiceName.
+        if 'authmechanismproperties' in extra:
+            props = extra['authmechanismproperties']
+            if 'SERVICE_NAME' in props:
+                gsn = props.get('SERVICE_NAME')
         # No password, source is always $external.
         return (mech, '$external', user, gsn)
     elif mech == 'MONGODB-X509':
         return (mech, '$external', user)
-    return (mech, source, user, passwd)
+    else:
+        if passwd is None:
+            raise ConfigurationError("A password is required.")
+        return (mech, source, user, unicode(passwd))
+
+
+if PY3:
+    def _xor(fir, sec):
+        """XOR two byte strings together (python 3.x)."""
+        return _EMPTY.join([bytes([x ^ y]) for x, y in zip(fir, sec)])
+else:
+    def _xor(fir, sec):
+        """XOR two byte strings together (python 2.x)."""
+        return _EMPTY.join([chr(ord(x) ^ ord(y)) for x, y in zip(fir, sec)])
+
+
+if sys.version_info[:2] >= (3, 2):
+    _from_bytes = int.from_bytes
+    _to_bytes = int.to_bytes
+else:
+    from binascii import (hexlify as _hexlify,
+                          unhexlify as _unhexlify)
+
+    def _from_bytes(value, dummy, int=int, _hexlify=_hexlify):
+        """An implementation of int.from_bytes for python 2.x."""
+        return int(_hexlify(value), 16)
+
+    def _to_bytes(value, dummy0, dummy1, _unhexlify=_unhexlify):
+        """An implementation of int.to_bytes for python 2.x."""
+        return _unhexlify('%040x' % value)
+
+
+_BIGONE = b('\x00\x00\x00\x01')
+
+try:
+    # The fastest option, if it's been compiled to use OpenSSL's HMAC.
+    from backports.pbkdf2 import pbkdf2_hmac
+
+    def _hi(data, salt, iterations):
+        return pbkdf2_hmac('sha1', data, salt, iterations)
+
+except ImportError:
+    try:
+        # Python 2.7.8+, or Python 3.4+.
+        from hashlib import pbkdf2_hmac
+
+        def _hi(data, salt, iterations):
+            return pbkdf2_hmac('sha1', data, salt, iterations)
+
+    except ImportError:
+
+        def _hi(data, salt, iterations):
+            """A simple implementation of PBKDF2."""
+            mac = hmac.HMAC(data, None, _SHA1MOD)
+
+            def _digest(msg, mac=mac):
+                """Get a digest for msg."""
+                _mac = mac.copy()
+                _mac.update(msg)
+                return _mac.digest()
+
+            from_bytes = _from_bytes
+            to_bytes = _to_bytes
+
+            _u1 = _digest(salt + _BIGONE)
+            _ui = from_bytes(_u1, 'big')
+            for _ in range(iterations - 1):
+                _u1 = _digest(_u1)
+                _ui ^= from_bytes(_u1, 'big')
+            return to_bytes(_ui, 20, 'big')
+
+try:
+    from hmac import compare_digest
+except ImportError:
+    if PY3:
+        def _xor_bytes(a, b):
+            return a ^ b
+    else:
+        def _xor_bytes(a, b, _ord=ord):
+            return _ord(a) ^ _ord(b)
+
+    # Python 2.x < 2.7.7 and Python 3.x < 3.3
+    # References:
+    #  - http://bugs.python.org/issue14532
+    #  - http://bugs.python.org/issue14955
+    #  - http://bugs.python.org/issue15061
+    def compare_digest(a, b, _xor_bytes=_xor_bytes):
+        left = None
+        right = b
+        if len(a) == len(b):
+            left = a
+            result = 0
+        if len(a) != len(b):
+            left = b
+            result = 1
+
+        for x, y in zip(left, right):
+            result |= _xor_bytes(x, y)
+        return result == 0
+
+
+_EMPTY = b("")
+_COMMA = b(",")
+_EQUAL = b("=")
+
+def _parse_scram_response(response):
+    """Split a scram response into key, value pairs."""
+    return dict([item.split(_EQUAL, 1) for item in response.split(_COMMA)])
+
+
+def _scram_sha1_conversation(
+        credentials,
+        sock_info,
+        cmd_func,
+        sasl_start,
+        sasl_continue):
+    """Authenticate or copydb using SCRAM-SHA-1.
+
+    sasl_start and sasl_continue are SONs, the base command documents for
+    beginning and continuing the SASL conversation. They may be modified
+    by the callee.
+
+    :Parameters:
+      - `credentials`: A credentials tuple from _build_credentials_tuple.
+      - `sock_info`: A SocketInfo instance.
+      - `cmd_func`: A callback taking args sock_info, database, command doc.
+      - `sasl_start`: A SON.
+      - `sasl_continue`: A SON.
+    """
+    source, username, password = credentials
+
+    # Make local
+    _hmac = hmac.HMAC
+    _sha1 = _SHA1
+    _sha1mod = _SHA1MOD
+
+    user = username.encode("utf-8").replace(
+        _EQUAL, b("=3D")).replace(_COMMA, b("=2C"))
+    nonce = standard_b64encode(
+        (("%s" % (SystemRandom().random(),))[2:]).encode("utf-8"))
+    first_bare = b("n=") + user + b(",r=") + nonce
+
+    sasl_start['payload'] = Binary(b("n,,") + first_bare)
+    res, _ = cmd_func(sock_info, source, sasl_start)
+
+    server_first = res['payload']
+    parsed = _parse_scram_response(server_first)
+    iterations = int(parsed[b('i')])
+    salt = parsed[b('s')]
+    rnonce = parsed[b('r')]
+    if not rnonce.startswith(nonce):
+        raise OperationFailure("Server returned an invalid nonce.")
+
+    without_proof = b("c=biws,r=") + rnonce
+    salted_pass = _hi(_password_digest(username, password).encode("utf-8"),
+                      standard_b64decode(salt),
+                      iterations)
+    client_key = _hmac(salted_pass, b("Client Key"), _sha1mod).digest()
+    stored_key = _sha1(client_key).digest()
+    auth_msg = _COMMA.join((first_bare, server_first, without_proof))
+    client_sig = _hmac(stored_key, auth_msg, _sha1mod).digest()
+    client_proof = b("p=") + standard_b64encode(_xor(client_key, client_sig))
+    client_final = _COMMA.join((without_proof, client_proof))
+
+    server_key = _hmac(salted_pass, b("Server Key"), _sha1mod).digest()
+    server_sig = standard_b64encode(
+        _hmac(server_key, auth_msg, _SHA1MOD).digest())
+
+    cmd = sasl_continue.copy()
+    cmd['conversationId'] = res['conversationId']
+    cmd['payload'] = Binary(client_final)
+    res, _ = cmd_func(sock_info, source, cmd)
+
+    parsed = _parse_scram_response(res['payload'])
+    if not compare_digest(parsed[b('v')], server_sig):
+        raise OperationFailure("Server returned an invalid signature.")
+
+    # Depending on how it's configured, Cyrus SASL (which the server uses)
+    # requires a third empty challenge.
+    if not res['done']:
+        cmd = sasl_continue.copy()
+        cmd['conversationId'] = res['conversationId']
+        cmd['payload'] = Binary(_EMPTY)
+        res, _ = cmd_func(sock_info, source, cmd)
+        if not res['done']:
+            raise OperationFailure('SASL conversation failed to complete.')
+
+
+def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
+    """Authenticate using SCRAM-SHA-1."""
+    # Base commands for starting and continuing SASL authentication.
+    sasl_start = SON([('saslStart', 1),
+                      ('mechanism', 'SCRAM-SHA-1'),
+                      ('autoAuthorize', 1)])
+    sasl_continue = SON([('saslContinue', 1)])
+    _scram_sha1_conversation(credentials, sock_info, cmd_func,
+                             sasl_start, sasl_continue)
+
+
+def _copydb_scram_sha1(
+        credentials,
+        sock_info,
+        cmd_func,
+        fromdb,
+        todb,
+        fromhost):
+    """Copy a database using SCRAM-SHA-1 authentication.
+
+    :Parameters:
+      - `credentials`: A tuple, (mechanism, source, username, password).
+      - `sock_info`: A SocketInfo instance.
+      - `cmd_func`: A callback taking args sock_info, database, command doc.
+      - `fromdb`: Source database.
+      - `todb`: Target database.
+      - `fromhost`: Source host or None.
+    """
+    assert credentials[0] == 'SCRAM-SHA-1'
+
+    sasl_start = SON([('copydbsaslstart', 1),
+                      ('mechanism', 'SCRAM-SHA-1'),
+                      ('autoAuthorize', 1),
+                      ('fromdb', fromdb),
+                      ('fromhost', fromhost)])
+
+    sasl_continue = SON([('copydb', 1),
+                         ('fromdb', fromdb),
+                         ('fromhost', fromhost),
+                         ('todb', todb)])
+
+    _scram_sha1_conversation(credentials[1:],
+                             sock_info,
+                             cmd_func,
+                             sasl_start,
+                             sasl_continue)
 
 
 def _password_digest(username, password):
@@ -75,7 +329,7 @@ def _auth_key(nonce, username, password):
     """
     digest = _password_digest(username, password)
     md5hash = _MD5()
-    data = "%s%s%s" % (nonce, unicode(username), digest)
+    data = "%s%s%s" % (nonce, username, digest)
     md5hash.update(data.encode('utf-8'))
     return unicode(md5hash.hexdigest())
 
@@ -222,12 +476,21 @@ def _authenticate_mongo_cr(credentials, sock_info, cmd_func):
     cmd_func(sock_info, source, query)
 
 
+def _authenticate_default(credentials, sock_info, cmd_func):
+    if sock_info.max_wire_version >= 3:
+        return _authenticate_scram_sha1(credentials, sock_info, cmd_func)
+    else:
+        return _authenticate_mongo_cr(credentials, sock_info, cmd_func)
+
+
 _AUTH_MAP = {
     'CRAM-MD5': _authenticate_cram_md5,
     'GSSAPI': _authenticate_gssapi,
     'MONGODB-CR': _authenticate_mongo_cr,
     'MONGODB-X509': _authenticate_x509,
     'PLAIN': _authenticate_plain,
+    'SCRAM-SHA-1': _authenticate_scram_sha1,
+    'DEFAULT': _authenticate_default,
 }
 
 

pymongo/bulk.py

@@ -1,4 +1,4 @@
-# Copyright 2014-2014 MongoDB, Inc.
+# Copyright 2014-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -82,16 +82,6 @@ def _make_error(index, code, errmsg, operation):
 def _merge_legacy(run, full_result, result, index):
     """Merge a result from a legacy opcode into the full results.
     """
-    # MongoDB 2.6 returns {'ok': 0, 'code': 2, ...} if the j write
-    # concern option is used with --nojournal or w > 1 is used with
-    # a standalone mongod instance. Raise immediately here for
-    # consistency when talking to older servers. Since these are
-    # configuration errors related to write concern the entire batch
-    # will fail.
-    note = result.get("jnote", result.get("wnote"))
-    if note:
-        raise OperationFailure(note, _BAD_VALUE, result)
-
     affected = result.get('n', 0)
 
     errmsg = result.get("errmsg", result.get("err", ""))
@@ -111,13 +101,25 @@ def _merge_legacy(run, full_result, result, index):
 
     if run.op_type == _INSERT:
         full_result['nInserted'] += 1
+
     elif run.op_type == _UPDATE:
         if "upserted" in result:
             doc = {u"index": run.index(index), u"_id": result["upserted"]}
             full_result["upserted"].append(doc)
             full_result['nUpserted'] += affected
+        # Versions of MongoDB before 2.6 don't return the _id for an
+        # upsert if _id is not an ObjectId.
+        elif result.get("updatedExisting") == False and affected == 1:
+            op = run.ops[index]
+            # If _id is in both the update document *and* the query spec
+            # the update document _id takes precedence.
+            _id = op['u'].get('_id', op['q'].get('_id'))
+            doc = {u"index": run.index(index), u"_id": _id}
+            full_result["upserted"].append(doc)
+            full_result['nUpserted'] += affected
         else:
             full_result['nMatched'] += affected
+
     elif run.op_type == _DELETE:
         full_result['nRemoved'] += affected
 

pymongo/command_cursor.py

@@ -1,4 +1,4 @@
-# Copyright 2014 MongoDB, Inc.
+# Copyright 2014-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,15 +32,16 @@ class CommandCursor(object):
         self.__id = cursor_info['id']
         self.__conn_id = conn_id
         self.__data = deque(cursor_info['firstBatch'])
-        self.__decode_opts = (
-            collection.database.connection.document_class,
-            collection.database.connection.tz_aware,
-            collection.uuid_subtype,
-            compile_re
-        )
+        self.__codec_options = collection.codec_options
+        self.__compile_re = compile_re
         self.__retrieved = retrieved
         self.__batch_size = 0
-        self.__killed = False
+        self.__killed = (self.__id == 0)
+
+        if "ns" in cursor_info:
+            self.__ns = cursor_info["ns"]
+        else:
+            self.__ns = collection.full_name
 
     def __del__(self):
         if self.__id and not self.__killed:
@@ -105,9 +106,13 @@ class CommandCursor(object):
             raise
 
         try:
-            response = helpers._unpack_response(response,
-                                                self.__id,
-                                                *self.__decode_opts)
+            response = helpers._unpack_response(
+                response,
+                self.__id,
+                self.__codec_options.document_class,
+                self.__codec_options.tz_aware,
+                self.__codec_options.uuid_representation,
+                self.__compile_re)
         except CursorNotFound:
             self.__killed = True
             raise
@@ -115,13 +120,11 @@ class CommandCursor(object):
             # Don't send kill cursors to another server after a "not master"
             # error. It's completely pointless.
             self.__killed = True
-            client.disconnect()
+            client._disconnect()
             raise
         self.__id = response["cursor_id"]
-
-        assert response["starting_from"] == self.__retrieved, (
-            "Result batch started from %s, expected %s" % (
-                response['starting_from'], self.__retrieved))
+        if self.__id == 0:
+            self.__killed = True
 
         self.__retrieved += response["number_returned"]
         self.__data = deque(response["data"])
@@ -138,7 +141,7 @@ class CommandCursor(object):
 
         if self.__id:  # Get More
             self.__send_message(
-                message.get_more(self.__collection.full_name,
+                message.get_more(self.__ns,
                                  self.__batch_size, self.__id))
 
         else:  # Cursor id is zero nothing else to return
@@ -148,7 +151,19 @@ class CommandCursor(object):
 
     @property
     def alive(self):
-        """Does this cursor have the potential to return more data?"""
+        """Does this cursor have the potential to return more data?
+
+        Even if :attr:`alive` is ``True``, :meth:`next` can raise
+        :exc:`StopIteration`. Best to use a for loop::
+
+            for doc in collection.aggregate(pipeline):
+                print(doc)
+
+        .. note:: :attr:`alive` can be True while iterating a cursor from
+          a failed server. In this case :attr:`alive` will return False after
+          :meth:`next` fails to retrieve the next batch of results from the
+          server.
+        """
         return bool(len(self.__data) or (not self.__killed))
 
     @property
@@ -160,11 +175,10 @@ class CommandCursor(object):
         return self
 
     def next(self):
-        """Advance the cursor.
-        """
+        """Advance the cursor."""
         if len(self.__data) or self._refresh():
             coll = self.__collection
-            return coll.database._fix_incoming(self.__data.popleft(), coll)
+            return coll.database._fix_outgoing(self.__data.popleft(), coll)
         else:
             raise StopIteration
 

pymongo/common.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2014 MongoDB, Inc.
+# Copyright 2011-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -16,13 +16,14 @@
 """Functions and classes common to multiple pymongo modules."""
 import sys
 import warnings
-from pymongo import read_preferences
 
+from bson.binary import (OLD_UUID_SUBTYPE, UUID_SUBTYPE,
+                         JAVA_LEGACY, CSHARP_LEGACY)
+from bson.codec_options import CodecOptions
+from pymongo import read_preferences
 from pymongo.auth import MECHANISMS
 from pymongo.read_preferences import ReadPreference
 from pymongo.errors import ConfigurationError
-from bson.binary import (OLD_UUID_SUBTYPE, UUID_SUBTYPE,
-                         JAVA_LEGACY, CSHARP_LEGACY)
 
 HAS_SSL = True
 try:
@@ -45,7 +46,15 @@ MAX_WRITE_BATCH_SIZE = 1000
 
 # What this version of PyMongo supports.
 MIN_SUPPORTED_WIRE_VERSION = 0
-MAX_SUPPORTED_WIRE_VERSION = 2
+MAX_SUPPORTED_WIRE_VERSION = 3
+
+# mongod/s 2.6 and above return code 59 when a
+# command doesn't exist. mongod versions previous
+# to 2.6 and mongos 2.4.x return no error code
+# when a command does exist. mongos versions previous
+# to 2.4.0 return code 13390 when a command does not
+# exist.
+COMMAND_NOT_FOUND_CODES = (59, 13390, None)
 
 
 def raise_config_error(key, dummy):
@@ -89,18 +98,30 @@ def validate_integer(option, value):
 
 
 def validate_positive_integer(option, value):
-    """Validate that 'value' is a positive integer.
+    """Validate that 'value' is a positive integer, which does not include 0.
+    """
+    val = validate_integer(option, value)
+    if val <= 0:
+        raise ConfigurationError("The value of %s must be "
+                                 "a positive integer" % (option,))
+    return val
+
+
+def validate_non_negative_integer(option, value):
+    """Validate that 'value' is a positive integer or 0.
     """
     val = validate_integer(option, value)
     if val < 0:
         raise ConfigurationError("The value of %s must be "
-                                 "a positive integer" % (option,))
+                                 "a non negative integer" % (option,))
     return val
 
 
 def validate_readable(option, value):
     """Validates that 'value' is file-like and readable.
     """
+    if value is None:
+        return value
     # First make sure its a string py3.3 open(True, 'r') succeeds
     # Used in ssl cert checking due to poor ssl module error reporting
     value = validate_basestring(option, value)
@@ -114,6 +135,8 @@ def validate_cert_reqs(option, value):
     if value is None:
         return value
     if HAS_SSL:
+        if isinstance(value, basestring) and hasattr(ssl, value):
+            value = getattr(ssl, value)
         if value in (ssl.CERT_NONE, ssl.CERT_OPTIONAL, ssl.CERT_REQUIRED):
             return value
         raise ConfigurationError("The value of %s must be one of: "
@@ -125,6 +148,14 @@ def validate_cert_reqs(option, value):
                                  % (option,))
 
 
+def validate_non_negative_integer_or_none(option, value):
+    """Validate that 'value' is a positive integer or 0 or None.
+    """
+    if value is None:
+        return value
+    return validate_non_negative_integer(option, value)
+
+
 def validate_positive_integer_or_none(option, value):
     """Validate that 'value' is a positive integer or None.
     """
@@ -142,6 +173,14 @@ def validate_basestring(option, value):
                     "instance of %s" % (option, basestring.__name__))
 
 
+def validate_basestring_or_none(option, value):
+    """Validates that 'value' is an instance of `basestring` or `None`.
+    """
+    if value is None:
+        return value
+    return validate_basestring(option, value)
+
+
 def validate_int_or_basestring(option, value):
     """Validates that 'value' is an integer or string.
     """
@@ -182,6 +221,15 @@ def validate_timeout_or_none(option, value):
     return validate_positive_float(option, value) / 1000.0
 
 
+def validate_positive_float_or_zero(option, value):
+    """Validates that 'value' is 0 or a positive float or can be converted to
+    0 or a positive float.
+    """
+    if value == 0 or value == "0":
+        return 0
+    return validate_positive_float(option, value)
+
+
 def validate_read_preference(dummy, value):
     """Validate read preference for a ReplicaSetConnection.
     """
@@ -195,24 +243,58 @@ def validate_read_preference(dummy, value):
         raise ConfigurationError("Not a valid read preference")
 
 
-def validate_tag_sets(dummy, value):
+def _validate_tag_sets_format(option, value):
+    if not isinstance(value, list):
+        raise ConfigurationError("%s %r invalid, must be "
+                                 "a list" % (option, value))
+    elif not value:
+        raise ConfigurationError("%s %r invalid, must be None or contain "
+                                 "at least one set of tags" % (option, value))
+
+
+def _validate_dict_list(option, value):
+    for elt in value:
+        if not isinstance(elt, dict):
+            raise ConfigurationError(
+                "%s %r invalid, must be a dict" % (option, elt))
+
+
+def validate_read_preference_tags(option, value):
+    """Parse readPreferenceTags if passed as a client kwarg.
+    """
+    if value is None:
+        return [{}]
+
+    if isinstance(value, basestring):
+        value = [value]
+    else:
+        _validate_tag_sets_format(option, value)
+        if isinstance(value[0], dict):
+            _validate_dict_list("Tag set", value)
+            return value
+
+    tag_sets = []
+    for tag_set in value:
+        if tag_set == '':
+            tag_sets.append({})
+            continue
+        try:
+            tag_sets.append(dict([tag.split(":")
+                                  for tag in tag_set.split(",")]))
+        except Exception:
+            raise ValueError("%r not a valid "
+                             "value for %s" % (tag_set, option))
+    return tag_sets
+
+
+def validate_tag_sets(option, value):
     """Validate tag sets for a ReplicaSetConnection.
     """
     if value is None:
         return [{}]
 
-    if not isinstance(value, list):
-        raise ConfigurationError((
-            "Tag sets %s invalid, must be a list") % repr(value))
-    if len(value) == 0:
-        raise ConfigurationError((
-            "Tag sets %s invalid, must be None or contain at least one set of"
-            " tags") % repr(value))
-
-    for tags in value:
-        if not isinstance(tags, dict):
-            raise ConfigurationError(
-                "Tag set %s invalid, must be a dict" % repr(tags))
+    _validate_tag_sets_format(option, value)
+    _validate_dict_list("Tag set", value)
 
     return value
 
@@ -247,11 +329,62 @@ def validate_uuid_subtype(dummy, value):
     return value
 
 
+_MECHANISM_PROPS = frozenset(['SERVICE_NAME'])
+
+
+def validate_auth_mechanism_properties(option, value):
+    """Validate authMechanismProperties."""
+    value = validate_basestring(option, value)
+    props = {}
+    for opt in value.split(','):
+        try:
+            key, val = opt.split(':')
+            if key not in _MECHANISM_PROPS:
+                raise ConfigurationError("%s is not a supported auth "
+                                         "mechanism property. Must be one of "
+                                         "%s." % (key, tuple(_MECHANISM_PROPS)))
+            props[key] = val
+        except ValueError:
+            raise ConfigurationError("auth mechanism properties must be "
+                                     "key:value pairs like SERVICE_NAME:"
+                                     "mongodb, not %s." % (opt,))
+    return props
+
+
+def validate_is_dict(option, value):
+    """Validate the type of method arguments that expect a document."""
+    if not isinstance(value, dict):
+        raise TypeError("%s must be an instance of dict, bson.son.SON, or"
+                        "another subclass of dict" % (option,))
+
+
+def validate_ok_for_replace(replacement):
+    """Validate a replacement document."""
+    validate_is_dict("replacement", replacement)
+    # Replacement can be {}
+    if replacement:
+        first = iter(replacement).next()
+        if first.startswith('$'):
+            raise ValueError('replacement can not include $ operators')
+
+
+def validate_ok_for_update(update):
+    """Validate an update document."""
+    validate_is_dict("update", update)
+    # Update can not be {}
+    if not update:
+        raise ValueError('update only works with $ operators')
+    first = iter(update).next()
+    if not first.startswith('$'):
+        raise ValueError('update only works with $ operators')
+
+
+
 # jounal is an alias for j,
 # wtimeoutms is an alias for wtimeout,
 # readpreferencetags is an alias for tag_sets.
 VALIDATORS = {
-    'replicaset': validate_basestring,
+    'replicaset': validate_basestring_or_none,
     'slaveok': validate_boolean,
     'slave_okay': validate_boolean,
     'safe': validate_boolean,
@@ -264,28 +397,35 @@ VALIDATORS = {
     'connecttimeoutms': validate_timeout_or_none,
     'sockettimeoutms': validate_timeout_or_none,
     'waitqueuetimeoutms': validate_timeout_or_none,
-    'waitqueuemultiple': validate_positive_integer_or_none,
+    'waitqueuemultiple': validate_non_negative_integer_or_none,
     'ssl': validate_boolean,
     'ssl_keyfile': validate_readable,
     'ssl_certfile': validate_readable,
     'ssl_cert_reqs': validate_cert_reqs,
     'ssl_ca_certs': validate_readable,
+    'ssl_match_hostname': validate_boolean,
     'readpreference': validate_read_preference,
     'read_preference': validate_read_preference,
-    'readpreferencetags': validate_tag_sets,
+    'readpreferencetags': validate_read_preference_tags,
     'tag_sets': validate_tag_sets,
-    'secondaryacceptablelatencyms': validate_positive_float,
-    'secondary_acceptable_latency_ms': validate_positive_float,
+    'secondaryacceptablelatencyms': validate_positive_float_or_zero,
+    'secondary_acceptable_latency_ms': validate_positive_float_or_zero,
+    'localthresholdms': validate_positive_float_or_zero,
     'auto_start_request': validate_boolean,
     'use_greenlets': validate_boolean,
     'authmechanism': validate_auth_mechanism,
     'authsource': validate_basestring,
     'gssapiservicename': validate_basestring,
+    'authmechanismproperties': validate_auth_mechanism_properties,
     'uuidrepresentation': validate_uuid_representation,
+    'socketkeepalive': validate_boolean,
+    'maxpoolsize': validate_positive_integer_or_none,
+    'connect': validate_boolean,
+    '_connect': validate_boolean
 }
 
 
-_AUTH_OPTIONS = frozenset(['gssapiservicename'])
+_AUTH_OPTIONS = frozenset(['gssapiservicename', 'authmechanismproperties'])
 
 
 def validate_auth_option(option, value):
@@ -342,16 +482,19 @@ class BaseObject(object):
 
     def __init__(self, **options):
 
+        self._codec_options = options.get('codec_options')
+        if not isinstance(self._codec_options, CodecOptions):
+            raise TypeError("codec_options must be an instance of "
+                            "bson.codec_options.CodecOptions")
+        self._read_pref = ReadPreference.PRIMARY
+        self._tag_sets = [{}]
+        self._secondary_acceptable_latency_ms = 15
+        self._write_concern = WriteConcern()
         self.__slave_okay = False
-        self.__read_pref = ReadPreference.PRIMARY
-        self.__tag_sets = [{}]
-        self.__secondary_acceptable_latency_ms = 15
         self.__safe = None
-        self.__uuid_subtype = OLD_UUID_SUBTYPE
-        self.__write_concern = WriteConcern()
         self.__set_options(options)
-        if (self.__read_pref == ReadPreference.PRIMARY
-                and self.__tag_sets != [{}]):
+        if (self._read_pref == ReadPreference.PRIMARY
+                and self._tag_sets != [{}]):
             raise ConfigurationError(
                 "ReadPreference PRIMARY cannot be combined with tags")
 
@@ -376,9 +519,9 @@ class BaseObject(object):
         object (Connection, Database, Collection, etc.)
         """
         if value is None:
-            self.__write_concern.pop(option, None)
+            self._write_concern.pop(option, None)
         else:
-            self.__write_concern[option] = value
+            self._write_concern[option] = value
             if option != "w" or value != 0:
                 self.__safe = True
 
@@ -388,17 +531,13 @@ class BaseObject(object):
             if option in ('slave_okay', 'slaveok'):
                 self.__slave_okay = validate_boolean(option, value)
             elif option in ('read_preference', "readpreference"):
-                self.__read_pref = validate_read_preference(option, value)
+                self._read_pref = validate_read_preference(option, value)
             elif option in ('tag_sets', 'readpreferencetags'):
-                self.__tag_sets = validate_tag_sets(option, value)
-            elif option == 'uuidrepresentation':
-                self.__uuid_subtype = validate_uuid_subtype(option, value)
-            elif option in (
-                'secondaryacceptablelatencyms',
-                'secondary_acceptable_latency_ms'
-            ):
-                self.__secondary_acceptable_latency_ms = \
-                    validate_positive_float(option, value)
+                self._tag_sets = validate_tag_sets(option, value)
+            elif option in ('secondaryacceptablelatencyms',
+                            'secondary_acceptable_latency_ms'):
+                self._secondary_acceptable_latency_ms = (
+                    validate_positive_float_or_zero(option, value))
             elif option in SAFE_OPTIONS:
                 if option == 'journal':
                     self.__set_safe_option('j', value)
@@ -407,8 +546,26 @@ class BaseObject(object):
                 else:
                     self.__set_safe_option(option, value)
 
+    @property
+    def codec_options(self):
+        """Read only access to the :class:`~bson.codec_options.CodecOptions`
+        of this instance.
+
+        The value of :attr:`codec_options` can be changed through
+        :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+        :meth:`~pymongo.database.Database.get_collection`,
+        or :meth:`~pymongo.collection.Collection.with_options`,
+
+        .. versionadded:: 2.9
+        """
+        return self._codec_options
+
     def __set_write_concern(self, value):
         """Property setter for write_concern."""
+        warnings.warn("Changing write_concern by setting it directly is "
+                      "deprecated in this version of PyMongo and prohibited "
+                      "in PyMongo 3. See the write_concern docstring for more "
+                      "information.", DeprecationWarning, stacklevel=2)
         if not isinstance(value, dict):
             raise ConfigurationError("write_concern must be an "
                                      "instance of dict or a subclass.")
@@ -418,13 +575,12 @@ class BaseObject(object):
         for k, v in value.iteritems():
             # Make sure we validate each option.
             wc[k] = v
-        self.__write_concern = wc
+        self._write_concern = wc
 
     def __get_write_concern(self):
         """The default write concern for this instance.
 
-        Supports dict style access for getting/setting write concern
-        options. Valid options include:
+        Valid options include:
 
         - `w`: (integer or string) If this is a replica set, write operations
           will block until they have been replicated to the specified number
@@ -448,23 +604,6 @@ class BaseObject(object):
           option, blocking until write operations have been committed to the
           journal. Cannot be used in combination with `j`.
 
-        >>> m = pymongo.MongoClient()
-        >>> m.write_concern
-        {}
-        >>> m.write_concern = {'w': 2, 'wtimeout': 1000}
-        >>> m.write_concern
-        {'wtimeout': 1000, 'w': 2}
-        >>> m.write_concern['j'] = True
-        >>> m.write_concern
-        {'wtimeout': 1000, 'j': True, 'w': 2}
-        >>> m.write_concern = {'j': True}
-        >>> m.write_concern
-        {'j': True}
-        >>> # Disable write acknowledgement and write concern
-        ...
-        >>> m.write_concern['w'] = 0
-
-
         .. note:: Accessing :attr:`write_concern` returns its value
            (a subclass of :class:`dict`), not a copy.
 
@@ -475,15 +614,34 @@ class BaseObject(object):
            :meth:`set_lasterror_options`, setting an option in
            :attr:`write_concern` does not implicitly set :attr:`safe`
            to ``True``.
+
+        .. warning:: :attr:`write_concern` is read only in PyMongo 3. Use
+          :class:`~pymongo.write_concern.WriteConcern` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` to set write
+          concern.
+          See the :doc:`/migrate-to-pymongo3` for examples.
+
+        .. versionchanged:: 2.9
+          Deprecated directly setting write_concern.
         """
         # To support dict style access we have to return the actual
         # WriteConcern here, not a copy.
-        return self.__write_concern
+        return self._write_concern
 
     write_concern = property(__get_write_concern, __set_write_concern)
 
     def __get_slave_okay(self):
-        """DEPRECATED. Use :attr:`read_preference` instead.
+        """**DEPRECATED** Use read preference "secondaryPreferred" instead.
+
+        .. warning:: :attr:`slave_okay` is deprecated in this version of
+          PyMongo and removed in PyMongo 3. Use read preference
+          :class:`~pymongo.read_preferences.SecondaryPreferred` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` instead.
+          See the :doc:`/migrate-to-pymongo3` for examples.
 
         .. versionchanged:: 2.1
            Deprecated slave_okay.
@@ -493,9 +651,10 @@ class BaseObject(object):
 
     def __set_slave_okay(self, value):
         """Property setter for slave_okay"""
-        warnings.warn("slave_okay is deprecated. Please use "
-                      "read_preference instead.", DeprecationWarning,
-                      stacklevel=2)
+        warnings.warn("slave_okay is deprecated in this version of PyMongo "
+                      "and removed in PyMongo 3. See the slave_okay docstring "
+                      "for more information.",
+                      DeprecationWarning, stacklevel=2)
         self.__slave_okay = validate_boolean('slave_okay', value)
 
     slave_okay = property(__get_slave_okay, __set_slave_okay)
@@ -506,114 +665,185 @@ class BaseObject(object):
         See :class:`~pymongo.read_preferences.ReadPreference` for
         available options.
 
+        .. warning:: :attr:`read_preference` is read only in PyMongo 3. Use the
+          read preference classes from :mod:`~pymongo.read_preferences` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` to set read
+          preference.
+          See the :doc:`/migrate-to-pymongo3` for examples.
+
+        .. versionchanged:: 2.9
+          Deprecated directly setting read_preference.
         .. versionadded:: 2.1
         """
-        return self.__read_pref
+        return self._read_pref
 
     def __set_read_pref(self, value):
         """Property setter for read_preference"""
-        self.__read_pref = validate_read_preference('read_preference', value)
+        warnings.warn("Changing read_preference by setting it directly is "
+                      "deprecated in this version of PyMongo and prohibited "
+                      "in PyMongo 3. See the read_preference docstring for "
+                      "more information.", DeprecationWarning, stacklevel=2)
+        self._read_pref = validate_read_preference('read_preference', value)
 
     read_preference = property(__get_read_pref, __set_read_pref)
 
     def __get_acceptable_latency(self):
-        """Any replica-set member whose ping time is within
-        secondary_acceptable_latency_ms of the nearest member may accept
-        reads. Defaults to 15 milliseconds.
+        """**DEPRECATED** Any replica set member whose ping time is within
+        :attr:`secondary_acceptable_latency_ms` of the nearest member may
+        accept reads. Defaults to 15 milliseconds.
 
         See :class:`~pymongo.read_preferences.ReadPreference`.
 
-        .. versionadded:: 2.3
-
-        .. note:: ``secondary_acceptable_latency_ms`` is ignored when talking
+        .. note:: :attr:`secondary_acceptable_latency_ms` is ignored when talking
           to a replica set *through* a mongos. The equivalent is the
           localThreshold_ command line option.
 
-        .. _localThreshold: http://docs.mongodb.org/manual/reference/mongos/#cmdoption-mongos--localThreshold
+        .. warning:: :attr:`secondary_acceptable_latency_ms` is deprecated in
+          this version of PyMongo and removed in PyMongo 3. Use the
+          `localThresholdMS` option with
+          :class:`~pymongo.mongo_client.MongoClient` or
+          :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`
+          instead. See the :doc:`/migrate-to-pymongo3` for more information.
+
+        .. versionchanged:: 2.9
+          Deprecated secondary_acceptable_latency_ms.
+        .. versionadded:: 2.3
+
+        .. _localThreshold:
+          http://docs.mongodb.org/manual/reference/program/mongos/#cmdoption--localThreshold
         """
-        return self.__secondary_acceptable_latency_ms
+        return self._secondary_acceptable_latency_ms
 
     def __set_acceptable_latency(self, value):
         """Property setter for secondary_acceptable_latency_ms"""
-        self.__secondary_acceptable_latency_ms = (validate_positive_float(
-            'secondary_acceptable_latency_ms', value))
+        warnings.warn("secondary_acceptable_latency_ms is deprecated in this "
+                      "version of PyMongo and removed in PyMongo 3. See the "
+                      "PyMongo 3 migration guide for more information.",
+                      DeprecationWarning, stacklevel=3)
+        self._secondary_acceptable_latency_ms = (
+            validate_positive_float_or_zero(
+                'secondary_acceptable_latency_ms', value))
 
     secondary_acceptable_latency_ms = property(
         __get_acceptable_latency, __set_acceptable_latency)
 
     def __get_tag_sets(self):
-        """Set ``tag_sets`` to a list of dictionaries like [{'dc': 'ny'}] to
-        read only from members whose ``dc`` tag has the value ``"ny"``.
-        To specify a priority-order for tag sets, provide a list of
+        """**DEPRECATED** Set ``tag_sets`` to a list of dictionaries like
+        [{'dc': 'ny'}] to read only from members whose ``dc`` tag has the value
+        ``"ny"``. To specify a priority-order for tag sets, provide a list of
         tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
         set, ``{}``, means "read from any member that matches the mode,
         ignoring tags." ReplicaSetConnection tries each set of tags in turn
         until it finds a set of tags with at least one matching member.
 
-           .. seealso:: `Data-Center Awareness
-               <http://www.mongodb.org/display/DOCS/Data+Center+Awareness>`_
+        .. seealso:: `Data-Center Awareness
+          <http://www.mongodb.org/display/DOCS/Data+Center+Awareness>`_
 
+        .. warning:: :attr:`tag_sets` is deprecated in this version of PyMongo
+          and removed in PyMongo 3. Use the read preference classes from
+          :mod:`~pymongo.read_preferences` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` instead.
+          See the :doc:`/migrate-to-pymongo3` for examples.
+
+        .. versionchanged:: 2.9
+          Deprecated tag_sets.
         .. versionadded:: 2.3
         """
-        return self.__tag_sets
+        return self._tag_sets
 
     def __set_tag_sets(self, value):
         """Property setter for tag_sets"""
-        self.__tag_sets = validate_tag_sets('tag_sets', value)
+        warnings.warn("tag_sets is deprecated in this version of PyMongo and "
+                      "removed in PyMongo 3. See the tag_sets docstring for "
+                      "more information.", DeprecationWarning, stacklevel=2)
+        self._tag_sets = validate_tag_sets('tag_sets', value)
 
     tag_sets = property(__get_tag_sets, __set_tag_sets)
 
     def __get_uuid_subtype(self):
-        """This attribute specifies which BSON Binary subtype is used when
-        storing UUIDs. Historically UUIDs have been stored as BSON Binary
+        """**DEPRECATED** This attribute specifies which BSON Binary subtype is
+        used when storing UUIDs. Historically UUIDs have been stored as BSON Binary
         subtype 3. This attribute is used to switch to the newer BSON Binary
         subtype 4. It can also be used to force legacy byte order and subtype
         compatibility with the Java and C# drivers. See the :mod:`bson.binary`
-        module for all options."""
-        return self.__uuid_subtype
+        module for all options.
+
+        .. warning:: :attr:`uuid_subtype` is deprecated in this version of
+          PyMongo and removed in PyMongo 3. Use
+          :class:`~bson.codec_options.CodecOptions` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` instead.
+          See the :doc:`/migrate-to-pymongo3` for examples.
+
+        .. versionchanged:: 2.9
+          Deprecated uuid_subtype.
+        """
+        return self._codec_options.uuid_representation
 
     def __set_uuid_subtype(self, value):
         """Sets the BSON Binary subtype to be used when storing UUIDs."""
-        self.__uuid_subtype = validate_uuid_subtype("uuid_subtype", value)
+        warnings.warn("uuid_subtype is deprecated in this version of PyMongo "
+                      "and removed in PyMongo 3. See the uuid_subtype "
+                      "docstring for more information.",
+                      DeprecationWarning, stacklevel=2)
+        as_class = self._codec_options.document_class
+        tz_aware = self._codec_options.tz_aware
+        uuid_rep = validate_uuid_subtype("uuid_subtype", value)
+        self._codec_options = CodecOptions(as_class, tz_aware, uuid_rep)
 
     uuid_subtype = property(__get_uuid_subtype, __set_uuid_subtype)
 
     def __get_safe(self):
-        """**DEPRECATED:** Use the 'w' :attr:`write_concern` option instead.
+        """**DEPRECATED** Use getlasterror with every write operation?
 
-        Use getlasterror with every write operation?
+        .. warning:: :attr:`safe` is deprecated in this version of PyMongo
+          and removed in PyMongo 3. Use
+          :class:`~pymongo.write_concern.WriteConcern` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` instead.
+          See the :doc:`/migrate-to-pymongo3` for examples.
 
+        .. versionchanged:: 2.4
+          Deprecated safe.
         .. versionadded:: 2.0
         """
         return self.__safe
 
     def __set_safe(self, value):
         """Property setter for safe"""
-        warnings.warn("safe is deprecated. Please use the"
-                      " 'w' write_concern option instead.",
-                      DeprecationWarning, stacklevel=2)
+        warnings.warn("safe is deprecated in this version of PyMongo and "
+                      "removed in PyMongo 3. See the safe docstring for more "
+                      "information.", DeprecationWarning, stacklevel=2)
         self.__safe = validate_boolean('safe', value)
 
     safe = property(__get_safe, __set_safe)
 
     def get_lasterror_options(self):
-        """DEPRECATED: Use :attr:`write_concern` instead.
+        """**DEPRECATED** Returns a dict of the getlasterror options set on this
+        instance.
 
-        Returns a dict of the getlasterror options set on this instance.
+        .. warning:: :meth:`get_lasterror_options` is deprecated in this
+          version of PyMongo and removed in PyMongo 3. Use
+          :attr:`write_concern` instead.
 
         .. versionchanged:: 2.4
            Deprecated get_lasterror_options.
         .. versionadded:: 2.0
         """
-        warnings.warn("get_lasterror_options is deprecated. Please use "
-                      "write_concern instead.", DeprecationWarning,
-                      stacklevel=2)
-        return self.__write_concern.copy()
+        warnings.warn("get_lasterror_options is deprecated in this version of "
+                      "PyMongo and removed in PyMongo 3. See the "
+                      "get_lasterror_options docstring for more information.",
+                      DeprecationWarning, stacklevel=2)
+        return self._write_concern.copy()
 
     def set_lasterror_options(self, **kwargs):
-        """DEPRECATED: Use :attr:`write_concern` instead.
-
-        Set getlasterror options for this instance.
+        """**DEPRECATED** Set getlasterror options for this instance.
 
         Valid options include j=<bool>, w=<int/string>, wtimeout=<int>,
         and fsync=<bool>. Implies safe=True.
@@ -622,20 +852,27 @@ class BaseObject(object):
             - `**kwargs`: Options should be passed as keyword
                           arguments (e.g. w=2, fsync=True)
 
+        .. warning:: :meth:`set_lasterror_options` is deprecated in this
+          version of PyMongo and removed in PyMongo 3. Use
+          :class:`~pymongo.write_concern.WriteConcern` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` instead.
+          See the :doc:`/migrate-to-pymongo3` for examples.
+
         .. versionchanged:: 2.4
            Deprecated set_lasterror_options.
         .. versionadded:: 2.0
         """
-        warnings.warn("set_lasterror_options is deprecated. Please use "
-                      "write_concern instead.", DeprecationWarning,
-                      stacklevel=2)
+        warnings.warn("set_lasterror_options is deprecated in this version of "
+                      "PyMongo and removed in PyMongo 3. See the "
+                      "set_lasterror_options docstring for more information.",
+                      DeprecationWarning, stacklevel=2)
         for key, value in kwargs.iteritems():
             self.__set_safe_option(key, value)
 
     def unset_lasterror_options(self, *options):
-        """DEPRECATED: Use :attr:`write_concern` instead.
-
-        Unset getlasterror options for this instance.
+        """**DEPRECATED** Unset getlasterror options for this instance.
 
         If no options are passed unsets all getlasterror options.
         This does not set `safe` to False.
@@ -643,18 +880,27 @@ class BaseObject(object):
         :Parameters:
             - `*options`: The list of options to unset.
 
+        .. warning:: :meth:`unset_lasterror_options` is deprecated in this
+          version of PyMongo and removed in PyMongo 3. Use
+          :class:`~pymongo.write_concern.WriteConcern` with
+          :meth:`~pymongo.mongo_client.MongoClient.get_database`,
+          :meth:`~pymongo.database.Database.get_collection`,
+          or :meth:`~pymongo.collection.Collection.with_options` instead.
+          See the :doc:`/migrate-to-pymongo3` for examples.
+
         .. versionchanged:: 2.4
            Deprecated unset_lasterror_options.
         .. versionadded:: 2.0
         """
-        warnings.warn("unset_lasterror_options is deprecated. Please use "
-                      "write_concern instead.", DeprecationWarning,
-                      stacklevel=2)
+        warnings.warn("unset_lasterror_options is deprecated in this version "
+                      "of PyMongo and removed in PyMongo 3. See the "
+                      "unset_lasterror_options docstring for more information.",
+                      DeprecationWarning, stacklevel=2)
         if len(options):
             for option in options:
-                self.__write_concern.pop(option, None)
+                self._write_concern.pop(option, None)
         else:
-            self.__write_concern = WriteConcern()
+            self._write_concern = WriteConcern()
 
     def _get_wc_override(self):
         """Get write concern override.
@@ -663,7 +909,7 @@ class BaseObject(object):
         We don't want to override user write concern options if write concern
         is already enabled.
         """
-        if self.safe and self.__write_concern.get('w') != 0:
+        if self.safe and self._write_concern.get('w') != 0:
             return {}
         return {'w': 1}
 
@@ -680,12 +926,6 @@ class BaseObject(object):
 
         .. versionadded:: 2.3
         """
-        # Don't ever send w=1 to the server.
-        def pop1(dct):
-            if dct.get('w') == 1:
-                dct.pop('w')
-            return dct
-
         if safe is not None:
             warnings.warn("The safe parameter is deprecated. Please use "
                           "write concern options instead.", DeprecationWarning,
@@ -696,7 +936,7 @@ class BaseObject(object):
         if safe is not None or options:
             if safe or options:
                 if not options:
-                    options = self.__write_concern.copy()
+                    options = self._write_concern.copy()
                     # Backwards compatability edge case. Call getLastError
                     # with no options if safe=True was passed but collection
                     # level defaults have been disabled with w=0.
@@ -706,14 +946,14 @@ class BaseObject(object):
                     if options.get('w') == 0:
                         return True, {}
                 # Passing w=0 overrides passing safe=True.
-                return options.get('w') != 0, pop1(options)
+                return options.get('w') != 0, options
             return False, {}
 
         # Fall back to collection level defaults.
         # w=0 takes precedence over self.safe = True
-        if self.__write_concern.get('w') == 0:
+        if self._write_concern.get('w') == 0:
             return False, {}
-        elif self.safe or self.__write_concern.get('w', 0) != 0:
-            return True, pop1(self.__write_concern.copy())
+        elif self.safe or self._write_concern.get('w', 0) != 0:
+            return True, self._write_concern.copy()
 
         return False, {}

pymongo/connection.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -46,7 +46,7 @@ class Connection(MongoClient):
 
     def __init__(self, host=None, port=None, max_pool_size=None,
                  network_timeout=None, document_class=dict,
-                 tz_aware=False, _connect=True, **kwargs):
+                 tz_aware=False, **kwargs):
         """Create a new connection to a single MongoDB instance at *host:port*.
 
         .. warning::
@@ -99,26 +99,24 @@ class Connection(MongoClient):
 
           | **Other optional parameters can be passed as keyword arguments:**
 
-          - `socketTimeoutMS`: (integer) How long (in milliseconds) a send or
-            receive on a socket can take before timing out. Defaults to ``None``
-            (no timeout).
-          - `connectTimeoutMS`: (integer) How long (in milliseconds) a
+          - `socketTimeoutMS`: (integer or None) How long (in milliseconds) a
+            send or receive on a socket can take before timing out. Defaults to
+            ``None`` (no timeout).
+          - `connectTimeoutMS`: (integer or None) How long (in milliseconds) a
             connection can take to be opened before timing out. Defaults to
             ``20000``.
-          - `waitQueueTimeoutMS`: (integer) How long (in milliseconds) a
-            thread will wait for a socket from the pool if the pool has no
+          - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds)
+            a thread will wait for a socket from the pool if the pool has no
             free sockets. Defaults to ``None`` (no timeout).
-          - `waitQueueMultiple`: (integer) Multiplied by max_pool_size to give
-            the number of threads allowed to wait for a socket at one time.
-            Defaults to ``None`` (no waiters).
-
+          - `waitQueueMultiple`: (integer or None) Multiplied by max_pool_size
+            to give the number of threads allowed to wait for a socket at one
+            time. Defaults to ``None`` (no waiters).
+          - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
           - `auto_start_request`: If ``True`` (the default), each thread that
             accesses this Connection has a socket allocated to it for the
-            thread's lifetime.  This ensures consistent reads, even if you read
-            after an unsafe write.
-          - `use_greenlets`: if ``True``, :meth:`start_request()` will ensure
-            that the current greenlet uses the same socket for all operations
-            until :meth:`end_request()`
+            thread's lifetime, or until :meth:`end_request` is called.
 
           | **Write Concern options:**
 
@@ -155,27 +153,32 @@ class Connection(MongoClient):
             The driver will verify that the replica-set it connects to matches
             this name. Implies that the hosts specified are a seed list and the
             driver should attempt to find all members of the set. *Ignored by
-            mongos*.
+            mongos*. Defaults to ``None``.
           - `read_preference`: The read preference for this client. If
             connecting to a secondary then a read preference mode *other* than
-            PRIMARY is required - otherwise all queries will throw a
+            primary is required - otherwise all queries will throw a
             :class:`~pymongo.errors.AutoReconnect` "not master" error.
-            See :class:`~pymongo.read_preferences.ReadPreference` for all
-            available read preference options.
-          - `tag_sets`: Ignored unless connecting to a replica-set via mongos.
-            Specify a priority-order for tag sets, provide a list of
-            tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
-            set, ``{}``, means "read from any member that matches the mode,
-            ignoring tags.
+            See :mod:`~pymongo.read_preferences` for all options.
+            Defaults to ``ReadPreference.PRIMARY``.
+          - `localThresholdMS`: (integer) Used with mongos high availability.
+            Any known mongos whose ping time is within localThresholdMS of the
+            nearest member may be chosen during a failover. Default 15
+            milliseconds. Ignored **by** mongos and must be configured on the
+            command line. See the localThreshold_ option for more information.
 
           | **SSL configuration:**
 
+            See :doc:`/examples/tls` for examples.
+
           - `ssl`: If ``True``, create the connection to the server using SSL.
+            Defaults to ``False``.
           - `ssl_keyfile`: The private keyfile used to identify the local
             connection against mongod.  If included with the ``certfile` then
             only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
+            Defaults to ``None``.
           - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``ssl=True``.
+            connection against mongod. Implies ``ssl=True``. Defaults to
+            ``None``.
           - `ssl_cert_reqs`: The parameter cert_reqs specifies whether a
             certificate is required from the other side of the connection,
             and whether it will be validated if provided. It must be one of the
@@ -184,11 +187,11 @@ class Connection(MongoClient):
             ``ssl.CERT_REQUIRED`` (required and validated). If the value of
             this parameter is not ``ssl.CERT_NONE``, then the ``ssl_ca_certs``
             parameter must point to a file of CA certificates.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``ssl.CERT_NONE``.
           - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``None``.
 
         .. seealso:: :meth:`end_request`
         .. versionchanged:: 2.5
@@ -232,8 +235,8 @@ class Connection(MongoClient):
         kwargs['auto_start_request'] = kwargs.get('auto_start_request', True)
         kwargs['safe'] = kwargs.get('safe', False)
 
-        super(Connection, self).__init__(host, port,
-                max_pool_size, document_class, tz_aware, _connect, **kwargs)
+        super(Connection, self).__init__(
+            host, port, max_pool_size, document_class, tz_aware, **kwargs)
 
     def __repr__(self):
         if len(self.nodes) == 1:

pymongo/cursor.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,12 +18,13 @@ from collections import deque
 
 from bson import RE_TYPE
 from bson.code import Code
+from bson.codec_options import CodecOptions as _CodecOptions
 from bson.son import SON
 from pymongo import helpers, message, read_preferences
 from pymongo.read_preferences import ReadPreference, secondary_ok_commands
 from pymongo.errors import (AutoReconnect,
-                            CursorNotFound,
-                            InvalidOperation)
+                            InvalidOperation,
+                            OperationFailure)
 
 _QUERY_OPTIONS = {
     "tailable_cursor": 2,
@@ -34,6 +35,30 @@ _QUERY_OPTIONS = {
     "exhaust": 64,
     "partial": 128}
 
+class CursorType(object):
+    NON_TAILABLE = 0
+    """The standard cursor type."""
+
+    TAILABLE = _QUERY_OPTIONS["tailable_cursor"]
+    """The tailable cursor type.
+    Tailable cursors are only for use with capped collections. They are not
+    closed when the last data is retrieved but are kept open and the cursor
+    location marks the final document position. If more data is received
+    iteration of the cursor will continue from the last document received.
+    """
+
+    TAILABLE_AWAIT = TAILABLE | _QUERY_OPTIONS["await_data"]
+    """A tailable cursor with the await option set.
+    Creates a tailable cursor that will wait for a few seconds after returning
+    the full result set so that it can capture and return additional data added
+    during the query.
+    """
+
+    EXHAUST = _QUERY_OPTIONS["exhaust"]
+    """An exhaust cursor.
+    MongoDB will stream batched results to the client without waiting for the
+    client to request each batch, reducing latency.
+    """
 
 # This has to be an old style class due to
 # http://bugs.jython.org/issue1057
@@ -56,6 +81,15 @@ class _SocketManager:
             self.pool.maybe_return_socket(self.sock)
             self.sock, self.pool = None, None
 
+    def error(self):
+        """Clean up after an error on the managed socket.
+        """
+        if self.sock:
+            self.sock.close()
+
+        # Return the closed socket to avoid a semaphore leak in the pool.
+        self.close()
+
 
 # TODO might be cool to be able to do find().include("foo") or
 # find().exclude(["bar", "baz"]) or find().slice("a", 1, 2) as an
@@ -70,8 +104,9 @@ class Cursor(object):
                  await_data=False, partial=False, manipulate=True,
                  read_preference=ReadPreference.PRIMARY,
                  tag_sets=[{}], secondary_acceptable_latency_ms=None,
-                 exhaust=False, compile_re=True, _must_use_master=False,
-                 _uuid_subtype=None, **kwargs):
+                 exhaust=False, compile_re=True, oplog_replay=False,
+                 modifiers=None, _must_use_master=False, _codec_options=None,
+                 **kwargs):
         """Create a new cursor.
 
         Should not be called directly by application developers - see
@@ -79,6 +114,37 @@ class Cursor(object):
 
         .. mongodoc:: cursors
         """
+
+        # Backport aliases.
+        if 'filter' in kwargs:
+            spec = kwargs['filter']
+        if 'projection' in kwargs:
+            fields = kwargs['projection']
+        if 'no_cursor_timeout' in kwargs:
+            timeout = not kwargs['no_cursor_timeout']
+        if 'allow_partial_results' in kwargs:
+            partial = kwargs['allow_partial_results']
+
+        if 'cursor_type' in kwargs:
+            crt = kwargs['cursor_type']
+            if crt not in (CursorType.NON_TAILABLE, CursorType.TAILABLE,
+                          CursorType.TAILABLE_AWAIT, CursorType.EXHAUST):
+                raise ValueError("not a valid value for cursor_type")
+            exhaust = crt == CursorType.EXHAUST
+            tailable = crt == CursorType.TAILABLE
+            if crt == CursorType.TAILABLE_AWAIT:
+                await_data = True
+                tailable = True
+
+        if modifiers is not None:
+            if not isinstance(modifiers, dict):
+                raise TypeError("%s must be an instance of dict or subclass"
+                                % (modifiers,))
+            if '$snapshot' in modifiers:
+                snapshot = modifiers['$snapshot']
+            if '$maxScan' in modifiers:
+                max_scan = modifiers['$maxScan']
+
         self.__id = None
 
         if spec is None:
@@ -104,6 +170,8 @@ class Cursor(object):
             raise TypeError("partial must be an instance of bool")
         if not isinstance(exhaust, bool):
             raise TypeError("exhaust must be an instance of bool")
+        if not isinstance(oplog_replay, bool):
+            raise TypeError("oplog_replay must be an instance of bool")
 
         if fields is not None:
             if not fields:
@@ -111,9 +179,6 @@ class Cursor(object):
             if not isinstance(fields, dict):
                 fields = helpers._fields_list_to_dict(fields)
 
-        if as_class is None:
-            as_class = collection.database.connection.document_class
-
         self.__collection = collection
         self.__spec = spec
         self.__fields = fields
@@ -123,6 +188,8 @@ class Cursor(object):
         self.__batch_size = 0
         self.__max = None
         self.__min = None
+        self.__modifiers = modifiers and modifiers.copy() or {}
+
 
         # Exhaust cursor support
         if self.__collection.database.connection.is_mongos and exhaust:
@@ -147,16 +214,19 @@ class Cursor(object):
         self.__explain = False
         self.__hint = None
         self.__comment = None
-        self.__as_class = as_class
         self.__slave_okay = slave_okay
         self.__manipulate = manipulate
         self.__read_preference = read_preference
         self.__tag_sets = tag_sets
         self.__secondary_acceptable_latency_ms = secondary_acceptable_latency_ms
-        self.__tz_aware = collection.database.connection.tz_aware
         self.__compile_re = compile_re
         self.__must_use_master = _must_use_master
-        self.__uuid_subtype = _uuid_subtype or collection.uuid_subtype
+
+        copts = _codec_options or collection.codec_options
+        if as_class is not None:
+            copts = _CodecOptions(
+                as_class, copts.tz_aware, copts.uuid_representation)
+        self.__codec_options = copts
 
         self.__data = deque()
         self.__connection_id = None
@@ -174,6 +244,8 @@ class Cursor(object):
             self.__query_flags |= _QUERY_OPTIONS["exhaust"]
         if partial:
             self.__query_flags |= _QUERY_OPTIONS["partial"]
+        if oplog_replay:
+            self.__query_flags |= _QUERY_OPTIONS["oplog_replay"]
 
         # this is for passing network_timeout through if it's specified
         # need to use kwargs as None is a legit value for network_timeout
@@ -190,11 +262,25 @@ class Cursor(object):
 
     @property
     def conn_id(self):
+        """**DEPRECATED** The server/client/pool this cursor lives on.
+
+        .. warning:: :attr:`conn_id` is deprecated in this version of
+          PyMongo and removed in PyMongo 3. Use :attr:`address` instead.
+
+        .. versionchanged:: 2.9.4
+          Deprecated conn_id.
+        """
+        return self.__connection_id
+
+    @property
+    def address(self):
         """The server/client/pool this cursor lives on.
 
         Could be (host, port), -1, or None depending on what
         client class executed the initial query or this cursor
         being advanced at all.
+
+        .. versionadded:: 2.9.4
         """
         return self.__connection_id
 
@@ -240,11 +326,11 @@ class Cursor(object):
         values_to_clone = ("spec", "fields", "skip", "limit", "max_time_ms",
                            "comment", "max", "min",
                            "snapshot", "ordering", "explain", "hint",
-                           "batch_size", "max_scan", "as_class", "slave_okay",
+                           "batch_size", "max_scan", "slave_okay",
                            "manipulate", "read_preference", "tag_sets",
                            "secondary_acceptable_latency_ms",
-                           "must_use_master", "uuid_subtype", "compile_re",
-                           "query_flags", "kwargs")
+                           "must_use_master", "codec_options", "compile_re",
+                           "query_flags", "modifiers", "kwargs")
         data = dict((k, v) for k, v in self.__dict__.iteritems()
                     if k.startswith('_Cursor__') and k[9:] in values_to_clone)
         if deepcopy:
@@ -286,7 +372,7 @@ class Cursor(object):
     def __query_spec(self):
         """Get the spec to use for a query.
         """
-        operators = {}
+        operators = self.__modifiers.copy()
         if self.__ordering:
             operators["$orderby"] = self.__ordering
         if self.__explain:
@@ -691,6 +777,12 @@ class Cursor(object):
         `with_limit_and_skip` to ``True`` if that is the desired behavior.
         Raises :class:`~pymongo.errors.OperationFailure` on a database error.
 
+        When used with MongoDB >= 2.6, :meth:`~count` uses any :meth:`~hint`
+        applied to the query. In the following example the hint is passed to
+        the count command:
+
+          collection.find({'field': 'value'}).hint('field_1').count()
+
         With :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`
         or :class:`~pymongo.master_slave_connection.MasterSlaveConnection`,
         if `read_preference` is not
@@ -712,6 +804,9 @@ class Cursor(object):
 
             collection.find({}, network_timeout=1).count()
 
+        .. versionchanged:: 2.8
+           The :meth:`~count` method now supports :meth:`~hint`.
+
         .. versionadded:: 1.1.1
            The `with_limit_and_skip` parameter.
            :meth:`~pymongo.cursor.Cursor.__len__` was deprecated in favor of
@@ -733,6 +828,9 @@ class Cursor(object):
         if self.__comment:
             command['$comment'] = self.__comment
 
+        if self.__hint is not None:
+            command['hint'] = self.__hint
+
         if with_limit_and_skip:
             if self.__limit:
                 command["limit"] = self.__limit
@@ -742,7 +840,7 @@ class Cursor(object):
         database = self.__collection.database
         r = database.command("count", self.__collection.name,
                              allowable_errors=["ns missing"],
-                             uuid_subtype=self.__uuid_subtype,
+                             codec_options=self.__codec_options,
                              compile_re=self.__compile_re,
                              **command)
         if r.get("errmsg", "") == "ns missing":
@@ -795,7 +893,7 @@ class Cursor(object):
         database = self.__collection.database
         return database.command("distinct",
                                 self.__collection.name,
-                                uuid_subtype=self.__uuid_subtype,
+                                codec_options=self.__codec_options,
                                 compile_re=self.__compile_re,
                                 **options)["values"]
 
@@ -825,20 +923,26 @@ class Cursor(object):
 
         `index` should be an index as passed to
         :meth:`~pymongo.collection.Collection.create_index`
-        (e.g. ``[('field', ASCENDING)]``). If `index`
-        is ``None`` any existing hints for this query are cleared. The
-        last hint applied to this cursor takes precedence over all
-        others.
+        (e.g. ``[('field', ASCENDING)]``) or the name of the index.
+        If `index` is ``None`` any existing hint for this query is
+        cleared. The last hint applied to this cursor takes precedence
+        over all others.
 
         :Parameters:
           - `index`: index to hint on (as an index specifier)
+
+        .. versionchanged:: 2.8
+           The :meth:`~hint` method accepts the name of the index.
         """
         self.__check_okay_to_chain()
         if index is None:
             self.__hint = None
             return self
 
-        self.__hint = helpers._index_document(index)
+        if isinstance(index, basestring):
+            self.__hint = index
+        else:
+            self.__hint = helpers._index_document(index)
         return self
 
     def comment(self, comment):
@@ -914,17 +1018,27 @@ class Cursor(object):
                 # due to a socket timeout.
                 self.__killed = True
                 raise
-        else: # exhaust cursor - no getMore message
-            response = client._exhaust_next(self.__exhaust_mgr.sock)
+        else:
+            # Exhaust cursor - no getMore message.
+            try:
+                response = client._exhaust_next(self.__exhaust_mgr.sock)
+            except AutoReconnect:
+                self.__killed = True
+                self.__exhaust_mgr.error()
+                raise
 
         try:
-            response = helpers._unpack_response(response, self.__id,
-                                                self.__as_class,
-                                                self.__tz_aware,
-                                                self.__uuid_subtype,
-                                                self.__compile_re)
-        except CursorNotFound:
+            response = helpers._unpack_response(
+                response,
+                self.__id,
+                self.__codec_options.document_class,
+                self.__codec_options.tz_aware,
+                self.__codec_options.uuid_representation,
+                self.__compile_re)
+        except OperationFailure:
             self.__killed = True
+            # Make sure exhaust socket is returned immediately, if necessary.
+            self.__die()
             # If this is a tailable cursor the error is likely
             # due to capped collection roll over. Setting
             # self.__killed to True ensures Cursor.alive will be
@@ -936,15 +1050,14 @@ class Cursor(object):
             # Don't send kill cursors to another server after a "not master"
             # error. It's completely pointless.
             self.__killed = True
-            client.disconnect()
+            # Make sure exhaust socket is returned immediately, if necessary.
+            self.__die()
+            client._disconnect()
             raise
-        self.__id = response["cursor_id"]
 
-        # starting from doesn't get set on getmore's for tailable cursors
-        if not (self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]):
-            assert response["starting_from"] == self.__retrieved, (
-                "Result batch started from %s, expected %s" % (
-                    response['starting_from'], self.__retrieved))
+        self.__id = response["cursor_id"]
+        if self.__id == 0:
+            self.__killed = True
 
         self.__retrieved += response["number_returned"]
         self.__data = deque(response["data"])
@@ -979,7 +1092,7 @@ class Cursor(object):
                               self.__collection.full_name,
                               self.__skip, ntoreturn,
                               self.__query_spec(), self.__fields,
-                              self.__uuid_subtype))
+                              self.__codec_options.uuid_representation))
             if not self.__id:
                 self.__killed = True
         elif self.__id:  # Get More
@@ -1012,6 +1125,17 @@ class Cursor(object):
         since they will stop iterating even though they *may* return more
         results in the future.
 
+        With regular cursors, simply use a for loop instead of :attr:`alive`::
+
+            for doc in collection.find():
+                print(doc)
+
+        .. note:: Even if :attr:`alive` is True, :meth:`next` can raise
+          :exc:`StopIteration`. :attr:`alive` can also be True while iterating
+          a cursor from a failed server. In this case :attr:`alive` will
+          return False after :meth:`next` fails to retrieve the next batch
+          of results from the server.
+
         .. versionadded:: 1.5
         """
         return bool(len(self.__data) or (not self.__killed))
@@ -1032,11 +1156,12 @@ class Cursor(object):
         return self
 
     def next(self):
+        """Advance the cursor."""
         if self.__empty:
             raise StopIteration
-        db = self.__collection.database
         if len(self.__data) or self._refresh():
             if self.__manipulate:
+                db = self.__collection.database
                 return db._fix_outgoing(self.__data.popleft(),
                                         self.__collection)
             else:

pymongo/cursor_manager.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

pymongo/database.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,34 +18,28 @@ import warnings
 
 from bson.binary import OLD_UUID_SUBTYPE
 from bson.code import Code
+from bson.codec_options import CodecOptions as _CodecOptions
 from bson.dbref import DBRef
 from bson.son import SON
 from pymongo import auth, common, helpers
 from pymongo.collection import Collection
+from pymongo.command_cursor import CommandCursor
 from pymongo.errors import (CollectionInvalid,
                             ConfigurationError,
-                            InvalidName,
                             OperationFailure)
-from pymongo import read_preferences as rp
-
-
-def _check_name(name):
-    """Check if a database name is valid.
-    """
-    if not name:
-        raise InvalidName("database name cannot be the empty string")
-
-    for invalid_char in [" ", ".", "$", "/", "\\", "\x00"]:
-        if invalid_char in name:
-            raise InvalidName("database names cannot contain the "
-                              "character %r" % invalid_char)
+from pymongo.read_preferences import (modes,
+                                      secondary_ok_commands,
+                                      ReadPreference,
+                                      _ServerMode)
+from pymongo.son_manipulator import SONManipulator
 
 
 class Database(common.BaseObject):
     """A Mongo database.
     """
 
-    def __init__(self, connection, name):
+    def __init__(self, connection, name, codec_options=None,
+                 read_preference=None, write_concern=None):
         """Get a database by connection and name.
 
         Raises :class:`TypeError` if `name` is not an instance of
@@ -56,25 +50,39 @@ class Database(common.BaseObject):
         :Parameters:
           - `connection`: a client instance
           - `name`: database name
+          - `codec_options` (optional): An instance of
+            :class:`~bson.codec_options.CodecOptions`. If ``None`` (the
+            default) connection.codec_options is used.
+          - `read_preference` (optional): The read preference to use. If
+            ``None`` (the default) connection.read_preference is used.
+          - `write_concern` (optional): An instance of
+            :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
+            default) connection.write_concern is used.
 
         .. mongodoc:: databases
+
+        .. versionchanged:: 2.9
+           Added the codec_options, read_preference, and write_concern options.
         """
-        super(Database,
-              self).__init__(slave_okay=connection.slave_okay,
-                             read_preference=connection.read_preference,
-                             tag_sets=connection.tag_sets,
-                             secondary_acceptable_latency_ms=(
-                                 connection.secondary_acceptable_latency_ms),
-                             safe=connection.safe,
-                             uuidrepresentation=connection.uuid_subtype,
-                             **connection.write_concern)
+        opts, mode, tags, wc_doc = helpers._get_common_options(
+            connection, codec_options, read_preference, write_concern)
+        salms = connection.secondary_acceptable_latency_ms
+
+        super(Database, self).__init__(
+            codec_options=opts,
+            read_preference=mode,
+            tag_sets=tags,
+            secondary_acceptable_latency_ms=salms,
+            slave_okay=connection.slave_okay,
+            safe=connection.safe,
+            **wc_doc)
 
         if not isinstance(name, basestring):
             raise TypeError("name must be an instance "
                             "of %s" % (basestring.__name__,))
 
         if name != '$external':
-            _check_name(name)
+            helpers._check_database_name(name)
 
         self.__name = unicode(name)
         self.__connection = connection
@@ -92,9 +100,10 @@ class Database(common.BaseObject):
         :Parameters:
           - `manipulator`: the manipulator to add
         """
+        base = SONManipulator()
         def method_overwritten(instance, method):
-            return getattr(instance, method) != \
-                getattr(super(instance.__class__, instance), method)
+            return (getattr(
+                instance, method).im_func != getattr(base, method).im_func)
 
         if manipulator.will_copy():
             if method_overwritten(manipulator, "transform_incoming"):
@@ -122,7 +131,16 @@ class Database(common.BaseObject):
         """The client instance for this :class:`Database`.
 
         .. versionchanged:: 1.3
-           ``connection`` is now a property rather than a method.
+           :attr:`connection` is now a property rather than a method.
+        """
+        return self.__connection
+
+    @property
+    def client(self):
+        """The client instance for this :class:`Database`.
+
+        .. versionadded:: 2.9
+           :attr:`client` is an alias for :attr:`connection`.
         """
         return self.__connection
 
@@ -208,7 +226,48 @@ class Database(common.BaseObject):
         """
         return self.__getattr__(name)
 
-    def create_collection(self, name, **kwargs):
+    def get_collection(self, name, codec_options=None,
+                       read_preference=None, write_concern=None):
+        """Get a :class:`~pymongo.collection.Collection` with the given name
+        and options.
+
+        Useful for creating a :class:`~pymongo.collection.Collection` with
+        different codec options, read preference, and/or write concern from
+        this :class:`Database`.
+
+          >>> from pymongo import ReadPreference
+          >>> db.read_preference == ReadPreference.PRIMARY
+          True
+          >>> coll1 = db.test
+          >>> coll1.read_preference == ReadPreference.PRIMARY
+          True
+          >>> coll2 = db.get_collection(
+          ...     'test', read_preference=ReadPreference.SECONDARY)
+          >>> coll2.read_preference == SECONDARY
+          True
+
+        :Parameters:
+          - `name`: The name of the collection - a string.
+          - `codec_options` (optional): An instance of
+            :class:`~bson.codec_options.CodecOptions`. If ``None`` (the
+            default) the :attr:`codec_options` of this :class:`Database` is
+            used.
+          - `read_preference` (optional): The read preference to use. If
+            ``None`` (the default) the :attr:`read_preference` of this
+            :class:`Database` is used. See :mod:`~pymongo.read_preferences`
+            for options.
+          - `write_concern` (optional): An instance of
+            :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
+            default) the :attr:`write_concern` of this :class:`Database` is
+            used.
+
+        .. versionadded:: 2.9
+        """
+        return Collection(
+            self, name, False, codec_options, read_preference, write_concern)
+
+    def create_collection(self, name, codec_options=None,
+                          read_preference=None, write_concern=None, **kwargs):
         """Create a new :class:`~pymongo.collection.Collection` in this
         database.
 
@@ -217,8 +276,8 @@ class Database(common.BaseObject):
         creation. :class:`~pymongo.errors.CollectionInvalid` will be
         raised if the collection already exists.
 
-        Options should be passed as keyword arguments to this
-        method. Any of the following options are valid:
+        Options should be passed as keyword arguments to this method. Supported
+        options vary with MongoDB release. Some examples include:
 
           - "size": desired initial size for the collection (in
             bytes). For capped collections this size is the max
@@ -226,24 +285,49 @@ class Database(common.BaseObject):
           - "capped": if True, this is a capped collection
           - "max": maximum number of objects if capped (optional)
 
+        See the MongoDB documentation for a full list of supported options by
+        server version.
+
         :Parameters:
           - `name`: the name of the collection to create
+          - `codec_options` (optional): An instance of
+            :class:`~bson.codec_options.CodecOptions`. If ``None`` (the
+            default) the :attr:`codec_options` of this :class:`Database` is
+            used.
+          - `read_preference` (optional): The read preference to use. If
+            ``None`` (the default) the :attr:`read_preference` of this
+            :class:`Database` is used.
+          - `write_concern` (optional): An instance of
+            :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
+            default) the :attr:`write_concern` of this :class:`Database` is
+            used.
           - `**kwargs` (optional): additional keyword arguments will
             be passed as options for the create collection command
 
+        .. versionchanged:: 2.9
+           Added the codec_options, read_preference, and write_concern options.
+
         .. versionchanged:: 2.2
            Removed deprecated argument: options
 
         .. versionchanged:: 1.5
            deprecating `options` in favor of kwargs
         """
-        opts = {"create": True}
-        opts.update(kwargs)
-
         if name in self.collection_names():
             raise CollectionInvalid("collection %s already exists" % name)
 
-        return Collection(self, name, **opts)
+        return Collection(self, name, True, codec_options,
+                          read_preference, write_concern, **kwargs)
+
+    def _apply_incoming_manipulators(self, son, collection):
+        for manipulator in self.__incoming_manipulators:
+            son = manipulator.transform_incoming(son, collection)
+        return son
+
+    def _apply_incoming_copying_manipulators(self, son, collection):
+        for manipulator in self.__incoming_copying_manipulators:
+            son = manipulator.transform_incoming(son, collection)
+        return son
 
     def _fix_incoming(self, son, collection):
         """Apply manipulators to an incoming SON object before it gets stored.
@@ -252,10 +336,8 @@ class Database(common.BaseObject):
           - `son`: the son object going into the database
           - `collection`: the collection the son object is being saved in
         """
-        for manipulator in self.__incoming_manipulators:
-            son = manipulator.transform_incoming(son, collection)
-        for manipulator in self.__incoming_copying_manipulators:
-            son = manipulator.transform_incoming(son, collection)
+        son = self._apply_incoming_manipulators(son, collection)
+        son = self._apply_incoming_copying_manipulators(son, collection)
         return son
 
     def _fix_outgoing(self, son, collection):
@@ -273,7 +355,8 @@ class Database(common.BaseObject):
 
     def _command(self, command, value=1,
                  check=True, allowable_errors=None,
-                 uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True, **kwargs):
+                 uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
+                 read_preference=None, codec_options=None, **kwargs):
         """Internal command helper.
         """
 
@@ -282,7 +365,7 @@ class Database(common.BaseObject):
 
         command_name = command.keys()[0].lower()
         must_use_master = kwargs.pop('_use_master', False)
-        if command_name not in rp.secondary_ok_commands:
+        if command_name not in secondary_ok_commands:
             must_use_master = True
 
         # Special-case: mapreduce can go to secondaries only if inline
@@ -298,19 +381,34 @@ class Database(common.BaseObject):
                     must_use_master = True
                     break
 
+        if codec_options is None or 'as_class' in kwargs:
+            opts = {}
+            if 'as_class' in kwargs:
+                opts['document_class'] = kwargs.pop('as_class')
+            # 'as_class' must be in kwargs so don't use document_class
+            if codec_options:
+                opts['tz_aware'] = codec_options.tz_aware
+                opts['uuid_representation'] = codec_options.uuid_representation
+            else:
+                opts['uuid_representation'] = uuid_subtype
+            codec_options = _CodecOptions(**opts)
+
         extra_opts = {
-            'as_class': kwargs.pop('as_class', None),
             'slave_okay': kwargs.pop('slave_okay', self.slave_okay),
+            '_codec_options': codec_options,
             '_must_use_master': must_use_master,
-            '_uuid_subtype': uuid_subtype
         }
 
-        extra_opts['read_preference'] = kwargs.pop(
-            'read_preference',
-            self.read_preference)
-        extra_opts['tag_sets'] = kwargs.pop(
-            'tag_sets',
-            self.tag_sets)
+        if isinstance(read_preference, _ServerMode):
+            extra_opts['read_preference'] = read_preference.mode
+            extra_opts['tag_sets'] = read_preference.tag_sets
+        else:
+            if read_preference is None:
+                read_preference = self.read_preference
+            extra_opts['read_preference'] = read_preference
+            extra_opts['tag_sets'] = kwargs.pop(
+                'tag_sets',
+                self.tag_sets)
         extra_opts['secondary_acceptable_latency_ms'] = kwargs.pop(
             'secondary_acceptable_latency_ms',
             self.secondary_acceptable_latency_ms)
@@ -323,28 +421,28 @@ class Database(common.BaseObject):
         command.update(kwargs)
 
         # Warn if must_use_master will override read_preference.
-        if (extra_opts['read_preference'] != rp.ReadPreference.PRIMARY and
-                extra_opts['_must_use_master']):
+        if (extra_opts['read_preference'] != ReadPreference.PRIMARY and
+                extra_opts['_must_use_master'] and self.connection._rs_client):
             warnings.warn("%s does not support %s read preference "
                           "and will be routed to the primary instead." %
                           (command_name,
-                           rp.modes[extra_opts['read_preference']]),
-                          UserWarning)
+                           modes[extra_opts['read_preference']]),
+                          UserWarning, stacklevel=3)
 
         cursor = self["$cmd"].find(command, **extra_opts).limit(-1)
         for doc in cursor:
             result = doc
 
         if check:
-            msg = "command %s failed: %%s" % repr(command).replace("%", "%%")
-            helpers._check_command_response(result, self.connection.disconnect,
-                                            msg, allowable_errors)
+            helpers._check_command_response(
+                result, self.connection._disconnect, None, allowable_errors)
 
         return result, cursor.conn_id
 
     def command(self, command, value=1,
                 check=True, allowable_errors=[],
-                uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True, **kwargs):
+                uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
+                read_preference=None, codec_options=None, **kwargs):
         """Issue a MongoDB command.
 
         Send command `command` to the database and return the
@@ -429,10 +527,10 @@ class Database(common.BaseObject):
         .. versionadded:: 1.4
 
         .. mongodoc:: commands
-        .. _localThreshold: http://docs.mongodb.org/manual/reference/mongos/#cmdoption-mongos--localThreshold
         """
         return self._command(command, value, check, allowable_errors,
-                             uuid_subtype, compile_re, **kwargs)[0]
+                             uuid_subtype, compile_re, read_preference,
+                             codec_options, **kwargs)[0]
 
     def collection_names(self, include_system_collections=True):
         """Get a list of all the collection names in this database.
@@ -441,10 +539,30 @@ class Database(common.BaseObject):
           - `include_system_collections` (optional): if ``False`` list
             will not include system collections (e.g ``system.indexes``)
         """
-        results = self["system.namespaces"].find(_must_use_master=True)
-        names = [r["name"] for r in results]
-        names = [n[len(self.__name) + 1:] for n in names
-                 if n.startswith(self.__name + ".") and "$" not in n]
+        client = self.connection
+        client._ensure_connected(True)
+
+        slave_okay = not client._rs_client and not client.is_mongos
+        if client.max_wire_version > 2:
+            res, addr = self._command("listCollections",
+                                      cursor={},
+                                      read_preference=ReadPreference.PRIMARY,
+                                      slave_okay=slave_okay)
+            # MongoDB 2.8rc2
+            if "collections" in res:
+                results = res["collections"]
+            # >= MongoDB 2.8rc3
+            else:
+                results = CommandCursor(self["$cmd"], res["cursor"], addr)
+            names = [result["name"] for result in results]
+        else:
+            names = [result["name"] for result
+                     in self["system.namespaces"].find(
+                         slave_okay=slave_okay,
+                         _must_use_master=True)]
+            names = [n[len(self.__name) + 1:] for n in names
+                     if n.startswith(self.__name + ".") and "$" not in n]
+
         if not include_system_collections:
             names = [n for n in names if not n.startswith("system.")]
         return names
@@ -466,7 +584,8 @@ class Database(common.BaseObject):
 
         self.__connection._purge_index(self.__name, name)
 
-        self.command("drop", unicode(name), allowable_errors=["ns not found"])
+        self.command("drop", unicode(name), allowable_errors=["ns not found"],
+                     read_preference=ReadPreference.PRIMARY)
 
     def validate_collection(self, name_or_collection,
                             scandata=False, full=False):
@@ -504,7 +623,8 @@ class Database(common.BaseObject):
                             "%s or Collection" % (basestring.__name__,))
 
         result = self.command("validate", unicode(name),
-                              scandata=scandata, full=full)
+                              scandata=scandata, full=full,
+                              read_preference=ReadPreference.PRIMARY)
 
         valid = True
         # Pre 1.9 results
@@ -553,7 +673,8 @@ class Database(common.BaseObject):
 
         .. mongodoc:: profiling
         """
-        result = self.command("profile", -1)
+        result = self.command("profile", -1,
+                              read_preference=ReadPreference.PRIMARY)
 
         assert result["was"] >= 0 and result["was"] <= 2
         return result["was"]
@@ -593,9 +714,11 @@ class Database(common.BaseObject):
             raise TypeError("slow_ms must be an integer")
 
         if slow_ms is not None:
-            self.command("profile", level, slowms=slow_ms)
+            self.command("profile", level, slowms=slow_ms,
+                         read_preference=ReadPreference.PRIMARY)
         else:
-            self.command("profile", level)
+            self.command("profile", level,
+                         read_preference=ReadPreference.PRIMARY)
 
     def profiling_info(self):
         """Returns a list containing current profiling information.
@@ -605,45 +728,111 @@ class Database(common.BaseObject):
         return list(self["system.profile"].find())
 
     def error(self):
-        """Get a database error if one occured on the last operation.
+        """**DEPRECATED**: Get the error if one occurred on the last operation.
+
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0.
 
         Return None if the last operation was error-free. Otherwise return the
         error that occurred.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        error = self.command("getlasterror")
+        warnings.warn("Database.error() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        error = self.command("getlasterror",
+                             read_preference=ReadPreference.PRIMARY)
         error_msg = error.get("err", "")
         if error_msg is None:
             return None
         if error_msg.startswith("not master"):
-            self.__connection.disconnect()
+            self.__connection._disconnect()
         return error
 
     def last_status(self):
-        """Get status information from the last operation.
+        """**DEPRECATED**: Get status information from the last operation.
+
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0.
 
         Returns a SON object with status information.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        return self.command("getlasterror")
+        warnings.warn("last_status() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        return self.command("getlasterror",
+                            read_preference=ReadPreference.PRIMARY)
 
     def previous_error(self):
-        """Get the most recent error to have occurred on this database.
+        """**DEPRECATED**: Get the most recent error on this database.
+
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0. Furthermore, the underlying database command
+        ``getpreverror`` will be removed in a future MongoDB release.
 
         Only returns errors that have occurred since the last call to
-        `Database.reset_error_history`. Returns None if no such errors have
+        :meth:`reset_error_history`. Returns None if no such errors have
         occurred.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        error = self.command("getpreverror")
+        warnings.warn("previous_error() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        error = self.command("getpreverror",
+                             read_preference=ReadPreference.PRIMARY)
         if error.get("err", 0) is None:
             return None
         return error
 
     def reset_error_history(self):
-        """Reset the error history of this database.
+        """**DEPRECATED**: Reset the error history of this database.
 
-        Calls to `Database.previous_error` will only return errors that have
+        This method is obsolete: all MongoDB write operations (insert, update,
+        remove, and so on) use the write concern ``w=1`` and report their
+        errors by default.
+
+        This method must be called in the same
+        :doc:`request </examples/requests>` as the preceding operation,
+        otherwise it is unreliable. Requests are deprecated and will be removed
+        in PyMongo 3.0. Furthermore, the underlying database command
+        ``reseterror`` will be removed in a future MongoDB release.
+
+        Calls to :meth:`previous_error` will only return errors that have
         occurred since the most recent call to this method.
+
+        .. versionchanged:: 2.8
+           Deprecated.
         """
-        self.command("reseterror")
+        warnings.warn("reset_error_history() is deprecated",
+                      DeprecationWarning, stacklevel=2)
+
+        self.command("reseterror",
+                     read_preference=ReadPreference.PRIMARY)
 
     def __iter__(self):
         return self
@@ -689,7 +878,9 @@ class Database(common.BaseObject):
             opts["pwd"] = auth._password_digest(name, password)
             opts["digestPassword"] = False
 
-        opts["writeConcern"] = self._get_wc_override()
+        write_concern = self._get_wc_override() or self.write_concern
+        if write_concern:
+            opts["writeConcern"] = write_concern
         opts.update(kwargs)
 
         if create:
@@ -697,7 +888,8 @@ class Database(common.BaseObject):
         else:
             command_name = "updateUser"
 
-        self.command(command_name, name, **opts)
+        self.command(command_name, name,
+                     read_preference=ReadPreference.PRIMARY, **opts)
 
     def _legacy_add_user(self, name, password, read_only, **kwargs):
         """Uses v1 system to add users, i.e. saving to system.users.
@@ -716,6 +908,11 @@ class Database(common.BaseObject):
             # See SERVER-4225 for more information.
             if 'login' in str(exc):
                 pass
+            # First admin user add fails gle from mongos 2.0.x
+            # and 2.2.x.
+            elif (exc.details and
+                    'getlasterror' in exc.details.get('note', '')):
+                pass
             else:
                 raise
 
@@ -763,21 +960,22 @@ class Database(common.BaseObject):
                                          "read_only and roles together")
 
         try:
-            uinfo = self.command("usersInfo", name)
+            uinfo = self.command("usersInfo", name,
+                                 read_preference=ReadPreference.PRIMARY)
+            self._create_or_update_user(
+                (not uinfo["users"]), name, password, read_only, **kwargs)
         except OperationFailure, exc:
             # MongoDB >= 2.5.3 requires the use of commands to manage
-            # users. "No such command" error didn't return an error
-            # code (59) before MongoDB 2.4.7 so we assume that an error
-            # code of None means the userInfo command doesn't exist and
-            # we should fall back to the legacy add user code.
-            if exc.code in (59, None):
+            # users.
+            if exc.code in common.COMMAND_NOT_FOUND_CODES:
                 self._legacy_add_user(name, password, read_only, **kwargs)
-                return
-            raise
-
-        # Create the user if not found in uinfo, otherwise update one.
-        self._create_or_update_user(
-            (not uinfo["users"]), name, password, read_only, **kwargs)
+            # Unauthorized. MongoDB >= 2.7.1 has a narrow localhost exception,
+            # and we must add a user before sending commands.
+            elif exc.code == 13:
+                self._create_or_update_user(
+                    True, name, password, read_only, **kwargs)
+            else:
+                raise
 
     def remove_user(self, name):
         """Remove user `name` from this :class:`Database`.
@@ -792,18 +990,22 @@ class Database(common.BaseObject):
         """
 
         try:
-            self.command("dropUser", name,
-                         writeConcern=self._get_wc_override())
+            cmd = SON([("dropUser", name)])
+            write_concern = self._get_wc_override() or self.write_concern
+            if write_concern:
+                cmd["writeConcern"] = write_concern
+            self.command(cmd,
+                         read_preference=ReadPreference.PRIMARY)
         except OperationFailure, exc:
             # See comment in add_user try / except above.
-            if exc.code in (59, None):
+            if exc.code in common.COMMAND_NOT_FOUND_CODES:
                 self.system.users.remove({"user": name},
                                          **self._get_wc_override())
                 return
             raise
 
     def authenticate(self, name, password=None,
-                     source=None, mechanism='MONGODB-CR', **kwargs):
+                     source=None, mechanism='DEFAULT', **kwargs):
         """Authenticate to use this database.
 
         Authentication lasts for the life of the underlying client
@@ -839,11 +1041,15 @@ class Database(common.BaseObject):
             specified the current database is used.
           - `mechanism` (optional): See
             :data:`~pymongo.auth.MECHANISMS` for options.
-            Defaults to MONGODB-CR (MongoDB Challenge Response protocol)
+            By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
+            MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
           - `gssapiServiceName` (optional): Used with the GSSAPI mechanism
             to specify the service name portion of the service principal name.
             Defaults to 'mongodb'.
 
+        .. versionadded:: 2.8
+           Use SCRAM-SHA-1 with MongoDB 3.0 and later.
+
         .. versionchanged:: 2.5
            Added the `source` and `mechanism` parameters. :meth:`authenticate`
            now raises a subclass of :class:`~pymongo.errors.PyMongoError` if
@@ -869,9 +1075,8 @@ class Database(common.BaseObject):
             validated_options[normalized] = val
 
         credentials = auth._build_credentials_tuple(mechanism,
-                                source or self.name, unicode(name),
-                                password and unicode(password) or None,
-                                validated_options)
+                                source or self.name, name,
+                                password, validated_options)
         self.connection._cache_credentials(self.name, credentials)
         return True
 
@@ -886,7 +1091,7 @@ class Database(common.BaseObject):
         # Sockets will be deauthenticated as they are used.
         self.connection._purge_credentials(self.name)
 
-    def dereference(self, dbref):
+    def dereference(self, dbref, **kwargs):
         """Dereference a :class:`~bson.dbref.DBRef`, getting the
         document it points to.
 
@@ -898,6 +1103,9 @@ class Database(common.BaseObject):
 
         :Parameters:
           - `dbref`: the reference
+          - `**kwargs` (optional): any additional keyword arguments
+            are the same as the arguments to
+            :meth:`~pymongo.collection.Collection.find`.
         """
         if not isinstance(dbref, DBRef):
             raise TypeError("cannot dereference a %s" % type(dbref))
@@ -905,7 +1113,7 @@ class Database(common.BaseObject):
             raise ValueError("trying to dereference a DBRef that points to "
                              "another database (%r not %r)" % (dbref.database,
                                                                self.__name))
-        return self[dbref.collection].find_one({"_id": dbref.id})
+        return self[dbref.collection].find_one({"_id": dbref.id}, **kwargs)
 
     def eval(self, code, *args):
         """Evaluate a JavaScript expression in MongoDB.
@@ -926,11 +1134,16 @@ class Database(common.BaseObject):
             evaluated
           - `args` (optional): additional positional arguments are
             passed to the `code` being evaluated
+
+        .. warning:: the eval command is deprecated in MongoDB 3.0 and
+          will be removed in a future server version.
         """
         if not isinstance(code, Code):
             code = Code(code)
 
-        result = self.command("$eval", code, args=args)
+        result = self.command("$eval", code,
+                              read_preference=ReadPreference.PRIMARY,
+                              args=args)
         return result.get("retval", None)
 
     def __call__(self, *args, **kwargs):

pymongo/errors.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

pymongo/helpers.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,18 +16,53 @@
 
 import random
 import struct
+import warnings
 
 import bson
 import pymongo
 
 from bson.binary import OLD_UUID_SUBTYPE
 from bson.son import SON
+from pymongo import auth
 from pymongo.errors import (AutoReconnect,
                             CursorNotFound,
                             DuplicateKeyError,
+                            InvalidName,
+                            InvalidOperation,
                             OperationFailure,
                             ExecutionTimeout,
                             WTimeoutError)
+from pymongo.read_preferences import _ServerMode
+from pymongo.write_concern import WriteConcern as _WriteConcern
+
+
+def _get_common_options(obj, codec_options, read_preference, write_concern):
+    """Get the codec options, read preference mode and tags, and write concern
+    necessary to create a new Database of Collection instance.
+    """
+    if codec_options is None:
+        codec_options = obj.codec_options
+
+    if read_preference is None:
+        rp_mode = obj.read_preference
+        rp_tags = obj.tag_sets
+    else:
+        if isinstance(read_preference, _ServerMode):
+            rp_mode = read_preference.mode
+            rp_tags = read_preference.tag_sets
+        else:
+            rp_mode = read_preference
+            rp_tags = [{}]
+
+    if write_concern is None:
+        wc_document = obj.write_concern
+    else:
+        if not isinstance(write_concern, _WriteConcern):
+            raise TypeError("write_concern must be an instance of "
+                            "pymongo.write_concern.WriteConcern")
+        wc_document = write_concern.document
+
+    return codec_options, rp_mode, rp_tags, wc_document
 
 
 def _index_list(key_or_list, direction=None):
@@ -143,8 +178,8 @@ def _check_command_response(response, reset, msg=None, allowable_errors=None):
         # for some errors.
         if "raw" in response:
             for shard in response["raw"].itervalues():
-                if not shard.get("ok"):
-                    # Just grab the first error...
+                # Grab the first non-empty raw error from a shard.
+                if shard.get("errmsg") and not shard.get("ok"):
                     details = shard
                     break
 
@@ -223,6 +258,121 @@ def _fields_list_to_dict(fields):
     return as_dict
 
 
+def _check_database_name(name):
+    """Check if a database name is valid."""
+    if not name:
+        raise InvalidName("database name cannot be the empty string")
+
+    for invalid_char in [' ', '.', '$', '/', '\\', '\x00', '"']:
+        if invalid_char in name:
+            raise InvalidName("database names cannot contain the "
+                              "character %r" % invalid_char)
+
+
+def _copy_database(
+        fromdb,
+        todb,
+        fromhost,
+        mechanism,
+        username,
+        password,
+        sock_info,
+        cmd_func):
+    """Copy a database, perhaps from a remote host.
+
+    :Parameters:
+      - `fromdb`: Source database.
+      - `todb`: Target database.
+      - `fromhost`: Source host like 'foo.com', 'foo.com:27017', or None.
+      - `mechanism`: An authentication mechanism.
+      - `username`: A str or unicode, or None.
+      - `password`: A str or unicode, or None.
+      - `sock_info`: A SocketInfo instance.
+      - `cmd_func`: A callback taking args sock_info, database, command doc.
+    """
+    if not isinstance(fromdb, basestring):
+        raise TypeError('from_name must be an instance '
+                        'of %s' % (basestring.__name__,))
+    if not isinstance(todb, basestring):
+        raise TypeError('to_name must be an instance '
+                        'of %s' % (basestring.__name__,))
+
+    _check_database_name(todb)
+
+    warnings.warn("copy_database is deprecated. Use the raw 'copydb' command"
+                  " or db.copyDatabase() in the mongo shell. See"
+                  " doc/examples/copydb.",
+                  DeprecationWarning, stacklevel=2)
+
+    # It would be better if the user told us what mechanism to use, but for
+    # backwards compatibility with earlier PyMongos we don't require the
+    # mechanism. Hope 'fromhost' runs the same version as the target.
+    if mechanism == 'DEFAULT':
+        if sock_info.max_wire_version >= 3:
+            mechanism = 'SCRAM-SHA-1'
+        else:
+            mechanism = 'MONGODB-CR'
+
+    if username is not None:
+        if mechanism == 'SCRAM-SHA-1':
+            credentials = auth._build_credentials_tuple(mech=mechanism,
+                                                        source='admin',
+                                                        user=username,
+                                                        passwd=password,
+                                                        extra=None)
+
+            try:
+                auth._copydb_scram_sha1(credentials=credentials,
+                                        sock_info=sock_info,
+                                        cmd_func=cmd_func,
+                                        fromdb=fromdb,
+                                        todb=todb,
+                                        fromhost=fromhost)
+            except OperationFailure, exc:
+                errmsg = exc.details and exc.details.get('errmsg') or ''
+                if 'no such cmd: saslStart' in errmsg:
+                    explanation = (
+                        "%s doesn't support SCRAM-SHA-1, pass"
+                        " mechanism='MONGODB-CR' to copy_database" % fromhost)
+
+                    raise OperationFailure(explanation,
+                                           exc.code,
+                                           exc.details)
+                else:
+                    raise
+
+        elif mechanism == 'MONGODB-CR':
+            get_nonce_cmd = SON([('copydbgetnonce', 1),
+                                 ('fromhost', fromhost)])
+
+            get_nonce_response, _ = cmd_func(sock_info, 'admin', get_nonce_cmd)
+            nonce = get_nonce_response['nonce']
+            copydb_cmd = SON([('copydb', 1),
+                              ('fromdb', fromdb),
+                              ('todb', todb)])
+
+            copydb_cmd['username'] = username
+            copydb_cmd['nonce'] = nonce
+            copydb_cmd['key'] = auth._auth_key(nonce, username, password)
+            if fromhost is not None:
+                copydb_cmd['fromhost'] = fromhost
+
+            cmd_func(sock_info, 'admin', copydb_cmd)
+        else:
+            raise InvalidOperation('Authentication mechanism %r not supported'
+                                   ' for copy_database' % mechanism)
+    else:
+        # No username.
+        copydb_cmd = SON([('copydb', 1),
+                          ('fromdb', fromdb),
+                          ('todb', todb)])
+
+        if fromhost:
+            copydb_cmd['fromhost'] = fromhost
+
+        cmd_func(sock_info, 'admin', copydb_cmd)
+
+
 def shuffled(sequence):
     """Returns a copy of the sequence (as a :class:`list`) which has been
     shuffled by :func:`random.shuffle`.

pymongo/master_slave_connection.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,24 +12,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Master-Slave connection to Mongo.
+"""**DEPRECATED**: Master-Slave connection to Mongo.
 
 Performs all writes to Master instance and distributes reads among all
 slaves. Reads are tried on each slave in turn until the read succeeds
 or all slaves failed.
+
+MasterSlaveConnection is deprecated and will be removed in PyMongo 3.0.
+Deploy your MongoDB servers as a replica set instead of a master-slave set,
+and use a :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`.
+If you cannot replace your master-slave set with a replica set, connect
+directly to the master and each slave with instances of
+:class:`~pymongo.mongo_client.MongoClient`.
+
+.. seealso:: `replica set documentation <http://dochub.mongodb.org/core/rs>`_.
+
+.. versionchanged:: 2.8
+   Deprecated.
 """
 
+import warnings
+
+from bson.codec_options import CodecOptions
 from pymongo import helpers, thread_util
 from pymongo import ReadPreference
-from pymongo.common import BaseObject
+from pymongo.common import BaseObject, validate_boolean
 from pymongo.mongo_client import MongoClient
 from pymongo.database import Database
 from pymongo.errors import AutoReconnect
 
 
 class MasterSlaveConnection(BaseObject):
-    """A master-slave connection to Mongo.
-    """
+
+    _rs_client = True
 
     def __init__(self, master, slaves=[], document_class=dict, tz_aware=False):
         """Create a new Master-Slave connection.
@@ -67,15 +82,20 @@ class MasterSlaveConnection(BaseObject):
                 raise TypeError("slave %r is not an instance of MongoClient" %
                                 slave)
 
+        warnings.warn("MasterSlaveConnection is deprecated, and will be"
+                      " removed in PyMongo 3.0.",
+                      DeprecationWarning, stacklevel=2)
+
+        validate_boolean('tz_aware', tz_aware)
+        codec_options = CodecOptions(document_class, tz_aware)
         super(MasterSlaveConnection,
               self).__init__(read_preference=ReadPreference.SECONDARY,
                              safe=master.safe,
+                             codec_options=codec_options,
                              **master.write_concern)
 
         self.__master = master
         self.__slaves = slaves
-        self.__document_class = document_class
-        self.__tz_aware = tz_aware
         self.__request_counter = thread_util.Counter(master.use_greenlets)
 
     @property
@@ -104,10 +124,12 @@ class MasterSlaveConnection(BaseObject):
         return self.master.use_greenlets
 
     def get_document_class(self):
-        return self.__document_class
+        return self._codec_options.document_class
 
     def set_document_class(self, klass):
-        self.__document_class = klass
+        tz_aware = self._codec_options.tz_aware
+        uuid_rep = self._codec_options.uuid_representation
+        self._codec_options = CodecOptions(klass, tz_aware, uuid_rep)
 
     document_class = property(get_document_class, set_document_class,
                               doc="""Default class to use for documents
@@ -115,7 +137,7 @@ class MasterSlaveConnection(BaseObject):
 
     @property
     def tz_aware(self):
-        return self.__tz_aware
+        return self._codec_options.tz_aware
 
     @property
     def max_bson_size(self):
@@ -175,10 +197,22 @@ class MasterSlaveConnection(BaseObject):
         .. seealso:: Module :mod:`~pymongo.mongo_client`
         .. versionadded:: 1.10.1
         """
+        self._disconnect()
+
+    def _disconnect(self):
+        """Internal disconnect helper."""
         self.__master.disconnect()
         for slave in self.__slaves:
             slave.disconnect()
 
+    def close(self):
+        """Alias for :meth:`disconnect`
+
+        .. seealso:: :meth:`end_request`
+        .. versionadded:: 2.8
+        """
+        self._disconnect()
+
     def set_cursor_manager(self, manager_class):
         """Set the cursor manager for this connection.
 
@@ -194,12 +228,9 @@ class MasterSlaveConnection(BaseObject):
         """
         self.__master._ensure_connected(sync)
 
-    # _connection_to_use is a hack that we need to include to make sure
-    # that killcursor operations can be sent to the same instance on which
-    # the cursor actually resides...
     def _send_message(self, message,
                       with_last_error=False,
-                      command=False, _connection_to_use=None):
+                      command=False):
         """Say something to Mongo.
 
         Sends a message on the Master connection. This is used for inserts,
@@ -213,11 +244,8 @@ class MasterSlaveConnection(BaseObject):
           - `data`: data to send
           - `safe`: perform a getLastError after sending the message
         """
-        if _connection_to_use is None or _connection_to_use == -1:
-            return self.__master._send_message(message,
-                                               with_last_error, command)
-        return self.__slaves[_connection_to_use]._send_message(
-            message, with_last_error, command, check_primary=False)
+        return self.__master._send_message(message,
+                                           with_last_error, command)
 
     # _connection_to_use is a hack that we need to include to make sure
     # that getmore operations can be sent to the same instance on which
@@ -366,3 +394,26 @@ class MasterSlaveConnection(BaseObject):
         return self.__master._purge_index(database_name,
                                           collection_name,
                                           index_name)
+
+    def server_info(self):
+        """Get information about the MongoDB master we're connected to.
+
+        .. versionadded:: 2.8
+        """
+        return self.__master.admin.command("buildinfo")
+
+    def _cache_credentials(self, source, credentials, connect=True):
+        self.__master._cache_credentials(source, credentials, connect)
+        for slave in self.__slaves:
+            # Use connect=False here so that credentials are cached
+            # on the slaves no matter what. Since auth succeeded on the
+            # master we know the credentials are correct and the slaves
+            # will authenticate as needed. This avoids issues with slave
+            # auth problems due to problems other than bad credentials
+            # (e.g. network errors).
+            slave._cache_credentials(source, credentials, connect=False)
+
+    def _purge_credentials(self, source):
+        self.__master._purge_credentials(source)
+        for slave in self.__slaves:
+            slave._purge_credentials(source)

pymongo/member.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2014 MongoDB, Inc.
+# Copyright 2013-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -145,6 +145,31 @@ class Member(object):
 
         return False
 
+    def get_socket(self, force=False):
+        sock_info = self.pool.get_socket(force)
+        sock_info.set_wire_version_range(self.min_wire_version,
+                                         self.max_wire_version)
+
+        return sock_info
+
+    def maybe_return_socket(self, sock_info):
+        self.pool.maybe_return_socket(sock_info)
+
+    def discard_socket(self, sock_info):
+        self.pool.discard_socket(sock_info)
+
+    def start_request(self):
+        self.pool.start_request()
+
+    def in_request(self):
+        return self.pool.in_request()
+
+    def end_request(self):
+        self.pool.end_request()
+
+    def reset(self):
+        self.pool.reset()
+
     def __str__(self):
         return '<Member "%s:%s" primary=%r>' % (
             self.host[0], self.host[1], self.is_primary)

pymongo/message.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -81,7 +81,8 @@ def __pack_message(operation, data):
 
 
 def insert(collection_name, docs, check_keys,
-           safe, last_error_args, continue_on_error, uuid_subtype):
+           safe, last_error_args, continue_on_error, uuid_subtype,
+           codec_options=None):
     """Get an **insert** message.
 
     .. note:: As of PyMongo 2.6, this function is no longer used. It
@@ -90,6 +91,8 @@ def insert(collection_name, docs, check_keys,
        be removed in a future release.
 
     """
+    if codec_options is not None:
+        uuid_subtype = codec_options.uuid_representation
     options = 0
     if continue_on_error:
         options += 1
@@ -113,9 +116,12 @@ if _use_c:
 
 
 def update(collection_name, upsert, multi,
-           spec, doc, safe, last_error_args, check_keys, uuid_subtype):
+           spec, doc, safe, last_error_args, check_keys, uuid_subtype,
+           codec_options=None):
     """Get an **update** message.
     """
+    if codec_options is not None:
+        uuid_subtype = codec_options.uuid_representation
     options = 0
     if upsert:
         options += 1
@@ -142,9 +148,11 @@ if _use_c:
 
 def query(options, collection_name, num_to_skip,
           num_to_return, query, field_selector=None,
-          uuid_subtype=OLD_UUID_SUBTYPE):
+          uuid_subtype=OLD_UUID_SUBTYPE, codec_options=None):
     """Get a **query** message.
     """
+    if codec_options is not None:
+        uuid_subtype = codec_options.uuid_representation
     data = struct.pack("<I", options)
     data += bson._make_c_string(collection_name)
     data += struct.pack("<i", num_to_skip)
@@ -175,9 +183,11 @@ if _use_c:
 
 
 def delete(collection_name, spec, safe,
-           last_error_args, uuid_subtype, options=0):
+           last_error_args, uuid_subtype, options=0, codec_options=None):
     """Get a **delete** message.
     """
+    if codec_options is not None:
+        uuid_subtype = codec_options.uuid_representation
     data = _ZERO_32
     data += bson._make_c_string(collection_name)
     data += struct.pack("<I", options)

pymongo/operations.py

@@ -0,0 +1,175 @@
+# Copyright 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Operation class definitions."""
+
+from pymongo.common import validate_boolean, validate_is_dict
+
+
+class _WriteOp(object):
+    """Private base class for all write operations."""
+
+    __slots__ = ("_filter", "_doc", "_upsert")
+
+    def __init__(self, filter=None, doc=None, upsert=None):
+        if filter is not None:
+            validate_is_dict("filter", filter)
+        if upsert is not None:
+            validate_boolean("upsert", upsert)
+        self._filter = filter
+        self._doc = doc
+        self._upsert = upsert
+
+
+class InsertOne(_WriteOp):
+    """Represents an insert_one operation."""
+
+    def __init__(self, document):
+        """Create an InsertOne instance.
+
+        For use with :meth:`~pymongo.collection.Collection.bulk_write`.
+
+        :Parameters:
+          - `document`: The document to insert. If the document is missing an
+            _id field one will be added.
+        """
+        super(InsertOne, self).__init__(doc=document)
+
+    def _add_to_bulk(self, bulkobj):
+        """Add this operation to the _Bulk instance `bulkobj`."""
+        bulkobj.add_insert(self._doc)
+
+    def __repr__(self):
+        return "InsertOne(%r)" % (self._doc,)
+
+
+class DeleteOne(_WriteOp):
+    """Represents a delete_one operation."""
+
+    def __init__(self, filter):
+        """Create a DeleteOne instance.
+
+        For use with :meth:`~pymongo.collection.Collection.bulk_write`.
+
+        :Parameters:
+          - `filter`: A query that matches the document to delete.
+        """
+        super(DeleteOne, self).__init__(filter)
+
+    def _add_to_bulk(self, bulkobj):
+        """Add this operation to the _Bulk instance `bulkobj`."""
+        bulkobj.add_delete(self._filter, 1)
+
+    def __repr__(self):
+        return "DeleteOne(%r)" % (self._filter,)
+
+
+class DeleteMany(_WriteOp):
+    """Represents a delete_many operation."""
+
+    def __init__(self, filter):
+        """Create a DeleteMany instance.
+
+        For use with :meth:`~pymongo.collection.Collection.bulk_write`.
+
+        :Parameters:
+          - `filter`: A query that matches the documents to delete.
+        """
+        super(DeleteMany, self).__init__(filter)
+
+    def _add_to_bulk(self, bulkobj):
+        """Add this operation to the _Bulk instance `bulkobj`."""
+        bulkobj.add_delete(self._filter, 0)
+
+    def __repr__(self):
+        return "DeleteMany(%r)" % (self._filter,)
+
+
+class ReplaceOne(_WriteOp):
+    """Represents a replace_one operation."""
+
+    def __init__(self, filter, replacement, upsert=False):
+        """Create a ReplaceOne instance.
+
+        For use with :meth:`~pymongo.collection.Collection.bulk_write`.
+
+        :Parameters:
+          - `filter`: A query that matches the document to replace.
+          - `replacement`: The new document.
+          - `upsert` (optional): If ``True``, perform an insert if no documents
+            match the filter.
+        """
+        super(ReplaceOne, self).__init__(filter, replacement, upsert)
+
+    def _add_to_bulk(self, bulkobj):
+        """Add this operation to the _Bulk instance `bulkobj`."""
+        bulkobj.add_replace(self._filter, self._doc, self._upsert)
+
+    def __repr__(self):
+        return "ReplaceOne(%r, %r, %r)" % (self._filter,
+                                           self._doc,
+                                           self._upsert)
+
+
+class UpdateOne(_WriteOp):
+    """Represents an update_one operation."""
+
+    def __init__(self, filter, update, upsert=False):
+        """Represents an update_one operation.
+
+        For use with :meth:`~pymongo.collection.Collection.bulk_write`.
+
+        :Parameters:
+          - `filter`: A query that matches the document to update.
+          - `update`: The modifications to apply.
+          - `upsert` (optional): If ``True``, perform an insert if no documents
+            match the filter.
+        """
+        super(UpdateOne, self).__init__(filter, update, upsert)
+
+    def _add_to_bulk(self, bulkobj):
+        """Add this operation to the _Bulk instance `bulkobj`."""
+        bulkobj.add_update(self._filter, self._doc, False, self._upsert)
+
+    def __repr__(self):
+        return "UpdateOne(%r, %r, %r)" % (self._filter,
+                                          self._doc,
+                                          self._upsert)
+
+
+class UpdateMany(_WriteOp):
+    """Represents an update_many operation."""
+
+    def __init__(self, filter, update, upsert=False):
+        """Create an UpdateMany instance.
+
+        For use with :meth:`~pymongo.collection.Collection.bulk_write`.
+
+        :Parameters:
+          - `filter`: A query that matches the documents to update.
+          - `update`: The modifications to apply.
+          - `upsert` (optional): If ``True``, perform an insert if no documents
+            match the filter.
+        """
+        super(UpdateMany, self).__init__(filter, update, upsert)
+
+    def _add_to_bulk(self, bulkobj):
+        """Add this operation to the _Bulk instance `bulkobj`."""
+        bulkobj.add_update(self._filter, self._doc, True, self._upsert)
+
+    def __repr__(self):
+        return "UpdateMany(%r, %r, %r)" % (self._filter,
+                                           self._doc,
+                                           self._upsert)
+

pymongo/pool.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2014 MongoDB, Inc.
+# Copyright 2011-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -24,9 +24,14 @@ from pymongo.common import HAS_SSL
 from pymongo.errors import ConnectionFailure, ConfigurationError
 
 try:
-    from ssl import match_hostname
+    from ssl import match_hostname, CertificateError
 except ImportError:
-    from pymongo.ssl_match_hostname import match_hostname
+    from pymongo.ssl_match_hostname import match_hostname, CertificateError
+
+try:
+    from ssl import SSLContext as _SSLContext
+except ImportError:
+    from pymongo.ssl_context import SSLContext as _SSLContext
 
 if HAS_SSL:
     import ssl
@@ -63,6 +68,9 @@ class SocketInfo(object):
         self.last_checkout = time.time()
         self.forced = False
 
+        self._min_wire_version = None
+        self._max_wire_version = None
+
         # The pool's pool_id changes with each reset() so we can close sockets
         # created before the last reset.
         self.pool_id = pool_id
@@ -75,6 +83,20 @@ class SocketInfo(object):
         except:
             pass
 
+    def set_wire_version_range(self, min_wire_version, max_wire_version):
+        self._min_wire_version = min_wire_version
+        self._max_wire_version = max_wire_version
+
+    @property
+    def min_wire_version(self):
+        assert self._min_wire_version is not None
+        return self._min_wire_version
+
+    @property
+    def max_wire_version(self):
+        assert self._max_wire_version is not None
+        return self._max_wire_version
+
     def __eq__(self, other):
         # Need to check if other is NO_REQUEST or NO_SOCKET_YET, and then check
         # if its sock is the same as ours
@@ -100,7 +122,8 @@ class Pool:
     def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl,
                  use_greenlets, ssl_keyfile=None, ssl_certfile=None,
                  ssl_cert_reqs=None, ssl_ca_certs=None,
-                 wait_queue_timeout=None, wait_queue_multiple=None):
+                 wait_queue_timeout=None, wait_queue_multiple=None,
+                 socket_keepalive=False, ssl_match_hostname=True):
         """
         :Parameters:
           - `pair`: a (hostname, port) tuple
@@ -136,6 +159,15 @@ class Pool:
             free sockets.
           - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
             the number of threads allowed to wait for a socket at one time.
+          - `socket_keepalive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
+          - `ssl_match_hostname`: If ``True`` (the default), and
+            `ssl_cert_reqs` is not ``ssl.CERT_NONE``, enables hostname
+            verification using the :func:`~ssl.match_hostname` function from
+            python's :mod:`~ssl` module. Think very carefully before setting
+            this to ``False`` as that could make your application vulnerable to
+            man-in-the-middle attacks.
         """
         # Only check a socket's health with _closed() every once in a while.
         # Can override for testing: 0 to always check, None to never check.
@@ -154,14 +186,39 @@ class Pool:
         self.conn_timeout = conn_timeout
         self.wait_queue_timeout = wait_queue_timeout
         self.wait_queue_multiple = wait_queue_multiple
-        self.use_ssl = use_ssl
-        self.ssl_keyfile = ssl_keyfile
-        self.ssl_certfile = ssl_certfile
-        self.ssl_cert_reqs = ssl_cert_reqs
-        self.ssl_ca_certs = ssl_ca_certs
+        self.socket_keepalive = socket_keepalive
+        self.ssl_ctx = None
+        self.ssl_match_hostname = ssl_match_hostname
 
-        if HAS_SSL and use_ssl and not ssl_cert_reqs:
-            self.ssl_cert_reqs = ssl.CERT_NONE
+        if HAS_SSL and use_ssl:
+            # PROTOCOL_TLS_CLIENT added and PROTOCOL_SSLv23
+            # deprecated in CPython 3.6
+            self.ssl_ctx = _SSLContext(
+                getattr(ssl, 'PROTOCOL_TLS_CLIENT', ssl.PROTOCOL_SSLv23))
+            # SSLContext.check_hostname was added in 2.7.9 and 3.4. Using it
+            # forces the use of SNI, which PyMongo 2.x doesn't support.
+            # PROTOCOL_TLS_CLIENT enables this by default. Since we call
+            # match_hostname directly disable this explicitly.
+            if hasattr(self.ssl_ctx, "check_hostname"):
+                self.ssl_ctx.check_hostname = False
+            # load_cert_chain and load_verify_locations can fail
+            # if the file contents are invalid.
+            if ssl_certfile is not None:
+                try:
+                    self.ssl_ctx.load_cert_chain(ssl_certfile, ssl_keyfile)
+                except ssl.SSLError:
+                    pass
+            if ssl_ca_certs is not None:
+                try:
+                    self.ssl_ctx.load_verify_locations(ssl_ca_certs)
+                except ssl.SSLError:
+                    pass
+            # PROTOCOL_TLS_CLIENT sets verify_mode to CERT_REQUIRED so
+            # we always have to set this explicitly.
+            if ssl_cert_reqs is not None:
+                self.ssl_ctx.verify_mode = ssl_cert_reqs
+            else:
+                self.ssl_ctx.verify_mode = ssl.CERT_NONE
 
         # Map self._ident.get() -> request socket
         self._tid_to_sock = {}
@@ -191,13 +248,12 @@ class Pool:
         self.pool_id += 1
         self.pid = os.getpid()
 
-        sockets = None
+        # Allocate outside the lock. Triggering a GC while holding the lock
+        # could run Cursor.__del__ and deadlock. See PYTHON-799.
+        new_sockets = set()
+        self.lock.acquire()
         try:
-            # Swapping variables is not atomic. We need to ensure no other
-            # thread is modifying self.sockets, or replacing it, in this
-            # critical section.
-            self.lock.acquire()
-            sockets, self.sockets = self.sockets, set()
+            sockets, self.sockets = self.sockets, new_sockets
         finally:
             self.lock.release()
 
@@ -240,7 +296,9 @@ class Pool:
             try:
                 sock = socket.socket(af, socktype, proto)
                 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-                sock.settimeout(self.conn_timeout or 20.0)
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE,
+                                self.socket_keepalive)
+                sock.settimeout(self.conn_timeout)
                 sock.connect(sa)
                 return sock
             except socket.error, e:
@@ -265,16 +323,15 @@ class Pool:
         sock = self.create_connection()
         hostname = self.pair[0]
 
-        if self.use_ssl:
+        if self.ssl_ctx is not None:
             try:
-                sock = ssl.wrap_socket(sock,
-                                       certfile=self.ssl_certfile,
-                                       keyfile=self.ssl_keyfile,
-                                       ca_certs=self.ssl_ca_certs,
-                                       cert_reqs=self.ssl_cert_reqs)
-                if self.ssl_cert_reqs:
+                sock = self.ssl_ctx.wrap_socket(sock)
+                if self.ssl_ctx.verify_mode and self.ssl_match_hostname:
                     match_hostname(sock.getpeercert(), hostname)
-
+            # CertificateError doesn't inherit from SSLError.
+            except CertificateError:
+                sock.close()
+                raise
             except ssl.SSLError:
                 sock.close()
                 raise ConnectionFailure("SSL handshake failed. MongoDB may "
@@ -528,8 +585,11 @@ class Pool:
         for sock_info in self.sockets:
             sock_info.close()
 
-        for request_sock in self._tid_to_sock.values():
-            if request_sock not in (NO_REQUEST, NO_SOCKET_YET):
+        # Don't use self._tid_to_sock.values(): 2to3 would translate to
+        # list(self._tid_to_sock.values()), but during interpreter shutdown
+        # list() may already be set to None.
+        for request_sock in self._tid_to_sock.itervalues():
+            if request_sock not in (None, -1):
                 request_sock.close()
 
 

pymongo/read_preferences.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2014 MongoDB, Inc.
+# Copyright 2012-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License",
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,13 @@ import random
 from pymongo.errors import ConfigurationError
 
 
+_PRIMARY = 0
+_PRIMARY_PREFERRED = 1
+_SECONDARY = 2
+_SECONDARY_PREFERRED = 3
+_NEAREST = 4
+
+
 class ReadPreference:
     """An enum that defines the read preference modes supported by PyMongo.
     Used in three cases:
@@ -54,20 +61,20 @@ class ReadPreference:
     * `NEAREST`: Queries are distributed among all members.
     """
 
-    PRIMARY = 0
-    PRIMARY_PREFERRED = 1
-    SECONDARY = 2
-    SECONDARY_ONLY = 2
-    SECONDARY_PREFERRED = 3
-    NEAREST = 4
+    PRIMARY = _PRIMARY
+    PRIMARY_PREFERRED = _PRIMARY_PREFERRED
+    SECONDARY = _SECONDARY
+    SECONDARY_ONLY = _SECONDARY
+    SECONDARY_PREFERRED = _SECONDARY_PREFERRED
+    NEAREST = _NEAREST
 
 # For formatting error messages
 modes = {
-    ReadPreference.PRIMARY:             'PRIMARY',
-    ReadPreference.PRIMARY_PREFERRED:   'PRIMARY_PREFERRED',
-    ReadPreference.SECONDARY:           'SECONDARY',
-    ReadPreference.SECONDARY_PREFERRED: 'SECONDARY_PREFERRED',
-    ReadPreference.NEAREST:             'NEAREST',
+    _PRIMARY:             'PRIMARY',
+    _PRIMARY_PREFERRED:   'PRIMARY_PREFERRED',
+    _SECONDARY:           'SECONDARY',
+    _SECONDARY_PREFERRED: 'SECONDARY_PREFERRED',
+    _NEAREST:             'NEAREST',
 }
 
 _mongos_modes = [
@@ -113,43 +120,34 @@ def select_member_with_tags(members, tags, secondary_only, latency):
     fastest = min([candidate.get_avg_ping_time() for candidate in candidates])
     near_candidates = [
         candidate for candidate in candidates
-        if candidate.get_avg_ping_time() - fastest < latency / 1000.]
+        if candidate.get_avg_ping_time() - fastest <= latency / 1000.]
 
     return random.choice(near_candidates)
 
 
-def select_member(
-    members,
-    mode=ReadPreference.PRIMARY,
-    tag_sets=None,
-    latency=15
-):
+def select_member(members,
+                  mode=ReadPreference.PRIMARY,
+                  tag_sets=None,
+                  latency=15):
     """Return a Member or None.
     """
     if tag_sets is None:
         tag_sets = [{}]
 
-    # For brevity
-    PRIMARY             = ReadPreference.PRIMARY
-    PRIMARY_PREFERRED   = ReadPreference.PRIMARY_PREFERRED
-    SECONDARY           = ReadPreference.SECONDARY
-    SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
-    NEAREST             = ReadPreference.NEAREST
-        
-    if mode == PRIMARY:
+    if mode == _PRIMARY:
         if tag_sets != [{}]:
             raise ConfigurationError("PRIMARY cannot be combined with tags")
         return select_primary(members)
 
-    elif mode == PRIMARY_PREFERRED:
+    elif mode == _PRIMARY_PREFERRED:
         # Recurse.
-        candidate_primary = select_member(members, PRIMARY, [{}], latency)
+        candidate_primary = select_member(members, _PRIMARY, [{}], latency)
         if candidate_primary:
             return candidate_primary
         else:
-            return select_member(members, SECONDARY, tag_sets, latency)
+            return select_member(members, _SECONDARY, tag_sets, latency)
 
-    elif mode == SECONDARY:
+    elif mode == _SECONDARY:
         for tags in tag_sets:
             candidate = select_member_with_tags(members, tags, True, latency)
             if candidate:
@@ -157,16 +155,16 @@ def select_member(
 
         return None
 
-    elif mode == SECONDARY_PREFERRED:
+    elif mode == _SECONDARY_PREFERRED:
         # Recurse.
         candidate_secondary = select_member(
-            members, SECONDARY, tag_sets, latency)
+            members, _SECONDARY, tag_sets, latency)
         if candidate_secondary:
             return candidate_secondary
         else:
-            return select_member(members, PRIMARY, [{}], latency)
+            return select_member(members, _PRIMARY, [{}], latency)
 
-    elif mode == NEAREST:
+    elif mode == _NEAREST:
         for tags in tag_sets:
             candidate = select_member_with_tags(members, tags, False, latency)
             if candidate:
@@ -203,3 +201,208 @@ class MovingAverage(object):
 
     def get(self):
         return self.average
+
+
+def _validate_tag_sets(tag_sets):
+    """Validate tag sets for a MongoReplicaSetClient.
+    """
+    if tag_sets is None:
+        return tag_sets
+
+    if not isinstance(tag_sets, list):
+        raise TypeError((
+            "Tag sets %r invalid, must be a list") % (tag_sets,))
+    if len(tag_sets) == 0:
+        raise ValueError((
+            "Tag sets %r invalid, must be None or contain at least one set of"
+            " tags") % (tag_sets,))
+
+    for tags in tag_sets:
+        if not isinstance(tags, dict):
+            raise TypeError(
+                "Tag set %r invalid, must be an instance of dict, or"
+                "bson.son.SON" % (tags,))
+
+    return tag_sets
+
+
+class _ServerMode(object):
+    """Base class for all read preferences.
+    """
+
+    __slots__ = ("__mongos_mode", "__mode", "__tag_sets")
+
+    def __init__(self, mode, tag_sets=None):
+        if mode == _PRIMARY and tag_sets is not None:
+            raise ConfigurationError("Read preference primary "
+                                     "cannot be combined with tags")
+        self.__mongos_mode = _mongos_modes[mode]
+        self.__mode = mode
+        self.__tag_sets = _validate_tag_sets(tag_sets)
+
+    @property
+    def name(self):
+        """The name of this read preference.
+        """
+        return self.__class__.__name__
+
+    @property
+    def document(self):
+        """Read preference as a document.
+        """
+        if self.__tag_sets in (None, [{}]):
+            return {'mode': self.__mongos_mode}
+        return {'mode': self.__mongos_mode, 'tags': self.__tag_sets}
+
+    @property
+    def mode(self):
+        """The mode of this read preference instance.
+        """
+        return self.__mode
+
+    @property
+    def tag_sets(self):
+        """Set ``tag_sets`` to a list of dictionaries like [{'dc': 'ny'}] to
+        read only from members whose ``dc`` tag has the value ``"ny"``.
+        To specify a priority-order for tag sets, provide a list of
+        tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
+        set, ``{}``, means "read from any member that matches the mode,
+        ignoring tags." MongoReplicaSetClient tries each set of tags in turn
+        until it finds a set of tags with at least one matching member.
+
+        .. seealso:: `Data-Center Awareness
+           <http://www.mongodb.org/display/DOCS/Data+Center+Awareness>`_
+        """
+        if self.__tag_sets:
+            return list(self.__tag_sets)
+        return [{}]
+
+    def __repr__(self):
+        return "%s(tag_sets=%r)" % (
+            self.name, self.__tag_sets)
+
+    def __eq__(self, other):
+        if isinstance(other, _ServerMode):
+            return (self.mode == other.mode and
+                    self.tag_sets == other.tag_sets)
+        raise NotImplementedError
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __getstate__(self):
+        """Return value of object for pickling.
+        Needed explicitly because __slots__() defined.
+        """
+        return {'mode': self.__mode, 'tag_sets': self.__tag_sets}
+
+    def __setstate__(self, value):
+        """Restore from pickling."""
+        self.__mode = value['mode']
+        self.__mongos_mode = _mongos_modes[self.__mode]
+        self.__tag_sets = _validate_tag_sets(value['tag_sets'])
+
+
+class Primary(_ServerMode):
+    """Primary read preference.
+
+    * When directly connected to one mongod queries are allowed if the server
+      is standalone or a replica set primary.
+    * When connected to a mongos queries are sent to the primary of a shard.
+    * When connected to a replica set queries are sent to the primary of
+      the replica set.
+
+    .. versionadded:: 2.9
+    """
+
+    def __init__(self):
+        super(Primary, self).__init__(_PRIMARY)
+
+    def __repr__(self):
+        return "Primary()"
+
+    def __eq__(self, other):
+        if isinstance(other, _ServerMode):
+            return other.mode == _PRIMARY
+        raise NotImplementedError
+
+
+class PrimaryPreferred(_ServerMode):
+    """PrimaryPreferred read preference.
+
+    * When directly connected to one mongod queries are allowed to standalone
+      servers, to a replica set primary, or to replica set secondaries.
+    * When connected to a mongos queries are sent to the primary of a shard if
+      available, otherwise a shard secondary.
+    * When connected to a replica set queries are sent to the primary if
+      available, otherwise a secondary.
+
+    :Parameters:
+      - `tag_sets`: The :attr:`~tag_sets` to use if the primary is not
+        available.
+
+    .. versionadded:: 2.9
+    """
+
+    def __init__(self, tag_sets=None):
+        super(PrimaryPreferred, self).__init__(_PRIMARY_PREFERRED, tag_sets)
+
+
+class Secondary(_ServerMode):
+    """Secondary read preference.
+
+    * When directly connected to one mongod queries are allowed to standalone
+      servers, to a replica set primary, or to replica set secondaries.
+    * When connected to a mongos queries are distributed among shard
+      secondaries. An error is raised if no secondaries are available.
+    * When connected to a replica set queries are distributed among
+      secondaries. An error is raised if no secondaries are available.
+
+    :Parameters:
+      - `tag_sets`: The :attr:`~tag_sets` to use with this read_preference
+
+    .. versionadded:: 2.9
+    """
+
+    def __init__(self, tag_sets=None):
+        super(Secondary, self).__init__(_SECONDARY, tag_sets)
+
+
+class SecondaryPreferred(_ServerMode):
+    """SecondaryPreferred read preference.
+
+    * When directly connected to one mongod queries are allowed to standalone
+      servers, to a replica set primary, or to replica set secondaries.
+    * When connected to a mongos queries are distributed among shard
+      secondaries, or the shard primary if no secondary is available.
+    * When connected to a replica set queries are distributed among
+      secondaries, or the primary if no secondary is available.
+
+    :Parameters:
+      - `tag_sets`: The :attr:`~tag_sets` to use with this read_preference
+
+    .. versionadded:: 2.9
+    """
+
+    def __init__(self, tag_sets=None):
+        super(SecondaryPreferred, self).__init__(_SECONDARY_PREFERRED, tag_sets)
+
+
+class Nearest(_ServerMode):
+    """Nearest read preference.
+
+    * When directly connected to one mongod queries are allowed to standalone
+      servers, to a replica set primary, or to replica set secondaries.
+    * When connected to a mongos queries are distributed among all members of
+      a shard.
+    * When connected to a replica set queries are distributed among all
+      members.
+
+    :Parameters:
+      - `tag_sets`: The :attr:`~tag_sets` to use with this read_preference
+
+    .. versionadded:: 2.9
+    """
+
+    def __init__(self, tag_sets=None):
+        super(Nearest, self).__init__(_NEAREST, tag_sets)

pymongo/replica_set_connection.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2014 MongoDB, Inc.
+# Copyright 2011-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -106,31 +106,25 @@ class ReplicaSetConnection(MongoReplicaSetClient):
             is no timeout. If both `network_timeout` and `socketTimeoutMS` are
             specified `network_timeout` takes precedence, matching
             connection.Connection.
-          - `socketTimeoutMS`: (integer) How long (in milliseconds) a send or
-            receive on a socket can take before timing out. Defaults to ``None``
-            (no timeout).
-          - `connectTimeoutMS`: (integer) How long (in milliseconds) a
+          - `socketTimeoutMS`: (integer or None) How long (in milliseconds) a
+            send or receive on a socket can take before timing out. Defaults
+            to ``None`` (no timeout).
+          - `connectTimeoutMS`: (integer or None) How long (in milliseconds) a
             connection can take to be opened before timing out. Defaults to
             ``20000``.
-          - `waitQueueTimeoutMS`: (integer) How long (in milliseconds) a
-            thread will wait for a socket from the pool if the pool has no
+          - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds)
+            a thread will wait for a socket from the pool if the pool has no
             free sockets. Defaults to ``None`` (no timeout).
-          - `waitQueueMultiple`: (integer) Multiplied by max_pool_size to give
-            the number of threads allowed to wait for a socket at one time.
-            Defaults to ``None`` (no waiters).
+          - `waitQueueMultiple`: (integer or None) Multiplied by max_pool_size
+            to give the number of threads allowed to wait for a socket at one
+            time. Defaults to ``None`` (no waiters).
+          - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive
+            packets on connected sockets. Defaults to ``False`` (do not send
+            keep-alive packets).
           - `auto_start_request`: If ``True`` (the default), each thread that
             accesses this :class:`ReplicaSetConnection` has a socket allocated
-            to it for the thread's lifetime, for each member of the set. For
-            :class:`~pymongo.read_preferences.ReadPreference` PRIMARY,
-            auto_start_request=True ensures consistent reads, even if you read
-            after an unsafe write. For read preferences other than PRIMARY,
-            there are no consistency guarantees.
-          - `use_greenlets`: if ``True``, use a background Greenlet instead of
-            a background thread to monitor state of replica set. Additionally,
-            :meth:`start_request()` will ensure that the current greenlet uses
-            the same socket for all operations until :meth:`end_request()`.
-            `use_greenlets` with ReplicaSetConnection requires `Gevent
-            <http://gevent.org/>`_ to be installed.
+            to it for each member of the set until the thread calls
+            :meth:`end_request` or terminates.
 
           | **Write Concern options:**
 
@@ -162,29 +156,28 @@ class ReplicaSetConnection(MongoReplicaSetClient):
 
           - `slave_okay` or `slaveOk` (deprecated): Use `read_preference`
             instead.
-          - `read_preference`: The read preference for this connection.
-            See :class:`~pymongo.read_preferences.ReadPreference` for available
-          - `tag_sets`: Read from replica-set members with these tags.
-            To specify a priority-order for tag sets, provide a list of
-            tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag
-            set, ``{}``, means "read from any member that matches the mode,
-            ignoring tags." :class:`MongoReplicaSetClient` tries each set of
-            tags in turn until it finds a set of tags with at least one matching
-            member.
-          - `secondary_acceptable_latency_ms`: (integer) Any replica-set member
-            whose ping time is within secondary_acceptable_latency_ms of the
+          - `read_preference`: The read preference for this client.
+            See :mod:`~pymongo.read_preferences` for available
+            options. Defaults to ``ReadPreference.PRIMARY``.
+          - `localThresholdMS`: (integer) Any replica-set member
+            whose ping time is within localThresholdMS of the
             nearest member may accept reads. Default 15 milliseconds.
             **Ignored by mongos** and must be configured on the command line.
             See the localThreshold_ option for more information.
 
           | **SSL configuration:**
 
+            See :doc:`/examples/tls` for examples.
+
           - `ssl`: If ``True``, create the connection to the servers using SSL.
+            Defaults to ``False``.
           - `ssl_keyfile`: The private keyfile used to identify the local
             connection against mongod.  If included with the ``certfile` then
             only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
+            Defaults to ``None``.
           - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``ssl=True``.
+            connection against mongod. Implies ``ssl=True``. Defaults to
+            ``None``.
           - `ssl_cert_reqs`: Specifies whether a certificate is required from
             the other side of the connection, and whether it will be validated
             if provided. It must be one of the three values ``ssl.CERT_NONE``
@@ -192,11 +185,12 @@ class ReplicaSetConnection(MongoReplicaSetClient):
             (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
             (required and validated). If the value of this parameter is not
             ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
-            to a file of CA certificates. Implies ``ssl=True``.
+            to a file of CA certificates. Implies ``ssl=True``. Defaults to
+            ``ssl.CERT_NONE``.
           - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
-            Implies ``ssl=True``.
+            Implies ``ssl=True``. Defaults to ``None``.
 
         .. versionchanged:: 2.5
            Added additional ssl options
@@ -207,8 +201,6 @@ class ReplicaSetConnection(MongoReplicaSetClient):
            Added support for `host`, `port`, and `network_timeout` keyword
            arguments for compatibility with connection.Connection.
         .. versionadded:: 2.1
-
-        .. _localThreshold: http://docs.mongodb.org/manual/reference/mongos/#cmdoption-mongos--localThreshold
         """
         network_timeout = kwargs.pop('network_timeout', None)
         if network_timeout is not None:

pymongo/results.py

@@ -0,0 +1,224 @@
+# Copyright 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Result class definitions."""
+
+from pymongo.errors import InvalidOperation
+
+
+class _WriteResult(object):
+    """Base class for write result classes."""
+
+    def __init__(self, acknowledged):
+        self.__acknowledged = acknowledged
+
+    def _raise_if_unacknowledged(self, property_name):
+        """Raise an exception on property access if unacknowledged."""
+        if not self.__acknowledged:
+            raise InvalidOperation("A value for %s is not available when "
+                                   "the write is unacknowledged. Check the "
+                                   "acknowledged attribute to avoid this "
+                                   "error." % (property_name,))
+
+    @property
+    def acknowledged(self):
+        """Is this the result of an acknowledged write operation?
+
+        The :attr:`acknowledged` attribute will be ``False`` when using
+        ``WriteConcern(w=0)``, otherwise ``True``.
+
+        .. note::
+          If the :attr:`acknowledged` attribute is ``False`` all other
+          attibutes of this class will raise
+          :class:`~pymongo.errors.InvalidOperation` when accessed. Values for
+          other attributes cannot be determined if the write operation was
+          unacknowledged.
+
+        .. seealso::
+          :class:`~pymongo.write_concern.WriteConcern`
+        """
+        return self.__acknowledged
+
+
+class InsertOneResult(_WriteResult):
+    """The return type for :meth:`~pymongo.collection.Collection.insert_one`.
+    """
+
+    __slots__ = ("__inserted_id", "__acknowledged")
+
+    def __init__(self, inserted_id, acknowledged):
+        self.__inserted_id = inserted_id
+        super(InsertOneResult, self).__init__(acknowledged)
+
+    @property
+    def inserted_id(self):
+        """The inserted document's _id."""
+        return self.__inserted_id
+
+
+class InsertManyResult(_WriteResult):
+    """The return type for :meth:`~pymongo.collection.Collection.insert_many`.
+    """
+
+    __slots__ = ("__inserted_ids", "__acknowledged")
+
+    def __init__(self, inserted_ids, acknowledged):
+        self.__inserted_ids = inserted_ids
+        super(InsertManyResult, self).__init__(acknowledged)
+
+    @property
+    def inserted_ids(self):
+        """A list of _ids of the inserted documents, in the order provided.
+
+        .. note:: If ``False`` is passed for the `ordered` parameter to
+          :meth:`~pymongo.collection.Collection.insert_many` the server
+          may have inserted the documents in a different order than what
+          is presented here.
+        """
+        return self.__inserted_ids
+
+
+class UpdateResult(_WriteResult):
+    """The return type for :meth:`~pymongo.collection.Collection.update_one`,
+    :meth:`~pymongo.collection.Collection.update_many`, and
+    :meth:`~pymongo.collection.Collection.replace_one`.
+    """
+
+    __slots__ = ("__raw_result", "__acknowledged")
+
+    def __init__(self, raw_result, acknowledged):
+        self.__raw_result = raw_result
+        super(UpdateResult, self).__init__(acknowledged)
+
+    @property
+    def raw_result(self):
+        """The raw result document returned by the server."""
+        return self.__raw_result
+
+    @property
+    def matched_count(self):
+        """The number of documents matched for this update."""
+        self._raise_if_unacknowledged("matched_count")
+        if self.upserted_id is not None:
+            return 0
+        return self.__raw_result.get("n", 0)
+
+    @property
+    def modified_count(self):
+        """The number of documents modified.
+
+        .. note:: modified_count is only reported by MongoDB 2.6 and later.
+          When connected to an earlier server version, or in certain mixed
+          version sharding configurations, this attribute will be set to
+          ``None``.
+        """
+        self._raise_if_unacknowledged("modified_count")
+        return self.__raw_result.get("nModified")
+
+    @property
+    def upserted_id(self):
+        """The _id of the inserted document if an upsert took place. Otherwise
+        ``None``.
+        """
+        self._raise_if_unacknowledged("upserted_id")
+        return self.__raw_result.get("upserted")
+
+
+class DeleteResult(_WriteResult):
+    """The return type for :meth:`~pymongo.collection.Collection.delete_one`
+    and :meth:`~pymongo.collection.Collection.delete_many`"""
+
+    __slots__ = ("__raw_result", "__acknowledged")
+
+    def __init__(self, raw_result, acknowledged):
+        self.__raw_result = raw_result
+        super(DeleteResult, self).__init__(acknowledged)
+
+    @property
+    def raw_result(self):
+        """The raw result document returned by the server."""
+        return self.__raw_result
+
+    @property
+    def deleted_count(self):
+        """The number of documents deleted."""
+        self._raise_if_unacknowledged("deleted_count")
+        return self.__raw_result.get("n", 0)
+
+
+class BulkWriteResult(_WriteResult):
+    """An object wrapper for bulk API write results."""
+
+    __slots__ = ("__bulk_api_result", "__acknowledged")
+
+    def __init__(self, bulk_api_result, acknowledged):
+        """Create a BulkWriteResult instance.
+
+        :Parameters:
+          - `bulk_api_result`: A result dict from the bulk API
+          - `acknowledged`: Was this write result acknowledged? If ``False``
+            then all properties of this object will raise
+            :exc:`~pymongo.errors.InvalidOperation`.
+        """
+        self.__bulk_api_result = bulk_api_result
+        super(BulkWriteResult, self).__init__(acknowledged)
+
+    @property
+    def bulk_api_result(self):
+        """The raw bulk API result."""
+        return self.__bulk_api_result
+
+    @property
+    def inserted_count(self):
+        """The number of documents inserted."""
+        self._raise_if_unacknowledged("inserted_count")
+        return self.__bulk_api_result.get("nInserted")
+
+    @property
+    def matched_count(self):
+        """The number of documents matched for an update."""
+        self._raise_if_unacknowledged("matched_count")
+        return self.__bulk_api_result.get("nMatched")
+
+    @property
+    def modified_count(self):
+        """The number of documents modified.
+
+        .. note:: modified_count is only reported by MongoDB 2.6 and later.
+          When connected to an earlier server version, or in certain mixed
+          version sharding configurations, this attribute will be set to
+          ``None``.
+        """
+        self._raise_if_unacknowledged("modified_count")
+        return self.__bulk_api_result.get("nModified")
+
+    @property
+    def deleted_count(self):
+        """The number of documents deleted."""
+        self._raise_if_unacknowledged("deleted_count")
+        return self.__bulk_api_result.get("nRemoved")
+
+    @property
+    def upserted_count(self):
+        """The number of documents upserted."""
+        self._raise_if_unacknowledged("upserted_count")
+        return self.__bulk_api_result.get("nUpserted")
+
+    @property
+    def upserted_ids(self):
+        """A map of operation index to the _id of the upserted document."""
+        self._raise_if_unacknowledged("upserted_ids")
+        if self.__bulk_api_result:
+            return dict((upsert["index"], upsert["_id"])
+                        for upsert in self.bulk_api_result["upserted"])

pymongo/son_manipulator.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2014 MongoDB, Inc.
+# Copyright 2009-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

pymongo/ssl_context.py

@@ -0,0 +1,99 @@
+# Copyright 2014-2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""A fake SSLContext implementation."""
+
+try:
+    import ssl
+    _CERT_NONE = ssl.CERT_NONE
+except ImportError:
+    _CERT_NONE = None
+
+
+class SSLContext(object):
+    """A fake SSLContext.
+
+    This implements an API similar to ssl.SSLContext from python 3.2
+    but does not implement methods or properties that would be
+    incompatible with ssl.wrap_socket from python 2.6.
+
+    You must pass protocol which must be one of the PROTOCOL_* constants
+    defined in the ssl module. ssl.PROTOCOL_SSLv23 is recommended for maximum
+    interoperability.
+    """
+
+    __slots__ = ('_cafile', '_certfile',
+                 '_keyfile', '_protocol', '_verify_mode')
+
+    def __init__(self, protocol):
+        self._cafile = None
+        self._certfile = None
+        self._keyfile = None
+        self._protocol = protocol
+        self._verify_mode = _CERT_NONE
+
+    @property
+    def protocol(self):
+        """The protocol version chosen when constructing the context.
+        This attribute is read-only.
+        """
+        return self._protocol
+
+    def __get_verify_mode(self):
+        """Whether to try to verify other peers' certificates and how to
+        behave if verification fails. This attribute must be one of
+        ssl.CERT_NONE, ssl.CERT_OPTIONAL or ssl.CERT_REQUIRED.
+        """
+        return self._verify_mode
+
+    def __set_verify_mode(self, value):
+        """Setter for verify_mode."""
+        self._verify_mode = value
+
+    verify_mode = property(__get_verify_mode, __set_verify_mode)
+
+    def load_cert_chain(self, certfile, keyfile=None):
+        """Load a private key and the corresponding certificate. The certfile
+        string must be the path to a single file in PEM format containing the
+        certificate as well as any number of CA certificates needed to
+        establish the certificate's authenticity. The keyfile string, if
+        present, must point to a file containing the private key. Otherwise
+        the private key will be taken from certfile as well.
+        """
+        self._certfile = certfile
+        self._keyfile = keyfile
+
+    def load_verify_locations(self, cafile=None, dummy=None):
+        """Load a set of "certification authority"(CA) certificates used to
+        validate other peers' certificates when `~verify_mode` is other than
+        ssl.CERT_NONE.
+        """
+        self._cafile = cafile
+
+    # suppress_ragged_eofs (dummy0) is not supported in the ssl module from pypi
+    # ciphers (dummy1) is not supported in CPython < 2.7.9 / 3.2
+    def wrap_socket(self, sock, server_side=False,
+                    do_handshake_on_connect=True,
+                    dummy0=None, dummy1=None):
+        """Wrap an existing Python socket sock and return an ssl.SSLSocket
+        object.
+        """
+        return ssl.wrap_socket(sock, keyfile=self._keyfile,
+                               certfile=self._certfile,
+                               server_side=server_side,
+                               cert_reqs=self._verify_mode,
+                               ssl_version=self._protocol,
+                               ca_certs=self._cafile,
+                               do_handshake_on_connect=do_handshake_on_connect)
+

pymongo/thread_util.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2014 MongoDB, Inc.
+# Copyright 2012-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

pymongo/uri_parser.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2014 MongoDB, Inc.
+# Copyright 2011-2015 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -170,17 +170,18 @@ def _parse_options(opts, delim):
         else:
             options[key] = val
     if 'readpreferencetags' in options:
-        new_tag_sets = []
+        tag_sets = []
         for tag_set in options['readpreferencetags']:
-            tag_dict = {}
+            if tag_set == '':
+                tag_sets.append({})
+                continue
             try:
-                for tag in tag_set.split(","):
-                    tag_parts = tag.split(":")
-                    tag_dict[tag_parts[0]] = tag_parts[1]
-                new_tag_sets.append(tag_dict)
-            except IndexError:
-                new_tag_sets.append({})
-        options['readpreferencetags'] = new_tag_sets
+                tag_sets.append(dict([tag.split(":")
+                                      for tag in tag_set.split(",")]))
+            except Exception:
+                raise ValueError("%s not a valid value "
+                                 "for readpreferencetags" % (tag_set,))
+        options['readpreferencetags'] = tag_sets
     return options
 
 

pymongo/write_concern.py

@@ -0,0 +1,109 @@
+# Copyright 2014-2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tools for working with write concerns."""
+
+from pymongo.errors import ConfigurationError
+
+class WriteConcern(object):
+    """WriteConcern backport from PyMongo 3.x
+
+    :Parameters:
+        - `w`: (integer or string) Used with replication, write operations
+          will block until they have been replicated to the specified number
+          or tagged set of servers. `w=<integer>` always includes the replica
+          set primary (e.g. w=3 means write to the primary and wait until
+          replicated to **two** secondaries). **w=0 disables acknowledgement
+          of write operations and can not be used with other write concern
+          options.**
+        - `wtimeout`: (integer) Used in conjunction with `w`. Specify a value
+          in milliseconds to control how long to wait for write propagation
+          to complete. If replication does not complete in the given
+          timeframe, a timeout exception is raised.
+        - `j`: If ``True`` block until write operations have been committed
+          to the journal. Cannot be used in combination with `fsync`. Prior
+          to MongoDB 2.6 this option was ignored if the server was running
+          without journaling. Starting with MongoDB 2.6 write operations will
+          fail with an exception if this option is used when the server is
+          running without journaling.
+        - `fsync`: If ``True`` and the server is running without journaling,
+          blocks until the server has synced all data files to disk. If the
+          server is running with journaling, this acts the same as the `j`
+          option, blocking until write operations have been committed to the
+          journal. Cannot be used in combination with `j`.
+
+    .. versionadded:: 2.9
+    """
+
+    __slots__ = ("__document", "__acknowledged")
+
+    def __init__(self, w=None, wtimeout=None, j=None, fsync=None):
+        self.__document = {}
+        self.__acknowledged = True
+
+        if wtimeout is not None:
+            if not isinstance(wtimeout, (int, long)):
+                raise TypeError("wtimeout must be an integer")
+            self.__document["wtimeout"] = wtimeout
+
+        if j is not None:
+            if not isinstance(j, bool):
+                raise TypeError("j must be True or False")
+            self.__document["j"] = j
+
+        if fsync is not None:
+            if not isinstance(fsync, bool):
+                raise TypeError("fsync must be True or False")
+            if j and fsync:
+                raise ConfigurationError("Can't set both j "
+                                         "and fsync at the same time")
+            self.__document["fsync"] = fsync
+
+        if self.__document and w == 0:
+            raise ConfigurationError("Can not use w value "
+                                     "of 0 with other options")
+        if w is not None:
+            if isinstance(w, (int, long)):
+                self.__acknowledged = w > 0
+            elif not isinstance(w, basestring):
+                raise TypeError("w must be an integer or string")
+            self.__document["w"] = w
+
+    @property
+    def document(self):
+        """The document representation of this write concern.
+
+        .. note::
+          :class:`WriteConcern` is immutable. Mutating the value of
+          :attr:`document` does not mutate this :class:`WriteConcern`.
+        """
+        return self.__document.copy()
+
+    @property
+    def acknowledged(self):
+        """If ``True`` write operations will wait for acknowledgement before
+        returning.
+        """
+        return self.__acknowledged
+
+    def __repr__(self):
+        return ("WriteConcern(%s)" % (
+            ", ".join("%s=%s" % kvt for kvt in self.document.items()),))
+
+    def __eq__(self, other):
+        return self.document == other.document
+
+    def __ne__(self, other):
+        return self.document != other.document
+

setup.py

@@ -2,7 +2,6 @@ import glob
 import os
 import platform
 import re
-import subprocess
 import sys
 import warnings
 
@@ -22,18 +21,25 @@ except ImportError:
 # we have to.
 try:
     from setuptools import setup
+    from setuptools.command.build_py import build_py
 except ImportError:
     from ez_setup import use_setuptools
     use_setuptools()
     from setuptools import setup
+    from setuptools.command.build_py import build_py
 
-from distutils.cmd import Command
 from distutils.command.build_ext import build_ext
 from distutils.errors import CCompilerError
 from distutils.errors import DistutilsPlatformError, DistutilsExecError
 from distutils.core import Extension
 
-version = "2.7"
+try:
+    import sphinx
+    _HAVE_SPHINX = True
+except ImportError:
+    _HAVE_SPHINX = False
+
+version = "2.9.5"
 
 f = open("README.rst")
 try:
@@ -88,27 +94,75 @@ if "test" in sys.argv or "nosetests" in sys.argv:
     should_run_tests = True
 
 
-class doc(Command):
+class doc(build_py):
 
     description = "generate or test documentation"
 
-    user_options = [("test", "t",
-                     "run doctests instead of generating documentation")]
+    build_py.user_options.append(
+        ("test", "t", "run doctests instead of generating documentation"))
 
-    boolean_options = ["test"]
+    build_py.boolean_options.append('test')
 
     def initialize_options(self):
         self.test = False
-
-    def finalize_options(self):
-        pass
+        build_py.initialize_options(self)
 
     def run(self):
+        if not _HAVE_SPHINX:
+            raise RuntimeError(
+                "You must install Sphinx to build or test the documentation.")
+
+        if PY3:
+            import doctest
+            from doctest import OutputChecker as _OutputChecker
+
+            # Match u or U (possibly followed by r or R), removing it.
+            # r/R can follow u/U but not precede it. Don't match the
+            # single character string 'u' or 'U'.
+            _u_literal_re = re.compile(
+                r"(\W|^)(?<![\'\"])[uU]([rR]?[\'\"])", re.UNICODE)
+            # Match b or B (possibly followed by r or R), removing.
+            # r/R can follow b/B but not precede it. Don't match the
+            # single character string 'b' or 'B'.
+            _b_literal_re = re.compile(
+                r"(\W|^)(?<![\'\"])[bB]([rR]?[\'\"])", re.UNICODE)
+
+            class _StringPrefixFixer(_OutputChecker):
+
+                def check_output(self, want, got, optionflags):
+                    # The docstrings are written with python 2.x in mind.
+                    # To make the doctests pass in python 3 we have to
+                    # strip the 'u' prefix from the expected results. The
+                    # actual results won't have that prefix.
+                    want = re.sub(_u_literal_re, r'\1\2', want)
+                    # We also have to strip the 'b' prefix from the actual
+                    # results since python 2.x expected results won't have
+                    # that prefix.
+                    got = re.sub(_b_literal_re, r'\1\2', got)
+                    return super(
+                        _StringPrefixFixer, self).check_output(
+                            want, got, optionflags)
+
+                def output_difference(self, example, got, optionflags):
+                    example.want = re.sub(
+                        _u_literal_re, r'\1\2', example.want)
+                    got = re.sub(_b_literal_re, r'\1\2', got)
+                    return super(
+                        _StringPrefixFixer, self).output_difference(
+                            example, got, optionflags)
+
+            doctest.OutputChecker = _StringPrefixFixer
+
+            # No need to run build_py for python 2.x.
+            build_py.run(self)
+
         if self.test:
-            path = "doc/_build/doctest"
+            path = os.path.join(
+                os.path.abspath('.'), "doc", "_build", "doctest")
             mode = "doctest"
         else:
-            path = "doc/_build/%s" % version
+            path = os.path.join(
+                os.path.abspath('.'), "doc", "_build", version)
             mode = "html"
 
             try:
@@ -116,8 +170,15 @@ class doc(Command):
             except:
                 pass
 
-        status = subprocess.call(["sphinx-build", "-E",
-                                  "-b", mode, "doc", path])
+        sphinx_args = ["-E", "-b", mode, "doc", path]
+
+        # sphinx.main calls sys.exit when sphinx.build_main exists.
+        # Call build_main directly so we can check status and print
+        # the full path to the built docs.
+        if hasattr(sphinx, 'build_main'):
+            status = sphinx.build_main(sphinx_args)
+        else:
+            status = sphinx.main(sphinx_args)
 
         if status:
             raise RuntimeError("documentation step '%s' failed" % (mode,))
@@ -135,6 +196,18 @@ else:
     build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 
 
+_COMPILER_ATTRS = (
+    'compiler', 'compiler_so', 'compiler_cxx', 'linker_exe', 'linker_so')
+
+
+# From distutils.cygwinccompiler in recent pythons.
+def _is_cygwingcc():
+    out = os.popen('gcc -dumpmachine', 'r')
+    out_string = out.read()
+    out.close()
+    return out_string.strip().endswith('cygwin')
+
+
 class custom_build_ext(build_ext):
     """Allow C extension building to fail.
 
@@ -203,6 +276,31 @@ http://api.mongodb.org/python/current/installation.html#osx
         write_nose_config()
 
     def build_extension(self, ext):
+        # http://bugs.python.org/issue12641
+        # This makes a number of well researched assumptions about distutils
+        # but is written in a defensive style to guard against those
+        # assumptions failing.
+        compiler = getattr(self, "compiler", None)
+        if compiler and getattr(compiler, "compiler_type", None) == "mingw32":
+            try:
+                from distutils import cygwinccompiler
+            except ImportError:
+                pass
+            else:
+                # If cygwinccompiler.is_cygwingcc exists the problem is
+                # already solved for us.
+                if not hasattr(cygwinccompiler, "is_cygwingcc"):
+                    gcc_version = getattr(compiler, "gcc_version", None)
+                    # If gcc_version is None assume we need to strip
+                    # -mno-cygwin.
+                    if (not _is_cygwingcc() and
+                            (not gcc_version or gcc_version >= "4.6")):
+                        for att in [getattr(compiler, attrname)
+                                    for attrname in _COMPILER_ATTRS
+                                    if hasattr(compiler, attrname)]:
+                            if isinstance(att, list) and '-mno-cygwin' in att:
+                                att.remove('-mno-cygwin')
+
         name = ext.name
         if sys.version_info[:3] >= (2, 4, 0):
             try:
@@ -262,6 +360,19 @@ Performance may be degraded.\n
 else:
     extra_opts['ext_modules'] = ext_modules
 
+# The nosetests command requires nose be available, otherwise the xunit
+# plugin will not work.
+# Note: projects listed in setup_requires will NOT be automatically installed
+# on the system where the setup script is being run. They are simply
+# downloaded to the ./.eggs directory if they're not locally available
+# already.
+# See:
+# https://nose.readthedocs.io/en/latest/api/commands.html#bootstrapping
+# https://nose.readthedocs.io/en/latest/setuptools_integration.html
+# https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
+if "nosetests" in sys.argv:
+    extra_opts["setup_requires"] = ["nose"]
+
 if PY3:
     extra_opts["use_2to3"] = True
     if should_run_tests:
@@ -317,6 +428,8 @@ setup(
         "Programming Language :: Python :: 3.2",
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: Jython",
         "Programming Language :: Python :: Implementation :: PyPy",