BUMP 3.7.2

(cherry picked from commit d43ca118f9)

.evergreen/config.yml

@@ -45,7 +45,7 @@ functions:
               CURRENT_VERSION=latest
            fi
 
-           export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
+           export DRIVERS_TOOLS="$(dirname $(pwd))/drivers-tools"
            export PROJECT_DIRECTORY="$(pwd)"
 
            # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
@@ -182,7 +182,7 @@ functions:
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file:  ${PROJECT_DIRECTORY}/doc/html/index.html
+        local_file:  src/doc/html/index.html
         remote_file: ${UPLOAD_BUCKET}/docs/${CURRENT_VERSION}/index.html
         bucket: mciuploads
         permissions: public-read
@@ -194,7 +194,7 @@ functions:
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file:  ${PROJECT_DIRECTORY}/.coverage
+        local_file:  src/.coverage
         optional: true
         # Upload the coverage report for all tasks in a single build to the same directory.
         remote_file: ${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
@@ -238,7 +238,7 @@ functions:
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file:  ${PROJECT_DIRECTORY}/htmlcov/index.html
+        local_file:  src/htmlcov/index.html
         remote_file: ${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/htmlcov/index.html
         bucket: mciuploads
         permissions: public-read
@@ -267,7 +267,7 @@ functions:
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file:  ${PROJECT_DIRECTORY}/scan.html
+        local_file:  src/scan.html
         remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html
         bucket: mciuploads
         permissions: public-read
@@ -294,7 +294,7 @@ functions:
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log
+        local_file: drivers-tools/.evergreen/orchestration/server.log
         remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log
         bucket: mciuploads
         permissions: public-read
@@ -870,11 +870,20 @@ axes:
         run_on: rhel70-small
         batchtime: 10080  # 7 days
 
+  # OSes that support versions of MongoDB>=2.6 and <4.0 with SSL.
+  - id: os-no-40-plus
+    display_name: OS
+    values:
       - id: debian71-test
         display_name: "Debian 7.1"
         run_on: debian71-test
         batchtime: 10080  # 7 days
 
+      - id: ubuntu-12.04
+        display_name: "Ubuntu 12.04"
+        run_on: ubuntu1204-test
+        batchtime: 10080  # 7 days
+
   # OSes that support versions of MongoDB without SSL.
   - id: os-nossl
     display_name: OS
@@ -1061,18 +1070,18 @@ axes:
   - id: mod-wsgi-version
     display_name: "mod_wsgi version"
     values:
-      - id: "2.8"
+      - id: "2"
         display_name: "mod_wsgi 2.8"
         variables:
-          MOD_WSGI_VERSION: "2.8"
+          MOD_WSGI_VERSION: "2"
-      - id: "3.5"
+      - id: "3"
         display_name: "mod_wsgi 3.5"
         variables:
-          MOD_WSGI_VERSION: "3.5"
+          MOD_WSGI_VERSION: "3"
-      - id: "4.5.20"
+      - id: "4"
-        display_name: "mod_wsgi 4.5.20"
+        display_name: "mod_wsgi 4.x"
         variables:
-          MOD_WSGI_VERSION: "4.5.20"
+          MOD_WSGI_VERSION: "4"
   - id: green-framework
     display_name: "Green Framework"
     values:
@@ -1164,6 +1173,16 @@ buildvariants:
      - ".3.0"
      - ".2.6"
 
+- matrix_name: "tests-no-40-plus"
+  matrix_spec: {"os-no-40-plus": "*", auth-ssl: "*"}
+  display_name: "${os-no-40-plus} ${auth-ssl}"
+  tasks:
+     - ".3.6"
+     - ".3.4"
+     - ".3.2"
+     - ".3.0"
+     - ".2.6"
+
 - matrix_name: "tests-nossl"
   matrix_spec: {"os-nossl": "*", auth: "*", ssl: "nossl"}
   display_name: "${os-nossl} ${auth} ${ssl}"
@@ -1457,7 +1476,7 @@ buildvariants:
   exclude_spec:
     # mod_wsgi 2.8 segfaults with the toolchain python 2.7, regardless of distro
     python-version: ["2.7", "3.4", "3.6"]
-    mod-wsgi-version: ["2.8"]
+    mod-wsgi-version: ["2"]
   display_name: "${mod-wsgi-version} ${python-version}"
   run_on: rhel62-small
   tasks:

README.rst

@@ -16,7 +16,7 @@ is a `gridfs
 <http://www.mongodb.org/display/DOCS/GridFS+Specification>`_
 implementation on top of ``pymongo``.
 
-PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, and 3.6.
+PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, 3.6 and 4.0.
 
 Support / Feedback
 ==================

bson/__init__.py

@@ -819,7 +819,7 @@ if _USE_C:
 def _millis_to_datetime(millis, opts):
     """Convert milliseconds since epoch UTC to datetime."""
     diff = ((millis % 1000) + 1000) % 1000
-    seconds = (millis - diff) / 1000
+    seconds = (millis - diff) // 1000
     micros = diff * 1000
     if opts.tz_aware:
         dt = EPOCH_AWARE + datetime.timedelta(seconds=seconds,
@@ -837,7 +837,7 @@ def _datetime_to_millis(dtm):
     if dtm.utcoffset() is not None:
         dtm = dtm - dtm.utcoffset()
     return int(calendar.timegm(dtm.timetuple()) * 1000 +
-               dtm.microsecond / 1000)
+               dtm.microsecond // 1000)
 
 
 _CODEC_OPTIONS_TYPE_ERROR = TypeError(

bson/binary.py

@@ -198,9 +198,9 @@ class UUIDLegacy(Binary):
       ...                          CodecOptions(uuid_representation=STANDARD))
       >>> coll.insert_one({'uuid': Binary(my_uuid.bytes, 3)}).inserted_id
       ObjectId('...')
-      >>> coll.find({'uuid': my_uuid}).count()
+      >>> coll.count_documents({'uuid': my_uuid})
       0
-      >>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count()
+      >>> coll.count_documents({'uuid': UUIDLegacy(my_uuid)})
       1
       >>> coll.find({'uuid': UUIDLegacy(my_uuid)})[0]['uuid']
       UUID('...')
@@ -209,9 +209,9 @@ class UUIDLegacy(Binary):
       >>> doc = coll.find_one({'uuid': UUIDLegacy(my_uuid)})
       >>> coll.replace_one({"_id": doc["_id"]}, doc).matched_count
       1
-      >>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count()
+      >>> coll.count_documents({'uuid': UUIDLegacy(my_uuid)})
       0
-      >>> coll.find({'uuid': {'$in': [UUIDLegacy(my_uuid), my_uuid]}}).count()
+      >>> coll.count_documents({'uuid': {'$in': [UUIDLegacy(my_uuid), my_uuid]}})
       1
       >>> coll.find_one({'uuid': my_uuid})['uuid']
       UUID('...')

doc/changelog.rst

@@ -1,6 +1,50 @@
 Changelog
 =========
 
+Changes in Version 3.7.2
+------------------------
+
+Version 3.7.2 fixes a few issues discovered since the release of 3.7.1.
+
+- Fixed a bug in retryable writes where a previous command's "txnNumber"
+  field could be sent leading to incorrect results.
+- Fixed a memory leak of a few bytes on some insert, update, or delete
+  commands when running against MongoDB 3.6+.
+- Fixed a bug that caused :meth:`pymongo.collection.Collection.ensure_index`
+  to only cache a single index per database.
+- Updated the documentation examples to use
+  :meth:`pymongo.collection.Collection.count_documents` instead of
+  :meth:`pymongo.collection.Collection.count` and
+  :meth:`pymongo.cursor.Cursor.count`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 3.7.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 3.7.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=21519
+
+Changes in Version 3.7.1
+------------------------
+
+Version 3.7.1 fixes a few issues discovered since the release of 3.7.0.
+
+- Calling :meth:`~pymongo.database.Database.authenticate` more than once
+  with the same credentials results in OperationFailure.
+- Authentication fails when SCRAM-SHA-1 is used to authenticate users with
+  only MONGODB-CR credentials.
+- A millisecond rounding problem when decoding datetimes in the pure Python
+  BSON decoder on 32 bit systems and AWS lambda.
+
+Issues Resolved
+...............
+
+See the `PyMongo 3.7.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 3.7.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=21096
+
 Changes in Version 3.7.0
 ------------------------
 

doc/examples/bulk.rst

@@ -29,7 +29,7 @@ bulk insert operations.
   >>> db = pymongo.MongoClient().bulk_example
   >>> db.test.insert_many([{'i': i} for i in range(10000)]).inserted_ids
   [...]
-  >>> db.test.count()
+  >>> db.test.count_documents({})
   10000
 
 Mixed Bulk Write Operations

doc/tutorial.rst

@@ -333,20 +333,20 @@ author is "Mike":
 Counting
 --------
 If we just want to know how many documents match a query we can
-perform a :meth:`~pymongo.cursor.Cursor.count` operation instead of a
+perform a :meth:`~pymongo.collection.Collection.count_documents` operation
-full query. We can get a count of all of the documents in a
+instead of a full query. We can get a count of all of the documents
-collection:
+in a collection:
 
 .. doctest::
 
-  >>> posts.count()
+  >>> posts.count_documents({})
   3
 
 or just of those documents that match a specific query:
 
 .. doctest::
 
-  >>> posts.find({"author": "Mike"}).count()
+  >>> posts.count_documents({"author": "Mike"})
   2
 
 Range Queries

pymongo/__init__.py

@@ -64,7 +64,7 @@ SLOW_ONLY = 1
 ALL = 2
 """Profile all operations."""
 
-version_tuple = (3, 7, 0)
+version_tuple = (3, 7, 2)
 
 def get_version_string():
     if isinstance(version_tuple[-1], str):

pymongo/_cmessagemodule.c

@@ -762,6 +762,7 @@ encodefail:
     Py_XDECREF(iterator);
     buffer_free(buffer);
 bufferfail:
+    PyMem_Free(identifier);
     destroy_codec_options(&options);
     return result;
 }

pymongo/auth.py

@@ -61,9 +61,26 @@ MECHANISMS = frozenset(
 class _Cache(object):
     __slots__ = ("data",)
 
+    _hash_val = hash('_Cache')
+
     def __init__(self):
         self.data = None
 
+    def __eq__(self, other):
+        # Two instances must always compare equal.
+        if isinstance(other, _Cache):
+            return True
+        return NotImplemented
+
+    def __ne__(self, other):
+        if isinstance(other, _Cache):
+            return False
+        return NotImplemented
+
+    def __hash__(self):
+        return self._hash_val
+
+
 
 MongoCredential = namedtuple(
     'MongoCredential',
@@ -255,15 +272,19 @@ def _authenticate_scram(credentials, sock_info, mechanism):
         raise OperationFailure("Server returned an invalid nonce.")
 
     without_proof = b"c=biws,r=" + rnonce
-    keys = cache.data
+    if cache.data:
-    if keys:
+        client_key, server_key, csalt, citerations = cache.data
-        client_key, server_key = keys
     else:
+        client_key, server_key, csalt, citerations = None, None, None, None
+
+    # Salt and / or iterations could change for a number of different
+    # reasons. Either changing invalidates the cache.
+    if not client_key or salt != csalt or iterations != citerations:
         salted_pass = _hi(
             digest, data, standard_b64decode(salt), iterations)
         client_key = _hmac(salted_pass, b"Client Key", digestmod).digest()
         server_key = _hmac(salted_pass, b"Server Key", digestmod).digest()
-        cache.data = (client_key, server_key)
+        cache.data = (client_key, server_key, salt, iterations)
     stored_key = digestmod(client_key).digest()
     auth_msg = b",".join((first_bare, server_first, without_proof))
     client_sig = _hmac(stored_key, auth_msg, digestmod).digest()

pymongo/bulk.py

@@ -160,6 +160,7 @@ class _Bulk(object):
         self.uses_array_filters = False
         self.is_retryable = True
         self.retrying = False
+        self.started_retryable_write = False
         # Extra state so that we know where to pick up on a retry attempt.
         self.current_run = None
 
@@ -275,6 +276,11 @@ class _Bulk(object):
 
             while run.idx_offset < len(run.ops):
                 if session:
+                    # Start a new retryable write unless one was already
+                    # started for this command.
+                    if retryable and not self.started_retryable_write:
+                        session._start_retryable_write()
+                        self.started_retryable_write = True
                     session._apply_to(cmd, retryable, ReadPreference.PRIMARY)
                 sock_info.send_cluster_time(cmd, session, client)
                 check_keys = run.op_type == _INSERT
@@ -300,6 +306,8 @@ class _Bulk(object):
                 _merge_command(run, full_result, run.idx_offset, result)
                 # We're no longer in a retry once a command succeeds.
                 self.retrying = False
+                self.started_retryable_write = False
+
                 if self.ordered and "writeErrors" in result:
                     break
                 run.idx_offset += len(to_send)

pymongo/client_session.py

@@ -367,7 +367,7 @@ class ClientSession(object):
         self._transaction.opts = TransactionOptions(
             read_concern, write_concern, read_preference)
         self._transaction.state = _TxnState.STARTING
-        self._server_session._transaction_id += 1
+        self._start_retryable_write()
         self._transaction.transaction_id = self._server_session.transaction_id
         return _TransactionContext(self)
 
@@ -544,7 +544,6 @@ class ClientSession(object):
             self._transaction.state = _TxnState.NONE
 
         if is_retryable:
-            self._server_session._transaction_id += 1
             command['txnNumber'] = self._server_session.transaction_id
             return
 
@@ -574,9 +573,9 @@ class ClientSession(object):
             command['txnNumber'] = self._server_session.transaction_id
             command['autocommit'] = False
 
-    def _retry_transaction_id(self):
+    def _start_retryable_write(self):
         self._check_ended()
-        self._server_session.retry_transaction_id()
+        self._server_session.inc_transaction_id()
 
 
 class _ServerSession(object):
@@ -597,8 +596,8 @@ class _ServerSession(object):
         """Positive 64-bit integer."""
         return Int64(self._transaction_id)
 
-    def retry_transaction_id(self):
+    def inc_transaction_id(self):
-        self._transaction_id -= 1
+        self._transaction_id += 1
 
 
 class _ServerSessionPool(collections.deque):

pymongo/collection.py

@@ -647,7 +647,7 @@ class Collection(common.BaseObject):
                    session=None):
         """Insert a single document.
 
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           0
           >>> result = db.test.insert_one({'x': 1})
           >>> result.inserted_id
@@ -697,12 +697,12 @@ class Collection(common.BaseObject):
                     bypass_document_validation=False, session=None):
         """Insert an iterable of documents.
 
-          >>> db.test.count()
+          >>> db.test.count_documents({})
           0
           >>> result = db.test.insert_many([{'x': i} for i in range(2)])
           >>> result.inserted_ids
           [ObjectId('54f113fffba522406c9cc20e'), ObjectId('54f113fffba522406c9cc20f')]
-          >>> db.test.count()
+          >>> db.test.count_documents({})
           2
 
         :Parameters:
@@ -1156,12 +1156,12 @@ class Collection(common.BaseObject):
     def delete_one(self, filter, collation=None, session=None):
         """Delete a single document matching the filter.
 
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           3
           >>> result = db.test.delete_one({'x': 1})
           >>> result.deleted_count
           1
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           2
 
         :Parameters:
@@ -1194,12 +1194,12 @@ class Collection(common.BaseObject):
     def delete_many(self, filter, collation=None, session=None):
         """Delete one or more documents matching the filter.
 
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           3
           >>> result = db.test.delete_many({'x': 1})
           >>> result.deleted_count
           3
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           0
 
         :Parameters:
@@ -2875,11 +2875,11 @@ class Collection(common.BaseObject):
                             projection=None, sort=None, session=None, **kwargs):
         """Finds a single document and deletes it, returning the document.
 
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           2
           >>> db.test.find_one_and_delete({'x': 1})
           {u'x': 1, u'_id': ObjectId('54f4e12bfba5220aa4d6dee8')}
-          >>> db.test.count({'x': 1})
+          >>> db.test.count_documents({'x': 1})
           1
 
         If multiple documents match *filter*, a *sort* can be applied.

pymongo/mongo_client.py

@@ -636,7 +636,7 @@ class MongoClient(common.BaseObject):
         expire = datetime.timedelta(seconds=cache_for) + now
 
         with self.__index_cache_lock:
-            if database not in self.__index_cache:
+            if dbname not in self.__index_cache:
                 self.__index_cache[dbname] = {}
                 self.__index_cache[dbname][collection] = {}
                 self.__index_cache[dbname][collection][index] = expire
@@ -1176,6 +1176,14 @@ class MongoClient(common.BaseObject):
 
         def is_retrying():
             return bulk.retrying if bulk else retrying
+        # Increment the transaction id up front to ensure any retry attempt
+        # will use the proper txnNumber, even if server or socket selection
+        # fails before the command can be sent.
+        if retryable:
+            session._start_retryable_write()
+            if bulk:
+                bulk.started_retryable_write = True
+
         while True:
             try:
                 server = self._get_topology().select_server(
@@ -1190,9 +1198,6 @@ class MongoClient(common.BaseObject):
                             # not support sessions raise the last error.
                             raise last_error
                         retryable = False
-                    if is_retrying():
-                        # Reset the transaction id and retry the operation.
-                        session._retry_transaction_id()
                     return func(session, sock_info, retryable)
             except ServerSelectionTimeoutError:
                 if is_retrying():

setup.py

@@ -34,7 +34,7 @@ except ImportError:
     except ImportError:
         _HAVE_SPHINX = False
 
-version = "3.7.0"
+version = "3.7.2"
 
 f = open("README.rst")
 try:

test/__init__.py

@@ -599,6 +599,13 @@ class ClientContext(object):
         new_func = self.require_version_min(4, 0, 0, -1)(func)
         return self.require_replica_set(new_func)
 
+    @property
+    def supports_reindex(self):
+        """Does the connected server support reindex?"""
+        return not (
+            self.version.at_least(4, 1, 0) and
+            self.is_mongos)
+
 
 # Reusable client context
 client_context = ClientContext()

test/test_auth.py

@@ -77,6 +77,31 @@ class AutoAuthenticateThread(threading.Thread):
         self.success = True
 
 
+class DBAuthenticateThread(threading.Thread):
+    """Used in testing threaded authentication.
+
+    This does db.test.find_one() with a 1-second delay to ensure it must
+    check out and authenticate multiple sockets from the pool concurrently.
+
+    :Parameters:
+      `db`: An auth-protected db with a 'test' collection containing one
+      document.
+    """
+
+    def __init__(self, db, username, password):
+        super(DBAuthenticateThread, self).__init__()
+        self.db = db
+        self.username = username
+        self.password = password
+        self.success = False
+
+    def run(self):
+        self.db.authenticate(self.username, self.password)
+        assert self.db.test.find_one({'$where': delay(1)}) is not None
+        self.success = True
+
+
+
 class TestGSSAPI(unittest.TestCase):
 
     @classmethod
@@ -559,11 +584,14 @@ class TestSCRAM(unittest.TestCase):
         credentials = all_credentials.get('admin')
         cache = credentials.cache
         self.assertIsNotNone(cache)
-        keys = cache.data
+        data = cache.data
-        self.assertIsNotNone(keys)
+        self.assertIsNotNone(data)
-        self.assertEqual(len(keys), 2)
+        self.assertEqual(len(data), 4)
-        for elt in keys:
+        ckey, skey, salt, iterations = data
-            self.assertIsInstance(elt, bytes)
+        self.assertIsInstance(ckey, bytes)
+        self.assertIsInstance(skey, bytes)
+        self.assertIsInstance(salt, bytes)
+        self.assertIsInstance(iterations, int)
 
         pool = next(iter(client._topology._servers.values()))._pool
         with pool.get_socket(all_credentials) as sock_info:
@@ -576,7 +604,7 @@ class TestSCRAM(unittest.TestCase):
         sock_credentials = next(iter(authset))
         sock_cache = sock_credentials.cache
         self.assertIsNotNone(sock_cache)
-        self.assertEqual(sock_cache.data, keys)
+        self.assertEqual(sock_cache.data, data)
 
     def test_scram_threaded(self):
 
@@ -595,6 +623,38 @@ class TestSCRAM(unittest.TestCase):
             thread.join()
             self.assertTrue(thread.success)
 
+class TestThreadedAuth(unittest.TestCase):
+
+    @client_context.require_auth
+    def test_db_authenticate_threaded(self):
+
+        db = client_context.client.db
+        coll = db.test
+        coll.drop()
+        coll.insert_one({'_id': 1})
+
+        client_context.create_user(
+            'db',
+            'user',
+            'pass',
+            roles=['dbOwner'])
+        self.addCleanup(db.command, 'dropUser', 'user')
+
+        db = rs_or_single_client_noauth().db
+        db.authenticate('user', 'pass')
+        # No error.
+        db.authenticate('user', 'pass')
+
+        db = rs_or_single_client_noauth().db
+        threads = []
+        for _ in range(4):
+            threads.append(DBAuthenticateThread(db, 'user', 'pass'))
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+            self.assertTrue(thread.success)
+
 
 class TestAuthURIOptions(unittest.TestCase):
 

test/test_bson.py

@@ -474,6 +474,13 @@ class TestBSON(unittest.TestCase):
         dt2 = BSON.encode({"date": dt1}).decode()["date"]
         self.assertEqual(dt1, dt2)
 
+    def test_large_datetime_truncation(self):
+        # Ensure that a large datetime is truncated correctly.
+        dt1 = datetime.datetime(9999, 1, 1, 1, 1, 1, 999999)
+        dt2 = BSON.encode({"date": dt1}).decode()["date"]
+        self.assertEqual(dt2.microsecond, 999000)
+        self.assertEqual(dt2.second, dt1.second)
+
     def test_aware_datetime(self):
         aware = datetime.datetime(1993, 4, 4, 2,
                                   tzinfo=FixedOffset(555, "SomeZone"))

test/test_collection.py

@@ -332,6 +332,9 @@ class TestCollection(IntegrationTest):
                                       mode="off")
 
     def test_reindex(self):
+        if not client_context.supports_reindex:
+            raise unittest.SkipTest(
+                "reindex is no longer supported by mongos 4.1+")
         db = self.db
         db.drop_collection("test")
         db.test.insert_one({"foo": "bar", "who": "what", "when": "how"})

test/test_database.py

@@ -434,13 +434,8 @@ class TestDatabase(IntegrationTest):
         first = db.command("buildinfo")
         second = db.command({"buildinfo": 1})
         third = db.command("buildinfo", 1)
-        # The logicalTime and operationTime fields were introduced in MongoDB
+        self.assertEqualReply(first, second)
-        # 3.5. Their value can change from one command call to the next.
+        self.assertEqualReply(second, third)
-        for doc in (first, second, third):
-            doc.pop("logicalTime", None)
-            doc.pop("operationTime", None)
-        self.assertEqual(first, second)
-        self.assertEqual(second, third)
 
     # We use 'aggregate' as our example command, since it's an easy way to
     # retrieve a BSON regex from a collection using a command. But until
@@ -721,6 +716,7 @@ class TestDatabase(IntegrationTest):
                                         projection={"_id": False}))
 
     @client_context.require_no_auth
+    @client_context.require_version_max(4, 1, 0)
     def test_eval(self):
         db = self.client.pymongo_test
         db.test.drop()
@@ -824,6 +820,7 @@ class TestDatabase(IntegrationTest):
         self.assertFalse(db.test.find_one())
 
     @client_context.require_no_auth
+    @client_context.require_version_max(4, 1, 0)
     def test_system_js(self):
         db = self.client.pymongo_test
         db.system.js.delete_many({})

test/test_retryable_writes.py

@@ -14,6 +14,7 @@
 
 """Test retryable writes."""
 
+import copy
 import json
 import os
 import sys
@@ -169,7 +170,7 @@ def create_tests():
 create_tests()
 
 
-def retryable_single_statement_ops(coll):
+def _retryable_single_statement_ops(coll):
     return [
         (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}),
         (coll.bulk_write, [[InsertOne({}),
@@ -188,6 +189,11 @@ def retryable_single_statement_ops(coll):
         (coll.find_one_and_replace, [{}, {'a': 3}], {}),
         (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}),
         (coll.find_one_and_delete, [{}, {}], {}),
+    ]
+
+
+def retryable_single_statement_ops(coll):
+    return _retryable_single_statement_ops(coll) + [
         # Deprecated methods.
         # Insert with single or multiple documents.
         (coll.insert, [{}], {}),
@@ -500,5 +506,46 @@ class TestRetryableWrites(IgnoreDeprecationsTest):
         self.assertEqual(coll.find_one(projection={'_id': True}), {'_id': 1})
 
 
+# TODO: Make this a real integration test where we stepdown the primary.
+class TestRetryableWritesTxnNumber(IgnoreDeprecationsTest):
+    @client_context.require_version_min(3, 6)
+    @client_context.require_replica_set
+    def test_increment_transaction_id_without_sending_command(self):
+        """Test that the txnNumber field is properly incremented, even when
+        the first attempt fails before sending the command.
+        """
+        listener = OvertCommandListener()
+        client = rs_or_single_client(
+            retryWrites=True, event_listeners=[listener])
+        topology = client._topology
+        select_server = topology.select_server
+
+        def raise_connection_err_select_server(*args, **kwargs):
+            # Raise ConnectionFailure on the first attempt and perform
+            # normal selection on the retry attempt.
+            topology.select_server = select_server
+            raise ConnectionFailure('Connection refused')
+
+        for method, args, kwargs in _retryable_single_statement_ops(
+                client.db.retryable_write_test):
+            listener.results.clear()
+            topology.select_server = raise_connection_err_select_server
+            with client.start_session() as session:
+                kwargs = copy.deepcopy(kwargs)
+                kwargs['session'] = session
+                msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs)
+                initial_txn_id = session._server_session.transaction_id
+
+                # Each operation should fail on the first attempt and succeed
+                # on the second.
+                method(*args, **kwargs)
+                self.assertEqual(len(listener.results['started']), 1, msg)
+                retry_cmd = listener.results['started'][0].command
+                sent_txn_id = retry_cmd['txnNumber']
+                final_txn_id = session._server_session.transaction_id
+                self.assertEqual(Int64(initial_txn_id + 1), sent_txn_id, msg)
+                self.assertEqual(sent_txn_id, final_txn_id, msg)
+
+
 if __name__ == '__main__':
     unittest.main()

test/test_session.py

@@ -27,7 +27,6 @@ from pymongo.errors import (ConfigurationError,
                             OperationFailure)
 from pymongo.monotonic import time as _time
 from pymongo.read_concern import ReadConcern
-from pymongo.write_concern import WriteConcern
 from test import IntegrationTest, client_context, db_user, db_pwd, unittest, SkipTest
 from test.utils import ignore_deprecations, rs_or_single_client, EventListener
 
@@ -321,12 +320,15 @@ class TestSession(IntegrationTest):
             (coll.count, [], {}),
             (coll.count_documents, [{}], {}),
             (coll.inline_map_reduce, ['function() {}', 'function() {}'], {}),
-            (coll.reindex, [], {}),
             (coll.list_indexes, [], {}),
             (coll.index_information, [], {}),
             (coll.options, [], {}),
             (coll.aggregate, [[]], {}),
         ])
+
+        if client_context.supports_reindex:
+            ops.append((coll.reindex, [], {}))
+
         self._test_ops(client, *ops)
 
     @client_context.require_no_mongos
@@ -922,8 +924,10 @@ class TestCausalConsistency(unittest.TestCase):
             lambda coll, session: coll.drop_index("foo_1", session=session))
         self._test_writes(
             lambda coll, session: coll.drop_indexes(session=session))
-        self._test_writes(
+
-            lambda coll, session: coll.reindex(session=session))
+        if client_context.supports_reindex:
+            self._test_writes(
+                lambda coll, session: coll.reindex(session=session))
 
     def _test_no_read_concern(self, op):
         coll = self.client.pymongo_test.test
@@ -977,8 +981,6 @@ class TestCausalConsistency(unittest.TestCase):
             lambda coll, session: coll.drop_index("foo_1", session=session))
         self._test_no_read_concern(
             lambda coll, session: coll.drop_indexes(session=session))
-        self._test_no_read_concern(
-            lambda coll, session: coll.reindex(session=session))
         self._test_no_read_concern(
                 lambda coll, session: list(
                     coll.aggregate([{"$out": "aggout"}], session=session)))
@@ -993,6 +995,10 @@ class TestCausalConsistency(unittest.TestCase):
         self._test_no_read_concern(
             lambda coll, session: coll.find({}, session=session).explain())
 
+        if client_context.supports_reindex:
+            self._test_no_read_concern(
+                lambda coll, session: coll.reindex(session=session))
+
     @client_context.require_no_standalone
     def test_get_more_does_not_include_read_concern(self):
         coll = self.client.pymongo_test.test

test/utils.py

@@ -480,7 +480,7 @@ def run_threads(collection, target):
         t.start()
 
     for t in threads:
-        t.join(30)
+        t.join(60)
         assert not t.isAlive()