BUMP 3.12.4.dev0

(cherry picked from commit 70f7fe7542)

.evergreen/build-mac.sh

@@ -8,7 +8,7 @@ rm -rf validdist
 mkdir -p validdist
 mv dist/* validdist || true
 
-for VERSION in 2.7 3.4 3.5 3.6 3.7 3.8 3.9; do
+for VERSION in 2.7 3.4 3.5 3.6 3.7 3.8 3.9 3.10; do
     if [[ $VERSION == "2.7" ]]; then
         PYTHON=/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python
         rm -rf build

.evergreen/build-manylinux-internal.sh

@@ -11,7 +11,7 @@ mv dist/* validdist || true
 
 # Compile wheels
 for PYTHON in /opt/python/*/bin/python; do
-    if [[ ! $PYTHON =~ (cp27|cp34|cp35|cp36|cp37|cp38|cp39) ]]; then
+    if [[ ! $PYTHON =~ (cp27|cp34|cp35|cp36|cp37|cp38|cp39|cp310) ]]; then
         continue
     fi
     # https://github.com/pypa/manylinux/issues/49
@@ -19,9 +19,9 @@ for PYTHON in /opt/python/*/bin/python; do
     $PYTHON setup.py bdist_wheel
     rm -rf build
 
-    # Audit wheels and write multilinux tag
+    # Audit wheels and write manylinux tag
     for whl in dist/*.whl; do
-        # Skip already built manylinux1 wheels.
+        # Skip already built manylinux wheels.
         if [[ "$whl" != *"manylinux"* ]]; then
             auditwheel repair $whl -w dist
             rm $whl

.evergreen/build-manylinux.sh

@@ -2,16 +2,32 @@
 
 docker version
 
-# 2020-03-20-2fda31c Was the last release to include Python 3.4.
-images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \
-        quay.io/pypa/manylinux1_i686:2020-03-20-2fda31c \
-        quay.io/pypa/manylinux1_x86_64 \
-        quay.io/pypa/manylinux1_i686 \
-        quay.io/pypa/manylinux2014_x86_64 \
-        quay.io/pypa/manylinux2014_i686 \
-        quay.io/pypa/manylinux2014_aarch64 \
-        quay.io/pypa/manylinux2014_ppc64le \
-        quay.io/pypa/manylinux2014_s390x)
+# manylinux1 2021-05-05-b64d921 and manylinux2014 2021-05-05-1ac6ef3 were
+# the last releases to generate pip < 20.3 compatible wheels. After that
+# auditwheel was upgraded to v4 which produces PEP 600 manylinux_x_y wheels
+# which requires pip >= 20.3. We use the older docker image to support older
+# pip versions.
+BUILD_WITH_TAG="$1"
+if [ -n "$BUILD_WITH_TAG" ]; then
+  # 2020-03-20-2fda31c Was the last release to include Python 3.4.
+  images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \
+          quay.io/pypa/manylinux1_i686:2020-03-20-2fda31c \
+          quay.io/pypa/manylinux1_x86_64:2021-05-05-b64d921 \
+          quay.io/pypa/manylinux1_i686:2021-05-05-b64d921 \
+          quay.io/pypa/manylinux2014_x86_64:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_i686:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_aarch64:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_ppc64le:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_s390x:2021-05-05-1ac6ef3)
+else
+  images=(quay.io/pypa/manylinux1_x86_64 \
+          quay.io/pypa/manylinux1_i686 \
+          quay.io/pypa/manylinux2014_x86_64 \
+          quay.io/pypa/manylinux2014_i686 \
+          quay.io/pypa/manylinux2014_aarch64 \
+          quay.io/pypa/manylinux2014_ppc64le \
+          quay.io/pypa/manylinux2014_s390x)
+fi
 
 for image in "${images[@]}"; do
   docker pull $image
@@ -28,7 +44,8 @@ unexpected=$(find dist \! \( -iname dist -or \
                              -iname '*cp36*' -or \
                              -iname '*cp37*' -or \
                              -iname '*cp38*' -or \
-                             -iname '*cp39*' \))
+                             -iname '*cp39*' -or \
+                             -iname '*cp310*' \))
 if [ -n "$unexpected" ]; then
   echo "Unexpected files:" $unexpected
   exit 1

.evergreen/build-windows.sh

@@ -8,7 +8,7 @@ rm -rf validdist
 mkdir -p validdist
 mv dist/* validdist || true
 
-for VERSION in 27 34 35 36 37 38 39; do
+for VERSION in 27 34 35 36 37 38 39 310; do
     _pythons=(C:/Python/Python${VERSION}/python.exe \
               C:/Python/32/Python${VERSION}/python.exe)
     for PYTHON in "${_pythons[@]}"; do

.evergreen/install-dependencies.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 

.evergreen/perf.yml

@@ -96,7 +96,7 @@ functions:
             # If this was a patch build, doing a fresh clone would not actually test the patch
             cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS
           else
-            git clone git://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
+            git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
           fi
           echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config
 

.evergreen/run-atlas-tests.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 
-# Don't trace to avoid secrets showing up in the logs
+# Exit on error and enable trace.
 set -o errexit
+set -o xtrace
 
 export JAVA_HOME=/opt/java/jdk8
 
+# Attempt to find system pip before creating a virtualenv
+PIP=$(command -v pip2 || command -v pip)
+
 if [ -z "$PYTHON_BINARY" ]; then
     echo "No python binary specified"
     PYTHON_BINARY=$(command -v python || command -v python3) || true
@@ -15,27 +19,41 @@ if [ -z "$PYTHON_BINARY" ]; then
 fi
 
 IMPL=$(${PYTHON_BINARY} -c "import platform, sys; sys.stdout.write(platform.python_implementation())")
-if [ $IMPL = "Jython" -o $IMPL = "PyPy" ]; then
-    echo "Using Jython or PyPy"
+
+if [ $IMPL = "Jython" ]; then
+    # The venv created by createvirtualenv is incompatible with Jython
     $PYTHON_BINARY -m virtualenv --never-download --no-wheel atlastest
     . atlastest/bin/activate
-    trap "deactivate; rm -rf atlastest" EXIT HUP
-    pip install certifi
-    PYTHON=python
 else
-    IS_PRE_279=$(${PYTHON_BINARY} -c "import sys; sys.stdout.write('1' if sys.version_info < (2, 7, 9) else '0')")
+    # All other pythons work with createvirtualenv.
+    . .evergreen/utils.sh
+    createvirtualenv $PYTHON_BINARY atlastest
+fi
+trap "deactivate; rm -rf atlastest" EXIT HUP
+
+if [ $IMPL = "Jython" ]; then
+    echo "Using Jython"
+    $PIP download certifi
+    python -m pip install --no-index -f file://$(pwd) certifi
+elif [ $IMPL = "PyPy" ]; then
+    echo "Using PyPy"
+    python -m pip install certifi
+else
+    IS_PRE_279=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (2, 7, 9) else '0')")
     if [ $IS_PRE_279 = "1" ]; then
         echo "Using a Pre-2.7.9 CPython"
-        $PYTHON_BINARY -m virtualenv --never-download --no-wheel atlastest
-        . atlastest/bin/activate
-        trap "deactivate; rm -rf atlastest" EXIT HUP
-        pip install pyopenssl>=17.2.0 service_identity>18.1.0
-        PYTHON=python
+        python -m pip install -r .evergreen/test-pyopenssl-requirements.txt
     else
         echo "Using CPython 2.7.9+"
-        PYTHON=$PYTHON_BINARY
     fi
 fi
 
-echo "Running tests"
-$PYTHON test/atlas/test_connection.py
+echo "Running tests without dnspython"
+python test/atlas/test_connection.py
+
+# dnspython is incompatible with Jython so don't test that combination.
+if [ $IMPL != "Jython" ]; then
+    python -m pip install dnspython
+    echo "Running tests with dnspython"
+    MUST_TEST_SRV="1" python test/atlas/test_connection.py
+fi

.evergreen/run-mod-wsgi-tests.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -o xtrace
 set -o errexit
 

.evergreen/run-mongodb-aws-test.sh

@@ -17,6 +17,8 @@ echo "Running MONGODB-AWS authentication tests"
 # ensure no secrets are printed in log files
 set +x
 
+. .evergreen/utils.sh
+
 # load the script
 shopt -s expand_aliases # needed for `urlencode` alias
 [ -s "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" ] && source "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
@@ -39,7 +41,12 @@ fi
 # show test output
 set -x
 
-VIRTUALENV=$(command -v virtualenv)
+# Workaround macOS python 3.9 incompatibility with system virtualenv.
+if [ $(uname -s) = "Darwin" ]; then
+    VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.9/bin/python3 -m virtualenv"
+else
+    VIRTUALENV=$(command -v virtualenv)
+fi
 
 authtest () {
     if [ "Windows_NT" = "$OS" ]; then
@@ -49,13 +56,8 @@ authtest () {
     echo "Running MONGODB-AWS authentication tests with $PYTHON"
     $PYTHON --version
 
-    $VIRTUALENV -p $PYTHON --system-site-packages --never-download venvaws
-    if [ "Windows_NT" = "$OS" ]; then
-      . venvaws/Scripts/activate
-    else
-      . venvaws/bin/activate
-    fi
-    pip install '.[aws]'
+    createvirtualenv $PYTHON venvaws
+    python -m pip install '.[aws]'
     python test/auth_aws/test_auth_aws.py
     deactivate
     rm -rf venvaws

.evergreen/run-ocsp-tests.sh

@@ -3,6 +3,9 @@
 set -o xtrace
 set -o errexit
 
+# For createvirtualenv.
+. .evergreen/utils.sh
+
 if [ -z "$PYTHON_BINARY" ]; then
     echo "No python binary specified"
     PYTHON=$(command -v python || command -v python3) || true
@@ -14,36 +17,9 @@ else
     PYTHON="$PYTHON_BINARY"
 fi
 
-if $PYTHON -m virtualenv --version; then
-    VIRTUALENV="$PYTHON -m virtualenv"
-elif command -v virtualenv; then
-    # We can remove this fallback after:
-    # https://github.com/10gen/mongo-python-toolchain/issues/8
-    VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-else
-    echo "Cannot test without virtualenv"
-    exit 1
-fi
-
-$VIRTUALENV --never-download --no-wheel ocsptest
-if [ "Windows_NT" = "$OS" ]; then
-  . ocsptest/Scripts/activate
-else
-  . ocsptest/bin/activate
-fi
+createvirtualenv $PYTHON ocsptest
 trap "deactivate; rm -rf ocsptest" EXIT HUP
 
-IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
-if [ $IS_PYTHON_2 = "1" ]; then
-    echo "Using a Python 2"
-    # Upgrade pip to install the cryptography wheel and not the tar.
-    # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-    python -m pip install --upgrade 'pip<20.1'
-    # Upgrade setuptools because cryptography requires 18.5+.
-    # <45 because 45.0 dropped support for 2.7.
-    python -m pip install --upgrade 'setuptools<45'
-fi
-
-python -m pip install pyopenssl requests service_identity
+python -m pip install --prefer-binary -r .evergreen/test-pyopenssl-requirements.txt
 
 OCSP_TLS_SHOULD_SUCCEED=${OCSP_TLS_SHOULD_SUCCEED} CA_FILE=${CA_FILE} python test/ocsp/test_ocsp.py

.evergreen/run-tests.sh

@@ -6,6 +6,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #  AUTH               Set to enable authentication. Defaults to "noauth"
 #  SSL                Set to enable SSL. Defaults to "nossl"
 #  PYTHON_BINARY      The Python version to use. Defaults to whatever is available
+#  PYTHON3_BINARY     Path to a working Python 3.5+ binary.
 #  GREEN_FRAMEWORK    The green framework to test with, if any.
 #  C_EXTENSIONS       Pass --no_ext to setup.py, or not.
 #  COVERAGE           If non-empty, run the test suite with coverage.
@@ -19,32 +20,52 @@ else
     set +x
 fi
 
-
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
 PYTHON_BINARY=${PYTHON_BINARY:-}
+PYTHON3_BINARY=${PYTHON3_BINARY:-python3}
 GREEN_FRAMEWORK=${GREEN_FRAMEWORK:-}
 C_EXTENSIONS=${C_EXTENSIONS:-}
 COVERAGE=${COVERAGE:-}
 COMPRESSORS=${COMPRESSORS:-}
+MONGODB_API_VERSION=${MONGODB_API_VERSION:-}
 TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
 LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-}
 SETDEFAULTENCODING=${SETDEFAULTENCODING:-}
+DATA_LAKE=${DATA_LAKE:-}
 
 if [ -n "$COMPRESSORS" ]; then
     export COMPRESSORS=$COMPRESSORS
 fi
 
+if [ -n "$MONGODB_API_VERSION" ]; then
+    export MONGODB_API_VERSION=$MONGODB_API_VERSION
+fi
+
+
 export JAVA_HOME=/opt/java/jdk8
 
 if [ "$AUTH" != "noauth" ]; then
-    export DB_USER="bob"
-    export DB_PASSWORD="pwd123"
+    if [ ! -z "$DATA_LAKE" ]; then
+        export DB_USER="mhuser"
+        export DB_PASSWORD="pencil"
+    elif [ ! -z "$TEST_SERVERLESS" ]; then
+        export DB_USER=$SERVERLESS_ATLAS_USER
+        export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
+    else
+        export DB_USER="bob"
+        export DB_PASSWORD="pwd123"
+    fi
 fi
 
 if [ "$SSL" != "nossl" ]; then
     export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
     export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
+
+    if [ -n "$TEST_LOADBALANCER" ]; then
+        export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}&tls=true"
+        export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}&tls=true"
+    fi
 fi
 
 # For createvirtualenv.
@@ -59,7 +80,7 @@ if [ -z "$PYTHON_BINARY" ]; then
             exit 1
         fi
     else
-        $VIRTUALENV pymongotestvenv
+        $VIRTUALENV --never-download pymongotestvenv
         . pymongotestvenv/bin/activate
         PYTHON=python
         trap "deactivate; rm -rf pymongotestvenv" EXIT HUP
@@ -94,38 +115,11 @@ fi
 
 # PyOpenSSL test setup.
 if [ -n "$TEST_PYOPENSSL" ]; then
-    if $PYTHON -m virtualenv --version; then
-        VIRTUALENV="$PYTHON -m virtualenv"
-    elif command -v virtualenv; then
-        # We can remove this fallback after:
-        # https://github.com/10gen/mongo-python-toolchain/issues/8
-        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-    else
-        echo "Cannot test without virtualenv"
-        exit 1
-    fi
-
-    $VIRTUALENV pyopenssltest
-    if [ "Windows_NT" = "$OS" ]; then
-        . pyopenssltest/Scripts/activate
-    else
-        . pyopenssltest/bin/activate
-    fi
+    createvirtualenv $PYTHON pyopenssltest
     trap "deactivate; rm -rf pyopenssltest" EXIT HUP
     PYTHON=python
 
-    IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
-    if [ $IS_PYTHON_2 = "1" ]; then
-        echo "Using a Python 2"
-        # Upgrade pip to install the cryptography wheel and not the tar.
-        # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-        python -m pip install --upgrade 'pip<20.1'
-        # Upgrade setuptools because cryptography requires 18.5+.
-        # <45 because 45.0 dropped support for 2.7.
-        python -m pip install --upgrade 'setuptools<45'
-    fi
-
-    python -m pip install pyopenssl requests service_identity
+    python -m pip install --prefer-binary -r .evergreen/test-pyopenssl-requirements.txt
 fi
 
 if [ -n "$TEST_ENCRYPTION" ]; then
@@ -135,6 +129,8 @@ if [ -n "$TEST_ENCRYPTION" ]; then
 
     if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
         $PYTHON -m pip install -U setuptools
+        # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
+        powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
     fi
 
     if [ -z "$LIBMONGOCRYPT_URL" ]; then
@@ -166,18 +162,51 @@ if [ -n "$TEST_ENCRYPTION" ]; then
 
     # TODO: Test with 'pip install pymongocrypt'
     git clone --branch master https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
-    python -m pip install --upgrade ./libmongocrypt_git/bindings/python
+    python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt
+    python -m pip install ./libmongocrypt_git/bindings/python
     python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
     python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
     # PATH is updated by PREPARE_SHELL for access to mongocryptd.
 
+    # Get access to the AWS temporary credentials:
+    # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
+    . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
+
+    # Start the mock KMS servers.
+    # The mock KMS server requires Python >=3.5 with boto3.
+    IS_PRE_35=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3, 5) else '0')")
+    if [ $IS_PRE_35 = "1" ]; then
+        deactivate
+        createvirtualenv $PYTHON3_BINARY venv-kms
+        python -m pip install boto3
+    fi
+    pushd ${DRIVERS_TOOLS}/.evergreen/csfle
+    python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
+    python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+    trap 'kill $(jobs -p)' EXIT HUP
+    popd
+    # Restore the test virtualenv.
+    if [ $IS_PRE_35 = "1" ]; then
+        deactivate
+        if [ "Windows_NT" = "$OS" ]; then
+            . venv-encryption/Scripts/activate
+        else
+            . venv-encryption/bin/activate
+        fi
+    fi
 fi
 
 PYTHON_IMPL=$($PYTHON -c "import platform, sys; sys.stdout.write(platform.python_implementation())")
 if [ $PYTHON_IMPL = "Jython" ]; then
-    EXTRA_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
+    PYTHON_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
 else
-    EXTRA_ARGS=""
+    PYTHON_ARGS=""
+fi
+
+if [ -z "$DATA_LAKE" ]; then
+    TEST_ARGS=""
+else
+    TEST_ARGS="-s test.test_data_lake"
 fi
 
 # Don't download unittest-xml-reporting from pypi, which often fails.
@@ -200,14 +229,14 @@ $PYTHON -c 'import sys; print(sys.version)'
 
 # Run the tests with coverage if requested and coverage is installed.
 # Only cover CPython. Jython and PyPy report suspiciously low coverage.
-COVERAGE_OR_PYTHON="$PYTHON"
+# Also skip CPython 3.4. It's not supported by coverage 5+, which uses
+# a new and incompatible data format.
+PYTHON_VERSION=$($PYTHON -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
 COVERAGE_ARGS=""
-if [ -n "$COVERAGE" -a $PYTHON_IMPL = "CPython" ]; then
-    COVERAGE_BIN="$(dirname "$PYTHON")/coverage"
-    if $COVERAGE_BIN --version; then
+if [ -n "$COVERAGE" -a $PYTHON_IMPL = "CPython" -a $PYTHON_VERSION != "3.4" ]; then
+    if $PYTHON -m coverage --version; then
         echo "INFO: coverage is installed, running tests with coverage..."
-        COVERAGE_OR_PYTHON="$COVERAGE_BIN"
-        COVERAGE_ARGS="run --branch"
+        COVERAGE_ARGS="-m coverage run --branch"
     else
         echo "INFO: coverage is not installed, running tests without coverage..."
     fi
@@ -226,7 +255,8 @@ if [ -z "$GREEN_FRAMEWORK" ]; then
         # causing this script to exit.
         $PYTHON -c "from bson import _cbson; from pymongo import _cmessage"
     fi
-    $COVERAGE_OR_PYTHON $EXTRA_ARGS $COVERAGE_ARGS setup.py $C_EXTENSIONS test $OUTPUT
+
+    $PYTHON $COVERAGE_ARGS setup.py $C_EXTENSIONS test $TEST_ARGS $OUTPUT
 else
     # --no_ext has to come before "test" so there is no way to toggle extensions here.
     $PYTHON green_framework_test.py $GREEN_FRAMEWORK $OUTPUT

.evergreen/test-cryptography-requirements.txt

@@ -0,0 +1,7 @@
+# cffi==1.14.3 was the last installable release on RHEL 6.2 with Python 3.4
+cffi==1.14.3;python_version=="3.4"
+cffi>=1.12.0,<2;python_version!="3.4"
+cryptography>=2,<3.4;python_version=="2.7"
+cryptography>=2,<2.9;python_version=="3.4"
+cryptography>=2,<3.3;python_version=="3.5"
+cryptography>=2;python_version>"3.5"

.evergreen/test-encryption-requirements.txt

@@ -0,0 +1,3 @@
+-r test-cryptography-requirements.txt
+# boto3 is required by drivers-evergreen-tools/.evergreen/csfle/set-temp-creds.sh
+boto3<2

.evergreen/test-pyopenssl-requirements.txt

@@ -0,0 +1,8 @@
+-r test-cryptography-requirements.txt
+pyopenssl>=17.2.0,<20;python_version=="3.4"
+pyopenssl>=17.2.0;python_version!="3.4"
+attrs<=20.3.0;python_version=="3.4"
+service-identity==18.1.0;python_version=="3.4"
+service-identity>=18.1.0;python_version!="3.4"
+requests<2.22;python_version=="3.4"
+requests<3.0;python_version!="3.4"

.evergreen/utils.sh

@@ -8,19 +8,31 @@ createvirtualenv () {
     PYTHON=$1
     VENVPATH=$2
     if $PYTHON -m virtualenv --version; then
-        VIRTUALENV="$PYTHON -m virtualenv"
+        VIRTUALENV="$PYTHON -m virtualenv --never-download"
+    elif $PYTHON -m venv -h>/dev/null; then
+        VIRTUALENV="$PYTHON -m venv"
     elif command -v virtualenv; then
-        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
+        VIRTUALENV="$(command -v virtualenv) -p $PYTHON --never-download"
     else
         echo "Cannot test without virtualenv"
         exit 1
     fi
-    $VIRTUALENV --system-site-packages --never-download $VENVPATH
+    $VIRTUALENV $VENVPATH
     if [ "Windows_NT" = "$OS" ]; then
         . $VENVPATH/Scripts/activate
     else
         . $VENVPATH/bin/activate
     fi
+    # Upgrade to the latest versions of pip setuptools wheel so that
+    # pip can always download the latest cryptography+cffi wheels.
+    PYTHON_VERSION=$(python -c 'import sys;print("%s.%s" % sys.version_info[:2])')
+    if [[ $PYTHON_VERSION == "2.7" || $PYTHON_VERSION == "3.4" || $PYTHON_VERSION == "3.5" ]]; then
+        # Use get-pip for EOL Python versions.
+        curl --retry 3 -L https://bootstrap.pypa.io/pip/$PYTHON_VERSION/get-pip.py | python
+    else
+        python -m pip install --upgrade pip
+    fi
+    python -m pip install --upgrade setuptools wheel
 }
 
 # Usage:

.readthedocs.yaml

@@ -0,0 +1,19 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the doc/ directory with Sphinx
+sphinx:
+   configuration: doc/conf.py
+
+# Set the version of Python and requirements required to build the docs.
+python:
+   version: 3.8
+   install:
+      # Install pymongo itself.
+      - method: pip
+        path: .
+      - requirements: doc/docs-requirements.txt

.travis.yml

@@ -1,17 +0,0 @@
-language: python
-
-python:
-  - 2.7
-  - 3.4
-  - 3.5
-  - 3.6
-  - 3.7
-  - 3.8
-  - pypy
-  - pypy3.5
-
-services:
-  - mongodb
-
-script: PYMONGO_MUST_CONNECT=1 python setup.py test
-

README.rst

@@ -17,7 +17,7 @@ is a `gridfs
 <http://www.mongodb.org/display/DOCS/GridFS+Specification>`_
 implementation on top of ``pymongo``.
 
-PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, 3.6, 4.0, 4.2, and 4.4.
+PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, 3.6, 4.0, 4.2, 4.4, and 5.0.
 
 Support / Feedback
 ==================
@@ -91,6 +91,9 @@ Dependencies
 
 PyMongo supports CPython 2.7, 3.4+, PyPy, and PyPy3.5+.
 
+**WARNING** Support for Python 2.7, 3.4 and 3.5 is deprecated. Those Python
+versions will not be supported by PyMongo 4.
+
 Optional dependencies:
 
 GSSAPI authentication requires `pykerberos
@@ -225,4 +228,4 @@ Or with Eventlet's::
 
     $ python green_framework_test.py eventlet
 
-.. _sphinx: http://sphinx.pocoo.org/
+.. _sphinx: https://www.sphinx-doc.org/en/master/

RELEASE.rst

@@ -55,50 +55,38 @@ Doing a Release
 8. Push commit / tag, eg ``git push && git push --tags``.
 
 9. Pushing a tag will trigger a release process in Evergreen which builds
-   wheels and eggs for manylinux, macOS, and Windows. Wait for these jobs to
-   complete and then download the "Release files" archive from each task. See:
+   wheels for manylinux, macOS, and Windows. Wait for the "release-combine"
+   task to complete and then download the "Release files all" archive. See:
    https://evergreen.mongodb.com/waterfall/mongo-python-driver?bv_filter=release
 
-   Unpack each downloaded archive so that we can upload the included files. For
-   the next steps let's assume we unpacked these files into the following paths::
+   The contents should look like this::
 
-     $ ls path/to/manylinux
-     pymongo-<version>-cp27-cp27m-manylinux1_i686.whl
+     $ ls path/to/archive
+     pymongo-<version>-cp310-cp310-macosx_10_9_universal2.whl
      ...
      pymongo-<version>-cp38-cp38-manylinux2014_x86_64.whl
-     $ ls path/to/mac/
      pymongo-<version>-cp27-cp27m-macosx_10_14_intel.whl
-     ...
      pymongo-<version>-py2.7-macosx-10.14-intel.egg
-     $ ls path/to/windows/
-     pymongo-<version>-cp27-cp27m-win32.whl
-     ...
      pymongo-<version>-cp38-cp38-win_amd64.whl
-
-10. Build the source distribution::
-
-     $ git clone git@github.com:mongodb/mongo-python-driver.git
-     $ cd mongo-python-driver
-     $ git checkout "<release version number>"
-     $ python3 setup.py sdist
-
-    This will create the following distribution::
-
-     $ ls dist
+     ...
      pymongo-<version>.tar.gz
 
-11. Upload all the release packages to PyPI with twine::
+10. Upload all the release packages to PyPI with twine::
 
-     $ python3 -m twine upload dist/*.tar.gz path/to/manylinux/* path/to/mac/* path/to/windows/*
+     $ python3 -m twine upload path/to/archive/*
 
-12. Make sure the new version appears on https://pymongo.readthedocs.io/. If the
+11. Make sure the new version appears on https://pymongo.readthedocs.io/. If the
     new version does not show up automatically, trigger a rebuild of "latest":
     https://readthedocs.org/projects/pymongo/builds/
 
-13. Bump the version number to <next version>.dev0 in setup.py/__init__.py,
+12. Bump the version number to <next version>.dev0 in setup.py/__init__.py,
     commit, push.
 
-14. Publish the release version in Jira.
+13. Publish the release version in Jira.
 
-15. Announce the release on:
+14. Announce the release on:
     https://developer.mongodb.com/community/forums/c/community/release-notes/
+
+15. File a ticket for DOCSP highlighting changes in server version and Python
+    version compatibility or the lack thereof, for example:
+    https://jira.mongodb.org/browse/DOCSP-13536

bson/__init__.py

@@ -1062,6 +1062,16 @@ def _decode_selective(rawdoc, fields, codec_options):
     return doc
 
 
+def _convert_raw_document_lists_to_streams(document):
+    cursor = document.get('cursor')
+    if cursor:
+        for key in ('firstBatch', 'nextBatch'):
+            batch = cursor.get(key)
+            if batch:
+                stream = b"".join(doc.raw for doc in batch)
+                cursor[key] = [stream]
+
+
 def _decode_all_selective(data, codec_options, fields):
     """Decode BSON data to a single document while using user-provided
     custom decoding logic.

bson/_cbsonmodule.c

@@ -2621,7 +2621,7 @@ static int _element_to_dict(PyObject* self, const char* string,
     if (name_length > BSON_MAX_SIZE || position + name_length >= max) {
         PyObject* InvalidBSON = _error("InvalidBSON");
         if (InvalidBSON) {
-            PyErr_SetNone(InvalidBSON);
+            PyErr_SetString(InvalidBSON, "field name too large");
             Py_DECREF(InvalidBSON);
         }
         return -1;

bson/codec_options.py

@@ -295,6 +295,17 @@ class CodecOptions(_options_base):
                  self.unicode_decode_error_handler, self.tzinfo,
                  self.type_registry))
 
+    def _options_dict(self):
+        """Dictionary of the arguments used to create this object."""
+        # TODO: PYTHON-2442 use _asdict() instead
+        return {
+            'document_class': self.document_class,
+            'tz_aware': self.tz_aware,
+            'uuid_representation': self.uuid_representation,
+            'unicode_decode_error_handler': self.unicode_decode_error_handler,
+            'tzinfo': self.tzinfo,
+            'type_registry': self.type_registry}
+
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self._arguments_repr())
 
@@ -310,7 +321,7 @@ class CodecOptions(_options_base):
 
         .. versionadded:: 3.5
         """
-        opts = self._asdict()
+        opts = self._options_dict()
         opts.update(kwargs)
         return CodecOptions(**opts)
 

bson/dbref.py

@@ -43,7 +43,7 @@ class DBRef(object):
           - `**kwargs` (optional): additional keyword arguments will
             create additional, custom fields
 
-        .. mongodoc:: dbrefs
+        .. seealso:: The MongoDB documentation on `dbrefs <https://dochub.mongodb.org/core/dbrefs>`_.
         """
         if not isinstance(collection, string_type):
             raise TypeError("collection must be an "

bson/json_util.py

@@ -311,6 +311,16 @@ class JSONOptions(CodecOptions):
                     self.json_mode,
                     super(JSONOptions, self)._arguments_repr()))
 
+    def _options_dict(self):
+        # TODO: PYTHON-2442 use _asdict() instead
+        options_dict = super(JSONOptions, self)._options_dict()
+        options_dict.update({
+            'strict_number_long': self.strict_number_long,
+            'datetime_representation': self.datetime_representation,
+            'strict_uuid': self.strict_uuid,
+            'json_mode': self.json_mode})
+        return options_dict
+
     def with_options(self, **kwargs):
         """
         Make a copy of this JSONOptions, overriding some options::
@@ -324,7 +334,7 @@ class JSONOptions(CodecOptions):
 
         .. versionadded:: 3.12
         """
-        opts = self._asdict()
+        opts = self._options_dict()
         for opt in ('strict_number_long', 'datetime_representation',
                     'strict_uuid', 'json_mode'):
             opts[opt] = kwargs.get(opt, getattr(self, opt))
@@ -495,7 +505,7 @@ def object_hook(dct, json_options=DEFAULT_JSON_OPTIONS):
 def _parse_legacy_regex(doc):
     pattern = doc["$regex"]
     # Check if this is the $regex query operator.
-    if isinstance(pattern, Regex):
+    if not isinstance(pattern, (text_type, bytes)):
         return doc
     flags = 0
     # PyMongo always adds $options but some other tools may not.

bson/objectid.py

@@ -94,7 +94,7 @@ class ObjectId(object):
         :Parameters:
           - `oid` (optional): a valid ObjectId.
 
-        .. mongodoc:: objectids
+        .. seealso:: The MongoDB documentation on `ObjectIds`_.
 
         .. versionchanged:: 3.8
            :class:`~bson.objectid.ObjectId` now implements the `ObjectID

bson/raw_bson.py

@@ -13,6 +13,43 @@
 # limitations under the License.
 
 """Tools for representing raw BSON documents.
+
+Inserting and Retrieving RawBSONDocuments
+=========================================
+
+Example: Moving a document between different databases/collections
+
+.. doctest::
+
+  >>> import bson
+  >>> from pymongo import MongoClient
+  >>> from bson.raw_bson import RawBSONDocument
+  >>> client = MongoClient(document_class=RawBSONDocument)
+  >>> client.drop_database('db')
+  >>> client.drop_database('replica_db')
+  >>> db = client.db
+  >>> result = db.test.insert_many([{'a': 1},
+  ...                               {'b': 1},
+  ...                               {'c': 1},
+  ...                               {'d': 1}])
+  >>> replica_db = client.replica_db
+  >>> for doc in db.test.find():
+  ...    print("raw document: %r" % (doc.raw,))
+  ...    result = replica_db.test.insert_one(doc)
+  raw document: '...'
+  raw document: '...'
+  raw document: '...'
+  raw document: '...'
+  >>> for doc in replica_db.test.find(projection={'_id': 0}):
+  ...     print("decoded document: %r" % (bson.decode(doc.raw),))
+  decoded document: {u'a': 1}
+  decoded document: {u'b': 1}
+  decoded document: {u'c': 1}
+  decoded document: {u'd': 1}
+
+For use cases like moving documents across different databases or writing binary
+blobs to disk, using raw BSON documents provides better speed and avoids the
+overhead of decoding or encoding BSON.
 """
 
 from bson import _raw_to_dict, _get_object_size

doc/Makefile

@@ -1,89 +1,20 @@
-# Makefile for Sphinx documentation
+# Minimal makefile for Sphinx documentation
 #
 
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
 BUILDDIR      = _build
 
-# Internal variables.
-PAPEROPT_a4     = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
-
+# Put it first so that "make" without argument is like "make help".
 help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html      to make standalone HTML files"
-	@echo "  dirhtml   to make HTML files named index.html in directories"
-	@echo "  pickle    to make pickle files"
-	@echo "  json      to make JSON files"
-	@echo "  htmlhelp  to make HTML files and a HTML help project"
-	@echo "  qthelp    to make HTML files and a qthelp project"
-	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  changes   to make an overview of all changed/added/deprecated items"
-	@echo "  linkcheck to check all external links for integrity"
-	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-clean:
-	-rm -rf $(BUILDDIR)/*
+.PHONY: help Makefile
 
-html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-pickle:
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json:
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyMongo.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyMongo.qhc"
-
-latex:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
-	      "run these through (pdf)latex."
-
-changes:
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
-	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
-	@echo
-	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/api/pymongo/collection.rst

@@ -47,8 +47,8 @@
       .. automethod:: aggregate
       .. automethod:: aggregate_raw_batches
       .. automethod:: watch
-      .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None)
-      .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+      .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
+      .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
       .. automethod:: find_one(filter=None, *args, **kwargs)
       .. automethod:: find_one_and_delete
       .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, hint=None, session=None, **kwargs)

doc/api/pymongo/cursor.rst

@@ -15,13 +15,13 @@
       .. autoattribute:: EXHAUST
          :annotation:
 
-   .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+   .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
       :members:
 
       .. describe:: c[index]
 
-         See :meth:`__getitem__`.
+         See :meth:`__getitem__` and read the warning.
 
       .. automethod:: __getitem__
 
-   .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+   .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)

doc/api/pymongo/hello.rst

@@ -0,0 +1,10 @@
+:orphan:
+
+:mod:`hello` -- A wrapper for hello command responses.
+======================================================
+
+.. automodule:: pymongo.hello
+
+   .. autoclass:: pymongo.hello.Hello(doc)
+
+      .. autoattribute:: document

doc/api/pymongo/index.rst

@@ -54,6 +54,9 @@ Sub-modules:
    read_preferences
    results
    son_manipulator
+   server_api
+   server_description
+   topology_description
    uri_parser
    write_concern
    event_loggers

doc/api/pymongo/ismaster.rst

@@ -1,7 +1,7 @@
 :orphan:
 
-:mod:`ismaster` -- A wrapper for ismaster command responses.
-============================================================
+:mod:`ismaster` -- **DEPRECATED** A wrapper for hello command responses.
+========================================================================
 
 .. automodule:: pymongo.ismaster
 

doc/api/pymongo/mongo_client.rst

@@ -15,6 +15,7 @@
          Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used.
 
       .. autoattribute:: event_listeners
+      .. autoattribute:: topology_description
       .. autoattribute:: address
       .. autoattribute:: primary
       .. autoattribute:: secondaries

doc/api/pymongo/server_api.rst

@@ -0,0 +1,11 @@
+:mod:`server_api` -- Support for MongoDB Versioned API
+======================================================
+
+.. automodule:: pymongo.server_api
+   :synopsis: Support for MongoDB Versioned API
+
+   .. autoclass:: pymongo.server_api.ServerApi
+      :members:
+
+   .. autoclass:: pymongo.server_api.ServerApiVersion
+      :members:

doc/api/pymongo/server_description.rst

@@ -6,8 +6,4 @@
 .. automodule:: pymongo.server_description
 
    .. autoclass:: pymongo.server_description.ServerDescription()
-
-      .. autoattribute:: address
-      .. autoattribute:: all_hosts
-      .. autoattribute:: server_type
-      .. autoattribute:: server_type_name
+      :members:

doc/api/pymongo/topology_description.rst

@@ -6,9 +6,5 @@
 .. automodule:: pymongo.topology_description
 
    .. autoclass:: pymongo.topology_description.TopologyDescription()
+      :members:
 
-      .. automethod:: has_readable_server(read_preference=ReadPreference.PRIMARY)
-      .. automethod:: has_writable_server
-      .. automethod:: server_descriptions
-      .. autoattribute:: topology_type
-      .. autoattribute:: topology_type_name

doc/changelog.rst

@@ -1,6 +1,195 @@
 Changelog
 =========
 
+Changes in Version 3.12.3
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.12.3 fixes a bug that prevented :meth:`bson.json_util.loads` from
+decoding a document with a non-string "$regex" field (`PYTHON-3028`_).
+
+See the `PyMongo 3.12.3 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-3028: https://jira.mongodb.org/browse/PYTHON-3028
+.. _PyMongo 3.12.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=32505
+
+Changes in Version 3.12.2
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.12.2 fixes a number of bugs:
+
+- Fixed a bug that prevented PyMongo from retrying bulk writes
+  after a ``writeConcernError`` on MongoDB 4.4+ (`PYTHON-2984`_).
+- Fixed a bug that could cause the driver to hang during automatic
+  client side field level encryption (`PYTHON-3017`_).
+
+See the `PyMongo 3.12.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2984: https://jira.mongodb.org/browse/PYTHON-2984
+.. _PYTHON-3017: https://jira.mongodb.org/browse/PYTHON-3017
+.. _PyMongo 3.12.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=32310
+
+Changes in Version 3.12.1
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.12.1 fixes a number of bugs:
+
+- Fixed a bug that caused a multi-document transaction to fail when the first
+  operation was large bulk write (>48MB) that required splitting a batched
+  write command (`PYTHON-2915`_).
+- Fixed a bug that caused the ``tlsDisableOCSPEndpointCheck`` URI option to
+  be applied incorrectly (`PYTHON-2866`_).
+
+See the `PyMongo 3.12.1 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2915: https://jira.mongodb.org/browse/PYTHON-2915
+.. _PYTHON-2866: https://jira.mongodb.org/browse/PYTHON-2866
+.. _PyMongo 3.12.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=31527
+
+Changes in Version 3.12.0
+-------------------------
+
+.. warning:: PyMongo 3.12.0 deprecates support for Python 2.7, 3.4 and 3.5.
+   These Python versions will not be supported by PyMongo 4.
+
+.. warning:: PyMongo now allows insertion of documents with keys that include
+   dots ('.') or start with dollar signs ('$').
+
+- PyMongoCrypt 1.1.0 or later is now required for client side field level
+  encryption support.
+
+Notable improvements
+....................
+
+- Added support for MongoDB 5.0.
+- Support for MongoDB Versioned API, see :class:`~pymongo.server_api.ServerApi`.
+- Support for snapshot reads on secondaries (see :ref:`snapshot-reads-ref`).
+- Support for Azure and GCP KMS providers for client side field level
+  encryption. See the docstring for :class:`~pymongo.mongo_client.MongoClient`,
+  :class:`~pymongo.encryption_options.AutoEncryptionOpts`,
+  and :mod:`~pymongo.encryption`.
+- Support AWS authentication with temporary credentials when connecting to KMS
+  in client side field level encryption.
+- Support for connecting to load balanced MongoDB clusters via the new
+  ``loadBalanced`` URI option.
+- Support for creating timeseries collections via the ``timeseries`` and
+  ``expireAfterSeconds`` arguments to
+  :meth:`~pymongo.database.Database.create_collection`.
+- Added :attr:`pymongo.mongo_client.MongoClient.topology_description`.
+- Added hash support to :class:`~pymongo.mongo_client.MongoClient`,
+  :class:`~pymongo.database.Database` and
+  :class:`~pymongo.collection.Collection` (`PYTHON-2466`_).
+- Improved the error message returned by
+  :meth:`~pymongo.collection.Collection.insert_many` when supplied with an
+  argument of incorrect type (`PYTHON-1690`_).
+- Added session and read concern support to
+  :meth:`~pymongo.collection.Collection.find_raw_batches`
+  and :meth:`~pymongo.collection.Collection.aggregate_raw_batches`.
+
+Bug fixes
+.........
+
+- Fixed a bug that could cause the driver to deadlock during automatic
+  client side field level encryption (`PYTHON-2472`_).
+- Fixed a potential deadlock when garbage collecting an unclosed exhaust
+  :class:`~pymongo.cursor.Cursor`.
+- Fixed an bug where using gevent.Timeout to timeout an operation could
+  lead to a deadlock.
+- Fixed the following bug with Atlas Data Lake. When closing cursors,
+  pymongo now sends killCursors with the namespace returned the cursor's
+  initial command response.
+- Fixed a bug in :class:`~pymongo.cursor.RawBatchCursor` that caused it to
+  return an empty bytestring when the cursor contained no results. It now
+  raises :exc:`StopIteration` instead.
+
+Deprecations
+............
+
+- Deprecated support for Python 2.7, 3.4 and 3.5.
+- Deprecated support for database profiler helpers
+  :meth:`~pymongo.database.Database.profiling_level`,
+  :meth:`~pymongo.database.Database.set_profiling_level`,
+  and :meth:`~pymongo.database.Database.profiling_info`. Instead, users
+  should run the `profile command`_ with the
+  :meth:`~pymongo.database.Database.command` helper directly.
+- Deprecated :exc:`~pymongo.errors.NotMasterError`. Users should
+  use :exc:`~pymongo.errors.NotPrimaryError` instead.
+- Deprecated :class:`~pymongo.ismaster.IsMaster` and :mod:`~pymongo.ismaster`
+  which will be removed in PyMongo 4.0 and are replaced by
+  :class:`~pymongo.hello.Hello` and :mod:`~pymongo.hello` which provide the
+  same API.
+- Deprecated the :mod:`pymongo.messeage` module.
+- Deprecated the ``ssl_keyfile`` and ``ssl_certfile`` URI options in favor
+  of ``tlsCertificateKeyFile`` (see :doc:`examples/tls`).
+
+.. _PYTHON-2466: https://jira.mongodb.org/browse/PYTHON-2466
+.. _PYTHON-1690: https://jira.mongodb.org/browse/PYTHON-1690
+.. _PYTHON-2472: https://jira.mongodb.org/browse/PYTHON-2472
+.. _profile command: https://docs.mongodb.com/manual/reference/command/profile/
+
+Issues Resolved
+...............
+
+See the `PyMongo 3.12.0 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 3.12.0 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=29594
+
+
+Changes in Version 3.11.3
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.11.3 fixes a bug that prevented PyMongo from retrying writes after
+a ``writeConcernError`` on MongoDB 4.4+ (`PYTHON-2452`_)
+
+See the `PyMongo 3.11.3 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2452: https://jira.mongodb.org/browse/PYTHON-2452
+.. _PyMongo 3.11.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=30355
+
+Changes in Version 3.11.2
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.11.2 includes a number of bugfixes. Highlights include:
+
+- Fixed a memory leak caused by failing SDAM monitor checks on Python 3 (`PYTHON-2433`_).
+- Fixed a regression that changed the string representation of
+  :exc:`~pymongo.errors.BulkWriteError` (`PYTHON-2438`_).
+- Fixed a bug that made it impossible to use
+  :meth:`bson.codec_options.CodecOptions.with_options` and
+  :meth:`~bson.json_util.JSONOptions.with_options` on some early versions of
+  Python 3.4 and Python 3.5 due to a bug in the standard library implementation
+  of :meth:`collections.namedtuple._asdict` (`PYTHON-2440`_).
+- Fixed a bug that resulted in a :exc:`TypeError` exception when a PyOpenSSL
+  socket was configured with a timeout of ``None`` (`PYTHON-2443`_).
+
+See the `PyMongo 3.11.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2433: https://jira.mongodb.org/browse/PYTHON-2433
+.. _PYTHON-2438: https://jira.mongodb.org/browse/PYTHON-2438
+.. _PYTHON-2440: https://jira.mongodb.org/browse/PYTHON-2440
+.. _PYTHON-2443: https://jira.mongodb.org/browse/PYTHON-2443
+.. _PyMongo 3.11.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=30315
+
 Changes in Version 3.11.1
 -------------------------
 
@@ -35,6 +224,7 @@ in this release.
 
 .. _PyMongo 3.11.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=29997
 
+
 Changes in Version 3.11.0
 -------------------------
 

doc/conf.py

@@ -14,8 +14,7 @@ import pymongo
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.coverage',
-              'sphinx.ext.todo', 'doc.mongo_extensions',
-              'sphinx.ext.intersphinx']
+              'sphinx.ext.todo', 'sphinx.ext.intersphinx']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -91,13 +90,6 @@ html_theme_options = {
 # Additional static files.
 html_static_path = ['static']
 
-# These paths are either relative to html_static_path
-# or fully qualified paths (eg. https://...)
-# Note: html_js_files was added in Sphinx 1.8.
-html_js_files = [
-    'delighted.js',
-]
-
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None

doc/contributors.rst

@@ -88,3 +88,4 @@ The following is a list of people who have contributed to
 - Terence Honles (terencehonles)
 - Paul Fisher (thetorpedodog)
 - Julius Park (juliusgeo)
+- Khanh Nguyen (KN99HN)

doc/docs-requirements.txt

@@ -0,0 +1,3 @@
+Sphinx~=4.2
+sphinx_rtd_theme~=0.5
+readthedocs-sphinx-search~=0.1

doc/examples/encryption.rst

@@ -16,7 +16,7 @@ level encryption supports workloads where applications must guarantee that
 unauthorized parties, including server administrators, cannot read the
 encrypted data.
 
-.. mongodoc:: client-side-field-level-encryption
+.. seealso:: The MongoDB documentation on `Client Side Field Level Encryption <https://dochub.mongodb.org/core/client-side-field-level-encryption>`_.
 
 Dependencies
 ------------

doc/examples/geo.rst

@@ -10,7 +10,7 @@ Geospatial Indexing Example
 This example shows how to create and use a :data:`~pymongo.GEO2D`
 index in PyMongo. To create a spherical (earth-like) geospatial index use :data:`~pymongo.GEOSPHERE` instead.
 
-.. mongodoc:: geo
+.. seealso:: The MongoDB documentation on `Geospatial Indexes <https://dochub.mongodb.org/core/geo>`_.
 
 Creating a Geospatial Index
 ---------------------------

doc/examples/high_availability.rst

@@ -14,7 +14,7 @@ PyMongo makes working with `replica sets
 replica set and show how to handle both initialization and normal
 connections with PyMongo.
 
-.. mongodoc:: rs
+.. seealso:: The MongoDB documentation on `replication <https://dochub.mongodb.org/core/rs>`_.
 
 Starting a Replica Set
 ~~~~~~~~~~~~~~~~~~~~~~

doc/examples/tls.rst

@@ -153,25 +153,31 @@ PyMongo can be configured to present a client certificate using the
   ...                              tls=True,
   ...                              tlsCertificateKeyFile='/path/to/client.pem')
 
-If the private key for the client certificate is stored in a separate file use
-the ``ssl_keyfile`` option::
+If the private key for the client certificate is stored in a separate file,
+it should be concatenated with the certificate file. For example, to
+concatenate a PEM-formatted certificate file ``cert.pem`` and a PEM-formatted
+keyfile ``key.pem`` into a single file ``combined.pem``, on Unix systems,
+users can run::
+
+  $ cat key.pem cert.pem > combined.pem
+
+PyMongo can be configured with the concatenated certificate keyfile using the
+``tlsCertificateKeyFile`` option::
 
   >>> client = pymongo.MongoClient('example.com',
   ...                              tls=True,
-  ...                              tlsCertificateKeyFile='/path/to/client.pem',
-  ...                              ssl_keyfile='/path/to/key.pem')
+  ...                              tlsCertificateKeyFile='/path/to/combined.pem')
 
-Python 2.7.9+ (pypy 2.5.1+) and 3.3+ support providing a password or passphrase
-to decrypt encrypted private keys. Use the ``tlsCertificateKeyFilePassword``
-option::
+If the private key contained in the certificate keyfile is encrypted,
+Python 2.7.9+ (pypy 2.5.1+) and 3.3+ support providing a password or
+passphrase to decrypt the encrypted private key. The password/passphrase
+can be specified using the ``tlsCertificateKeyFilePassword`` option::
 
   >>> client = pymongo.MongoClient('example.com',
   ...                              tls=True,
-  ...                              tlsCertificateKeyFile='/path/to/client.pem',
-  ...                              ssl_keyfile='/path/to/key.pem',
+  ...                              tlsCertificateKeyFile='/path/to/combined.pem',
   ...                              tlsCertificateKeyFilePassword=<passphrase>)
 
-
 These options can also be passed as part of the MongoDB URI.
 
 .. _OCSP:
@@ -243,3 +249,21 @@ revocation checking failed::
   [('SSL routines', 'tls_process_initial_server_flight', 'invalid status response')]
 
 See :ref:`OCSP` for more details.
+
+Python 3.10+ incompatibilities with TLS/SSL on MongoDB <= 4.0
+.............................................................
+
+Note that `changes made to the ssl module in Python 3.10+
+<https://docs.python.org/3/whatsnew/3.10.html#ssl>`_ may cause incompatibilities
+with MongoDB <= 4.0. The following are some example errors that may occur with this
+combination::
+
+  SSL handshake failed: localhost:27017: [SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:997)
+  SSL handshake failed: localhost:27017: EOF occurred in violation of protocol (_ssl.c:997)
+
+The MongoDB server logs may show the following error::
+
+  2021-06-30T21:22:44.917+0100 E NETWORK  [conn16] SSL: error:1408A0C1:SSL routines:ssl3_get_client_hello:no shared cipher
+
+To resolve this issue, use Python <=3.10, upgrade to MongoDB 4.2+, or install
+pymongo with the :ref:`OCSP` extra which relies on PyOpenSSL.

doc/examples/uuid.rst

@@ -1,3 +1,4 @@
+
 .. _handling-uuid-data-example:
 
 Handling UUID Data
@@ -12,7 +13,7 @@ to MongoDB and retrieve them as native :class:`uuid.UUID` objects::
   from uuid import uuid4
 
   # use the 'standard' representation for cross-language compatibility.
-  client = MongoClient(uuid_representation=UuidRepresentation.STANDARD)
+  client = MongoClient(uuidRepresentation='standard')
   collection = client.get_database('uuid_db').get_collection('uuid_coll')
 
   # remove all documents from collection
@@ -255,19 +256,27 @@ Applications can set the UUID representation in one of the following ways:
       * - ``unspecified``
         - :ref:`unspecified-representation-details`
 
-#. Using the ``uuid_representation`` kwarg option, e.g.::
+#. At the ``MongoClient`` level using the ``uuidRepresentation`` kwarg
+   option, e.g.::
 
      from bson.binary import UuidRepresentation
-     client = MongoClient(uuid_representation=UuidRepresentation.PYTHON_LEGACY)
+     client = MongoClient(uuidRepresentation=UuidRepresentation.PYTHON_LEGACY)
 
-#. By supplying a suitable :class:`~bson.codec_options.CodecOptions`
-   instance, e.g.::
+#. At the ``Database`` or ``Collection`` level by supplying a suitable
+   :class:`~bson.codec_options.CodecOptions` instance, e.g.::
 
      from bson.codec_options import CodecOptions
      csharp_opts = CodecOptions(uuid_representation=UuidRepresentation.CSHARP_LEGACY)
+     java_opts = CodecOptions(uuid_representation=UuidRepresentation.JAVA_LEGACY)
+
+     # Get database/collection from client with csharpLegacy UUID representation
      csharp_database = client.get_database('csharp_db', codec_options=csharp_opts)
      csharp_collection = client.testdb.get_collection('csharp_coll', codec_options=csharp_opts)
 
+     # Get database/collection from existing database/collection with javaLegacy UUID representation
+     java_database = csharp_database.with_options(codec_options=java_opts)
+     java_collection = csharp_collection.with_options(codec_options=java_opts)
+
 Supported UUID Representations
 ------------------------------
 

doc/faq.rst

@@ -45,6 +45,17 @@ multithreaded contexts with ``fork()``, see http://bugs.python.org/issue6721.
 
 .. _connection-pooling:
 
+Can PyMongo help me load the results of my query as a Pandas ``DataFrame``?
+---------------------------------------------------------------------------
+
+While PyMongo itself does not provide any APIs for working with
+numerical or columnar data,
+`PyMongoArrow <https://mongo-arrow.readthedocs.io/en/pymongoarrow-0.1.1/>`_
+is a companion library to PyMongo that makes it easy to load MongoDB query result sets as
+`Pandas DataFrames <https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html>`_,
+`NumPy ndarrays <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_, or
+`Apache Arrow Tables <https://arrow.apache.org/docs/python/generated/pyarrow.Table.html>`_.
+
 How does connection pooling work in PyMongo?
 --------------------------------------------
 
@@ -440,8 +451,8 @@ No. PyMongo creates Python threads which
 `PythonAnywhere <https://www.pythonanywhere.com>`_ does not support. For more
 information see `PYTHON-1495 <https://jira.mongodb.org/browse/PYTHON-1495>`_.
 
-How can I use something like Python's :mod:`json` module to encode my documents to JSON?
-----------------------------------------------------------------------------------------
+How can I use something like Python's ``json`` module to encode my documents to JSON?
+-------------------------------------------------------------------------------------
 :mod:`~bson.json_util` is PyMongo's built in, flexible tool for using
 Python's :mod:`json` module with BSON documents and `MongoDB Extended JSON
 <https://docs.mongodb.com/manual/reference/mongodb-extended-json/>`_. The

doc/index.rst

@@ -89,7 +89,7 @@ For older versions of the documentation please see the
 About This Documentation
 ------------------------
 This documentation is generated using the `Sphinx
-<http://sphinx.pocoo.org/>`_ documentation generator. The source files
+<https://www.sphinx-doc.org/en/master/>`_ documentation generator. The source files
 for the documentation are located in the *doc/* directory of the
 **PyMongo** distribution. To generate the docs locally run the
 following command from the root directory of the **PyMongo** source:

doc/installation.rst

@@ -47,6 +47,9 @@ Dependencies
 
 PyMongo supports CPython 2.7, 3.4+, PyPy, and PyPy3.5+.
 
+.. warning:: Support for Python 2.7, 3.4 and 3.5 is deprecated. Those Python
+    versions will not be supported by PyMongo 4.
+
 Optional dependencies:
 
 GSSAPI authentication requires `pykerberos
@@ -122,7 +125,7 @@ If you'd rather install directly from the source (i.e. to stay on the
 bleeding edge), install the C extension dependencies then check out the
 latest source from GitHub and install the driver from the resulting tree::
 
-  $ git clone git://github.com/mongodb/mongo-python-driver.git pymongo
+  $ git clone https://github.com/mongodb/mongo-python-driver.git pymongo
   $ cd pymongo/
   $ python setup.py install
 
@@ -275,4 +278,4 @@ but can be found on the
 `GitHub tags page <https://github.com/mongodb/mongo-python-driver/tags>`_.
 They can be installed by passing the full URL for the tag to pip::
 
-  $ python -m pip install https://github.com/mongodb/mongo-python-driver/archive/3.11.0rc0.tar.gz
+  $ python -m pip install https://github.com/mongodb/mongo-python-driver/archive/3.12.0b1.tar.gz

doc/make.bat

@@ -1,113 +1,35 @@
-@ECHO OFF
-
-REM Command file for Sphinx documentation
-
-set SPHINXBUILD=sphinx-build
-set BUILDDIR=_build
-set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
-if NOT "%PAPER%" == "" (
-	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
-)
-
-if "%1" == "" goto help
-
-if "%1" == "help" (
-	:help
-	echo.Please use `make ^<target^>` where ^<target^> is one of
-	echo.  html      to make standalone HTML files
-	echo.  dirhtml   to make HTML files named index.html in directories
-	echo.  pickle    to make pickle files
-	echo.  json      to make JSON files
-	echo.  htmlhelp  to make HTML files and a HTML help project
-	echo.  qthelp    to make HTML files and a qthelp project
-	echo.  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter
-	echo.  changes   to make an overview over all changed/added/deprecated items
-	echo.  linkcheck to check all external links for integrity
-	echo.  doctest   to run all doctests embedded in the documentation if enabled
-	goto end
-)
-
-if "%1" == "clean" (
-	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
-	del /q /s %BUILDDIR%\*
-	goto end
-)
-
-if "%1" == "html" (
-	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
-	goto end
-)
-
-if "%1" == "dirhtml" (
-	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
-	goto end
-)
-
-if "%1" == "pickle" (
-	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
-	echo.
-	echo.Build finished; now you can process the pickle files.
-	goto end
-)
-
-if "%1" == "json" (
-	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
-	echo.
-	echo.Build finished; now you can process the JSON files.
-	goto end
-)
-
-if "%1" == "htmlhelp" (
-	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
-	echo.
-	echo.Build finished; now you can run HTML Help Workshop with the ^
-.hhp project file in %BUILDDIR%/htmlhelp.
-	goto end
-)
-
-if "%1" == "qthelp" (
-	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
-	echo.
-	echo.Build finished; now you can run "qcollectiongenerator" with the ^
-.qhcp project file in %BUILDDIR%/qthelp, like this:
-	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyMongo.qhcp
-	echo.To view the help file:
-	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyMongo.ghc
-	goto end
-)
-
-if "%1" == "latex" (
-	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
-	echo.
-	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
-	goto end
-)
-
-if "%1" == "changes" (
-	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
-	echo.
-	echo.The overview file is in %BUILDDIR%/changes.
-	goto end
-)
-
-if "%1" == "linkcheck" (
-	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
-	echo.
-	echo.Link check complete; look for any errors in the above output ^
-or in %BUILDDIR%/linkcheck/output.txt.
-	goto end
-)
-
-if "%1" == "doctest" (
-	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
-	echo.
-	echo.Testing of doctests in the sources finished, look at the ^
-results in %BUILDDIR%/doctest/output.txt.
-	goto end
-)
-
-:end
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

doc/migrate-to-pymongo3.rst

@@ -433,13 +433,13 @@ can be changed to this with PyMongo 2.9 or later:
   >>> from pymongo.errors import ConnectionFailure
   >>> client = MongoClient(connect=False)
   >>> try:
-  ...     result = client.admin.command("ismaster")
+  ...     result = client.admin.command("ping")
   ... except ConnectionFailure:
   ...     print("Server not available")
   >>>
 
 Any operation can be used to determine if the server is available. We choose
-the "ismaster" command here because it is cheap and does not require auth, so
+the "ping" command here because it is cheap and does not require auth, so
 it is a simple way to check whether the server is available.
 
 The max_pool_size parameter is removed
@@ -516,9 +516,10 @@ Removed features with no migration path
 MasterSlaveConnection is removed
 ................................
 
-Master slave deployments are deprecated in MongoDB. Starting with MongoDB 3.0
-a replica set can have up to 50 members and that limit is likely to be
-removed in later releases. We recommend migrating to replica sets instead.
+Master slave deployments are no longer supported by MongoDB. Starting with
+MongoDB 3.0 a replica set can have up to 50 members and that limit is likely
+to be removed in later releases. We recommend migrating to replica sets
+instead.
 
 Requests are removed
 ....................

doc/mongo_extensions.py

@@ -1,97 +0,0 @@
-# Copyright 2009-present MongoDB, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""MongoDB specific extensions to Sphinx."""
-
-from docutils import nodes
-from docutils.parsers import rst
-from sphinx import addnodes
-
-
-class mongodoc(nodes.Admonition, nodes.Element):
-    pass
-
-
-class mongoref(nodes.reference):
-    pass
-
-
-def visit_mongodoc_node(self, node):
-    self.visit_admonition(node, "seealso")
-
-
-def depart_mongodoc_node(self, node):
-    self.depart_admonition(node)
-
-
-def visit_mongoref_node(self, node):
-    atts = {"class": "reference external",
-            "href": node["refuri"],
-            "name": node["name"]}
-    self.body.append(self.starttag(node, 'a', '', **atts))
-
-
-def depart_mongoref_node(self, node):
-    self.body.append('</a>')
-    if not isinstance(node.parent, nodes.TextElement):
-        self.body.append('\n')
-
-
-class MongodocDirective(rst.Directive):
-
-    has_content = True
-    required_arguments = 0
-    optional_arguments = 0
-    final_argument_whitespace = False
-    option_spec = {}
-
-    def run(self):
-        node = mongodoc()
-        title = 'The MongoDB documentation on'
-        node += nodes.title(title, title)
-        self.state.nested_parse(self.content, self.content_offset, node)
-        return [node]
-
-
-def process_mongodoc_nodes(app, doctree, fromdocname):
-    for node in doctree.traverse(mongodoc):
-        anchor = None
-        for name in node.parent.parent.traverse(addnodes.desc_signature):
-            anchor = name["ids"][0]
-            break
-        if not anchor:
-            for name in node.parent.traverse(nodes.section):
-                anchor = name["ids"][0]
-                break
-        for para in node.traverse(nodes.paragraph):
-            tag = str(para.traverse()[1])
-            link = mongoref("", "")
-            link["refuri"] = "http://dochub.mongodb.org/core/%s" % tag
-            link["name"] = anchor
-            link.append(nodes.emphasis(tag, tag))
-            new_para = nodes.paragraph()
-            new_para += link
-            node.replace(para, new_para)
-
-
-def setup(app):
-    app.add_node(mongodoc,
-                 html=(visit_mongodoc_node, depart_mongodoc_node),
-                 latex=(visit_mongodoc_node, depart_mongodoc_node),
-                 text=(visit_mongodoc_node, depart_mongodoc_node))
-    app.add_node(mongoref,
-                 html=(visit_mongoref_node, depart_mongoref_node))
-
-    app.add_directive("mongodoc", MongodocDirective)
-    app.connect("doctree-resolved", process_mongodoc_nodes)

doc/static/delighted.js

@@ -1,22 +0,0 @@
-/* eslint-disable */
-// Delighted
-!function(e,t,r,n,a){if(!e[a]){for(var i=e[a]=[],s=0;s<r.length;s++){var c=r[s];i[c]=i[c]||function(e){return function(){var t=Array.prototype.slice.call(arguments);i.push([e,t])}}(c)}i.SNIPPET_VERSION="1.0.1";var o=t.createElement("script");o.type="text/javascript",o.async=!0,o.src="https://d2yyd1h5u9mauk.cloudfront.net/integrations/web/v1/library/"+n+"/"+a+".js";var l=t.getElementsByTagName("script")[0];l.parentNode.insertBefore(o,l)}}(window,document,["survey","reset","config","init","set","get","event","identify","track","page","screen","group","alias"],"Dk30CC86ba0nATlK","delighted");
-// Segment
-!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t,e){var n=document.createElement("script");n.type="text/javascript";n.async=!0;n.src="https://cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var a=document.getElementsByTagName("script")[0];a.parentNode.insertBefore(n,a);analytics._loadOptions=e};analytics.SNIPPET_VERSION="4.1.0";
-        analytics.load("aGhVvyxnPWlyP71vVl9ZjGWxAtoVGLXX");
-    }}();
-
-delighted.survey({
-    minTimeOnPage: 180,
-    sampleFactor: 0.1,
-    properties: {
-        project: 'pymongo'
-    }
-});
-
-// Update Segment
-analytics.page({
-    path: location.pathname,
-    url: location.href,
-    project: 'pymongo'
-});

gridfs/__init__.py

@@ -17,7 +17,7 @@
 The :mod:`gridfs` package is an implementation of GridFS on top of
 :mod:`pymongo`, exposing a file-like interface.
 
-.. mongodoc:: gridfs
+.. seealso:: The MongoDB documentation on `gridfs <https://dochub.mongodb.org/core/gridfs>`_.
 """
 
 from bson.py3compat import abc
@@ -62,7 +62,7 @@ class GridFS(object):
            `database` must use an acknowledged
            :attr:`~pymongo.database.Database.write_concern`
 
-        .. mongodoc:: gridfs
+        .. seealso:: The MongoDB documentation on `gridfs <https://dochub.mongodb.org/core/gridfs>`_.
         """
         if not isinstance(database, Database):
             raise TypeError("database must be an instance of Database")
@@ -367,7 +367,7 @@ class GridFS(object):
            Removed the read_preference, tag_sets, and
            secondary_acceptable_latency_ms options.
         .. versionadded:: 2.7
-        .. mongodoc:: find
+        .. seealso:: The MongoDB documentation on `find <https://dochub.mongodb.org/core/find>`_.
         """
         return GridOutCursor(self.__collection, *args, **kwargs)
 
@@ -452,7 +452,7 @@ class GridFSBucket(object):
 
         .. versionadded:: 3.1
 
-        .. mongodoc:: gridfs
+        .. seealso:: The MongoDB documentation on `gridfs <https://dochub.mongodb.org/core/gridfs>`_.
         """
         if not isinstance(db, Database):
             raise TypeError("database must be an instance of Database")

gridfs/grid_file.py

@@ -820,7 +820,7 @@ class GridOutCursor(Cursor):
 
         .. versionadded 2.7
 
-        .. mongodoc:: cursors
+        .. seealso:: The MongoDB documentation on `cursors <https://dochub.mongodb.org/core/cursors>`_.
         """
         _disallow_transactions(session)
         collection = _clear_entity_type_registry(collection)

pymongo/__init__.py

@@ -68,13 +68,38 @@ TEXT = "text"
 """
 
 OFF = 0
-"""No database profiling."""
-SLOW_ONLY = 1
-"""Only profile slow operations."""
-ALL = 2
-"""Profile all operations."""
+"""**DEPRECATED** - No database profiling.
 
-version_tuple = (3, 11, 1)
+**DEPRECATED** - :attr:`OFF` is deprecated and will be removed in PyMongo 4.0.
+Instead, specify this profiling level using the numeric value ``0``.
+See https://docs.mongodb.com/manual/tutorial/manage-the-database-profiler
+
+.. versionchanged:: 3.12
+   Deprecated
+"""
+SLOW_ONLY = 1
+"""**DEPRECATED** - Only profile slow operations.
+
+**DEPRECATED** - :attr:`SLOW_ONLY` is deprecated and will be removed in
+PyMongo 4.0. Instead, specify this profiling level using the numeric
+value ``1``.
+See https://docs.mongodb.com/manual/tutorial/manage-the-database-profiler
+
+.. versionchanged:: 3.12
+   Deprecated
+"""
+ALL = 2
+"""**DEPRECATED** - Profile all operations.
+
+**DEPRECATED** - :attr:`ALL` is deprecated and will be removed in PyMongo 4.0.
+Instead, specify this profiling level using the numeric value ``2``.
+See https://docs.mongodb.com/manual/tutorial/manage-the-database-profiler
+
+.. versionchanged:: 3.12
+   Deprecated
+"""
+
+version_tuple = (3, 12, 4, '.dev0')
 
 def get_version_string():
     if isinstance(version_tuple[-1], str):

pymongo/_ipaddress.py

@@ -0,0 +1,56 @@
+# Copyright 2021-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test if a string is an IP Address"""
+
+import socket
+
+from bson.py3compat import _unicode
+
+try:
+    from ipaddress import ip_address
+    def is_ip_address(address):
+        try:
+            ip_address(_unicode(address))
+            return True
+        except (ValueError, UnicodeError):
+            return False
+except ImportError:
+    if hasattr(socket, 'inet_pton') and socket.has_ipv6:
+        # Most *nix, Windows newer than XP
+        def is_ip_address(address):
+            try:
+                # inet_pton rejects IPv4 literals with leading zeros
+                # (e.g. 192.168.0.01), inet_aton does not, and we
+                # can connect to them without issue. Use inet_aton.
+                socket.inet_aton(address)
+                return True
+            except socket.error:
+                try:
+                    socket.inet_pton(socket.AF_INET6, address)
+                    return True
+                except socket.error:
+                    return False
+    else:
+        # No inet_pton
+        def is_ip_address(address):
+            try:
+                socket.inet_aton(address)
+                return True
+            except socket.error:
+                if ':' in address:
+                    # ':' is not a valid character for a hostname.
+                    return True
+                return False
+

pymongo/aggregation.py

@@ -93,17 +93,18 @@ class _AggregationCommand(object):
         """Check whether the server version in-use supports aggregation."""
         pass
 
-    def _process_result(self, result, session, server, sock_info, slave_ok):
+    def _process_result(
+            self, result, session, server, sock_info, secondary_ok):
         if self._result_processor:
             self._result_processor(
-                result, session, server, sock_info, slave_ok)
+                result, session, server, sock_info, secondary_ok)
 
     def get_read_preference(self, session):
         if self._performs_write:
             return ReadPreference.PRIMARY
         return self._target._read_preference_for(session)
 
-    def get_cursor(self, session, server, sock_info, slave_ok):
+    def get_cursor(self, session, server, sock_info, secondary_ok):
         # Ensure command compatibility.
         self._check_compat(sock_info)
 
@@ -136,7 +137,7 @@ class _AggregationCommand(object):
         result = sock_info.command(
             self._database.name,
             cmd,
-            slave_ok,
+            secondary_ok,
             self.get_read_preference(session),
             self._target.codec_options,
             parse_write_concern_error=True,
@@ -147,7 +148,7 @@ class _AggregationCommand(object):
             client=self._database.client,
             user_fields=self._user_fields)
 
-        self._process_result(result, session, server, sock_info, slave_ok)
+        self._process_result(result, session, server, sock_info, secondary_ok)
 
         # Extract cursor from result or mock/fake one if necessary.
         if 'cursor' in result:
@@ -161,11 +162,13 @@ class _AggregationCommand(object):
             }
 
         # Create and return cursor instance.
-        return self._cursor_class(
+        cmd_cursor = self._cursor_class(
             self._cursor_collection(cursor), cursor, sock_info.address,
             batch_size=self._batch_size or 0,
             max_await_time_ms=self._max_await_time_ms,
             session=session, explicit_session=self._explicit_session)
+        cmd_cursor._maybe_pin_connection(sock_info)
+        return cmd_cursor
 
 
 class _CollectionAggregationCommand(_AggregationCommand):

pymongo/auth.py

@@ -579,9 +579,8 @@ def _authenticate_default(credentials, sock_info):
             mechs = sock_info.negotiated_mechanisms[credentials]
         else:
             source = credentials.source
-            cmd = SON([
-                ('ismaster', 1),
-                ('saslSupportedMechs', source + '.' + credentials.username)])
+            cmd = sock_info.hello_cmd()
+            cmd['saslSupportedMechs'] = source + '.' + credentials.username
             mechs = sock_info.command(
                 source, cmd, publish_events=False).get(
                 'saslSupportedMechs', [])
@@ -625,8 +624,8 @@ class _AuthContext(object):
     def speculate_command(self):
         raise NotImplementedError
 
-    def parse_response(self, ismaster):
-        self.speculative_authenticate = ismaster.speculative_authenticate
+    def parse_response(self, hello):
+        self.speculative_authenticate = hello.speculative_authenticate
 
     def speculate_succeeded(self):
         return bool(self.speculative_authenticate)

pymongo/bulk.py

@@ -28,7 +28,7 @@ from pymongo.common import (validate_is_mapping,
                             validate_is_document_type,
                             validate_ok_for_replace,
                             validate_ok_for_update)
-from pymongo.helpers import _RETRYABLE_ERROR_CODES
+from pymongo.helpers import _RETRYABLE_ERROR_CODES, _get_wce_doc
 from pymongo.collation import validate_collation_or_none
 from pymongo.errors import (BulkWriteError,
                             ConfigurationError,
@@ -126,9 +126,9 @@ def _merge_command(run, full_result, offset, result):
             replacement[_UOP] = run.ops[idx]
             full_result["writeErrors"].append(replacement)
 
-    wc_error = result.get("writeConcernError")
-    if wc_error:
-        full_result["writeConcernErrors"].append(wc_error)
+    wce = _get_wce_doc(result)
+    if wce:
+        full_result["writeConcernErrors"].append(wce)
 
 
 def _raise_bulk_write_error(full_result):
@@ -285,28 +285,31 @@ class _Bulk(object):
         # sock_info.write_command.
         sock_info.validate_session(client, session)
         while run:
-            cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
-                       ('ordered', self.ordered)])
-            if not write_concern.is_server_default:
-                cmd['writeConcern'] = write_concern.document
-            if self.bypass_doc_val and sock_info.max_wire_version >= 4:
-                cmd['bypassDocumentValidation'] = True
+            cmd_name = _COMMANDS[run.op_type]
             bwc = self.bulk_ctx_class(
-                db_name, cmd, sock_info, op_id, listeners, session,
+                db_name, cmd_name, sock_info, op_id, listeners, session,
                 run.op_type, self.collection.codec_options)
 
             while run.idx_offset < len(run.ops):
+                cmd = SON([(cmd_name, self.collection.name),
+                           ('ordered', self.ordered)])
+                if not write_concern.is_server_default:
+                    cmd['writeConcern'] = write_concern.document
+                if self.bypass_doc_val and sock_info.max_wire_version >= 4:
+                    cmd['bypassDocumentValidation'] = True
                 if session:
                     # Start a new retryable write unless one was already
                     # started for this command.
                     if retryable and not self.started_retryable_write:
                         session._start_retryable_write()
                         self.started_retryable_write = True
-                    session._apply_to(cmd, retryable, ReadPreference.PRIMARY)
+                    session._apply_to(cmd, retryable, ReadPreference.PRIMARY,
+                                      sock_info)
                 sock_info.send_cluster_time(cmd, session, client)
+                sock_info.add_server_api(cmd)
                 ops = islice(run.ops, run.idx_offset, None)
                 # Run as many ops as possible in one command.
-                result, to_send = bwc.execute(ops, client)
+                result, to_send = bwc.execute(cmd, ops, client)
 
                 # Retryable writeConcernErrors halt the execution of this run.
                 wce = result.get('writeConcernError', {})
@@ -366,16 +369,16 @@ class _Bulk(object):
     def execute_insert_no_results(self, sock_info, run, op_id, acknowledged):
         """Execute insert, returning no results.
         """
-        command = SON([('insert', self.collection.name),
-                       ('ordered', self.ordered)])
-        concern = {'w': int(self.ordered)}
-        command['writeConcern'] = concern
-        if self.bypass_doc_val and sock_info.max_wire_version >= 4:
-            command['bypassDocumentValidation'] = True
         db = self.collection.database
+        concern = {'w': int(self.ordered)}
+        cmd = SON([('insert', self.collection.name),
+                   ('ordered', self.ordered),
+                   ('writeConcern', concern)])
+        if self.bypass_doc_val and sock_info.max_wire_version >= 4:
+            cmd['bypassDocumentValidation'] = True
         bwc = _BulkWriteContext(
-            db.name, command, sock_info, op_id, db.client._event_listeners,
-            None, _INSERT, self.collection.codec_options)
+            db.name, 'insert', sock_info, op_id, db.client._event_listeners,
+            None, _INSERT, self.collection.codec_options, cmd_legacy=cmd)
         # Legacy batched OP_INSERT.
         _do_batched_insert(
             self.collection.full_name, run.ops, True, acknowledged, concern,
@@ -394,17 +397,19 @@ class _Bulk(object):
         run = self.current_run
 
         while run:
-            cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
-                       ('ordered', False),
-                       ('writeConcern', {'w': 0})])
+            cmd_name = _COMMANDS[run.op_type]
             bwc = self.bulk_ctx_class(
-                db_name, cmd, sock_info, op_id, listeners, None,
+                db_name, cmd_name, sock_info, op_id, listeners, None,
                 run.op_type, self.collection.codec_options)
 
             while run.idx_offset < len(run.ops):
+                cmd = SON([(cmd_name, self.collection.name),
+                           ('ordered', False),
+                           ('writeConcern', {'w': 0})])
+                sock_info.add_server_api(cmd)
                 ops = islice(run.ops, run.idx_offset, None)
                 # Run as many ops as possible.
-                to_send = bwc.execute_unack(ops, client)
+                to_send = bwc.execute_unack(cmd, ops, client)
                 run.idx_offset += len(to_send)
             self.current_run = run = next(generator, None)
 

pymongo/change_stream.py

@@ -41,11 +41,11 @@ _RESUMABLE_GETMORE_ERRORS = frozenset([
     189,    # PrimarySteppedDown
     262,    # ExceededTimeLimit
     9001,   # SocketException
-    10107,  # NotMaster
+    10107,  # NotWritablePrimary
     11600,  # InterruptedAtShutdown
     11602,  # InterruptedDueToReplStateChange
-    13435,  # NotMasterNoSlaveOk
-    13436,  # NotMasterOrSecondary
+    13435,  # NotPrimaryNoSecondaryOk
+    13436,  # NotPrimaryOrSecondary
     63,     # StaleShardVersion
     150,    # StaleEpoch
     13388,  # StaleConfig
@@ -64,7 +64,7 @@ class ChangeStream(object):
     :meth:`pymongo.mongo_client.MongoClient.watch` instead.
 
     .. versionadded:: 3.6
-    .. mongodoc:: changeStreams
+    .. seealso:: The MongoDB documentation on `changeStreams <https://dochub.mongodb.org/core/changeStreams>`_.
     """
     def __init__(self, target, pipeline, full_document, resume_after,
                  max_await_time_ms, batch_size, collation,
@@ -148,7 +148,8 @@ class ChangeStream(object):
         full_pipeline.extend(self._pipeline)
         return full_pipeline
 
-    def _process_result(self, result, session, server, sock_info, slave_ok):
+    def _process_result(
+            self, result, session, server, sock_info, secondary_ok):
         """Callback that caches the postBatchResumeToken or
         startAtOperationTime from a changeStream aggregate command response
         containing an empty batch of change documents.

pymongo/client_options.py

@@ -125,10 +125,12 @@ def _parse_pool_options(options):
     event_listeners = options.get('event_listeners')
     appname = options.get('appname')
     driver = options.get('driver')
+    server_api = options.get('server_api')
     compression_settings = CompressionSettings(
         options.get('compressors', []),
         options.get('zlibcompressionlevel', -1))
     ssl_context, ssl_match_hostname = _parse_ssl_options(options)
+    load_balanced = options.get('loadbalanced')
     return PoolOptions(max_pool_size,
                        min_pool_size,
                        max_idle_time_seconds,
@@ -138,7 +140,9 @@ def _parse_pool_options(options):
                        _EventListeners(event_listeners),
                        appname,
                        driver,
-                       compression_settings)
+                       compression_settings,
+                       server_api=server_api,
+                       load_balanced=load_balanced)
 
 
 class ClientOptions(object):
@@ -171,6 +175,7 @@ class ClientOptions(object):
         self.__server_selector = options.get(
             'server_selector', any_server_selector)
         self.__auto_encryption_opts = options.get('auto_encryption_opts')
+        self.__load_balanced = options.get('loadbalanced')
 
     @property
     def _options(self):
@@ -255,3 +260,8 @@ class ClientOptions(object):
     def auto_encryption_opts(self):
         """A :class:`~pymongo.encryption.AutoEncryptionOpts` or None."""
         return self.__auto_encryption_opts
+
+    @property
+    def load_balanced(self):
+        """True if the client was configured to connect to a load balancer."""
+        return self.__load_balanced

pymongo/client_session.py

@@ -37,13 +37,15 @@ the session are causally after previous read and write operations. Using a
 causally consistent session, an application can read its own writes and is
 guaranteed monotonic reads, even when reading from replica set secondaries.
 
-.. mongodoc:: causal-consistency
+.. seealso:: The MongoDB documentation on `causal-consistency <https://dochub.mongodb.org/core/causal-consistency>`_.
 
 .. _transactions-ref:
 
 Transactions
 ============
 
+.. versionadded:: 3.7
+
 MongoDB 4.0 adds support for transactions on replica set primaries. A
 transaction is associated with a :class:`ClientSession`. To start a transaction
 on a session, use :meth:`ClientSession.start_transaction` in a with-statement.
@@ -76,22 +78,56 @@ see the `MongoDB server's documentation for transactions
 A session may only have a single active transaction at a time, multiple
 transactions on the same session can be executed in sequence.
 
-.. versionadded:: 3.7
-
 Sharded Transactions
 ^^^^^^^^^^^^^^^^^^^^
 
+.. versionadded:: 3.9
+
 PyMongo 3.9 adds support for transactions on sharded clusters running MongoDB
-4.2. Sharded transactions have the same API as replica set transactions.
+>=4.2. Sharded transactions have the same API as replica set transactions.
 When running a transaction against a sharded cluster, the session is
 pinned to the mongos server selected for the first operation in the
 transaction. All subsequent operations that are part of the same transaction
 are routed to the same mongos server. When the transaction is completed, by
 running either commitTransaction or abortTransaction, the session is unpinned.
 
-.. versionadded:: 3.9
+.. seealso:: The MongoDB documentation on `transactions <https://dochub.mongodb.org/core/transactions>`_.
 
-.. mongodoc:: transactions
+.. _snapshot-reads-ref:
+
+Snapshot Reads
+==============
+
+.. versionadded:: 3.12
+
+MongoDB 5.0 adds support for snapshot reads. Snapshot reads are requested by
+passing the ``snapshot`` option to
+:meth:`~pymongo.mongo_client.MongoClient.start_session`.
+If ``snapshot`` is True, all read operations that use this session read data
+from the same snapshot timestamp. The server chooses the latest
+majority-committed snapshot timestamp when executing the first read operation
+using the session. Subsequent reads on this session read from the same
+snapshot timestamp. Snapshot reads are also supported when reading from
+replica set secondaries.
+
+.. code-block:: python
+
+  # Each read using this session reads data from the same point in time.
+  with client.start_session(snapshot=True) as session:
+      order = orders.find_one({"sku": "abc123"}, session=session)
+      inventory = inventory.find_one({"sku": "abc123"}, session=session)
+
+Snapshot Reads Limitations
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Snapshot reads sessions are incompatible with ``causal_consistency=True``.
+Only the following read operations are supported in a snapshot reads session:
+
+- :meth:`~pymongo.collection.Collection.find`
+- :meth:`~pymongo.collection.Collection.find_one`
+- :meth:`~pymongo.collection.Collection.aggregate`
+- :meth:`~pymongo.collection.Collection.count_documents`
+- :meth:`~pymongo.collection.Collection.distinct` (on unsharded collections)
 
 Classes
 =======
@@ -107,6 +143,7 @@ from bson.son import SON
 from bson.timestamp import Timestamp
 
 from pymongo import monotonic
+from pymongo.cursor import _SocketManager
 from pymongo.errors import (ConfigurationError,
                             ConnectionFailure,
                             InvalidOperation,
@@ -116,6 +153,7 @@ from pymongo.errors import (ConfigurationError,
 from pymongo.helpers import _RETRYABLE_ERROR_CODES
 from pymongo.read_concern import ReadConcern
 from pymongo.read_preferences import ReadPreference, _ServerMode
+from pymongo.server_type import SERVER_TYPE
 from pymongo.write_concern import WriteConcern
 
 
@@ -123,14 +161,29 @@ class SessionOptions(object):
     """Options for a new :class:`ClientSession`.
 
     :Parameters:
-      - `causal_consistency` (optional): If True (the default), read
-        operations are causally ordered within the session.
+      - `causal_consistency` (optional): If True, read operations are causally
+        ordered within the session. Defaults to True when the ``snapshot``
+        option is ``False``.
       - `default_transaction_options` (optional): The default
         TransactionOptions to use for transactions started on this session.
+      - `snapshot` (optional): If True, then all reads performed using this
+        session will read from the same snapshot. This option is incompatible
+        with ``causal_consistency=True``. Defaults to ``False``.
+
+    .. versionchanged:: 3.12
+       Added the ``snapshot`` parameter.
     """
     def __init__(self,
-                 causal_consistency=True,
-                 default_transaction_options=None):
+                 causal_consistency=None,
+                 default_transaction_options=None,
+                 snapshot=False):
+        if snapshot:
+            if causal_consistency:
+                raise ConfigurationError('snapshot reads do not support '
+                                         'causal_consistency=True')
+            causal_consistency = False
+        elif causal_consistency is None:
+            causal_consistency = True
         self._causal_consistency = causal_consistency
         if default_transaction_options is not None:
             if not isinstance(default_transaction_options, TransactionOptions):
@@ -139,6 +192,7 @@ class SessionOptions(object):
                     "pymongo.client_session.TransactionOptions, not: %r" %
                     (default_transaction_options,))
         self._default_transaction_options = default_transaction_options
+        self._snapshot = snapshot
 
     @property
     def causal_consistency(self):
@@ -154,6 +208,14 @@ class SessionOptions(object):
         """
         return self._default_transaction_options
 
+    @property
+    def snapshot(self):
+        """Whether snapshot reads are configured.
+
+        .. versionadded:: 3.12
+        """
+        return self._snapshot
+
 
 class TransactionOptions(object):
     """Options for :meth:`ClientSession.start_transaction`.
@@ -286,24 +348,55 @@ class _TxnState(object):
 
 class _Transaction(object):
     """Internal class to hold transaction information in a ClientSession."""
-    def __init__(self, opts):
+    def __init__(self, opts, client):
         self.opts = opts
         self.state = _TxnState.NONE
         self.sharded = False
         self.pinned_address = None
+        self.sock_mgr = None
         self.recovery_token = None
         self.attempt = 0
+        self.client = client
 
     def active(self):
         return self.state in (_TxnState.STARTING, _TxnState.IN_PROGRESS)
 
+    def starting(self):
+        return self.state == _TxnState.STARTING
+
+    @property
+    def pinned_conn(self):
+        if self.active() and self.sock_mgr:
+            return self.sock_mgr.sock
+        return None
+
+    def pin(self, server, sock_info):
+        self.sharded = True
+        self.pinned_address = server.description.address
+        if server.description.server_type == SERVER_TYPE.LoadBalancer:
+            sock_info.pin_txn()
+            self.sock_mgr = _SocketManager(sock_info, False)
+
+    def unpin(self):
+        self.pinned_address = None
+        if self.sock_mgr:
+            self.sock_mgr.close()
+        self.sock_mgr = None
+
     def reset(self):
+        self.unpin()
         self.state = _TxnState.NONE
         self.sharded = False
-        self.pinned_address = None
         self.recovery_token = None
         self.attempt = 0
 
+    def __del__(self):
+        if self.sock_mgr:
+            # Reuse the cursor closing machinery to return the socket to the
+            # pool soon.
+            self.client._close_cursor_soon(0, None, self.sock_mgr)
+            self.sock_mgr = None
+
 
 def _reraise_with_unknown_commit(exc):
     """Re-raise an exception with the UnknownTransactionCommitResult label."""
@@ -355,9 +448,10 @@ class ClientSession(object):
         self._authset = authset
         self._cluster_time = None
         self._operation_time = None
+        self._snapshot_time = None
         # Is this an implicitly created session?
         self._implicit = implicit
-        self._transaction = _Transaction(None)
+        self._transaction = _Transaction(None, client)
 
     def end_session(self):
         """Finish this session. If a transaction has started, abort it.
@@ -371,6 +465,9 @@ class ClientSession(object):
             try:
                 if self.in_transaction:
                     self.abort_transaction()
+                # It's possible we're still pinned here when the transaction
+                # is in the committed state when the session is discarded.
+                self._unpin()
             finally:
                 self._client._return_server_session(self._server_session, lock)
                 self._server_session = None
@@ -567,6 +664,10 @@ class ClientSession(object):
         """
         self._check_ended()
 
+        if self.options.snapshot:
+            raise InvalidOperation("Transactions are not supported in "
+                                   "snapshot sessions")
+
         if self.in_transaction:
             raise InvalidOperation("Transaction already in progress")
 
@@ -657,6 +758,7 @@ class ClientSession(object):
             pass
         finally:
             self._transaction.state = _TxnState.ABORTED
+            self._unpin()
 
     def _finish_transaction_with_retry(self, command_name):
         """Run commit or abort with one retry after any retryable error.
@@ -744,6 +846,12 @@ class ClientSession(object):
         """Process a response to a command that was run with this session."""
         self._advance_cluster_time(reply.get('$clusterTime'))
         self._advance_operation_time(reply.get('operationTime'))
+        if self._options.snapshot and self._snapshot_time is None:
+            if 'cursor' in reply:
+                ct = reply['cursor'].get('atClusterTime')
+            else:
+                ct = reply.get('atClusterTime')
+            self._snapshot_time = ct
         if self.in_transaction and self._transaction.sharded:
             recovery_token = reply.get('recoveryToken')
             if recovery_token:
@@ -762,6 +870,12 @@ class ClientSession(object):
         """
         return self._transaction.active()
 
+    @property
+    def _starting_transaction(self):
+        """True if this session is starting a multi-statement transaction.
+        """
+        return self._transaction.starting()
+
     @property
     def _pinned_address(self):
         """The mongos address this transaction was created on."""
@@ -769,14 +883,18 @@ class ClientSession(object):
             return self._transaction.pinned_address
         return None
 
-    def _pin_mongos(self, server):
-        """Pin this session to the given mongos Server."""
-        self._transaction.sharded = True
-        self._transaction.pinned_address = server.description.address
+    @property
+    def _pinned_connection(self):
+        """The connection this transaction was started on."""
+        return self._transaction.pinned_conn
 
-    def _unpin_mongos(self):
-        """Unpin this session from any pinned mongos address."""
-        self._transaction.pinned_address = None
+    def _pin(self, server, sock_info):
+        """Pin this session to the given Server or to the given connection."""
+        self._transaction.pin(server, sock_info)
+
+    def _unpin(self):
+        """Unpin this session from any pinned Server."""
+        self._transaction.unpin()
 
     def _txn_read_preference(self):
         """Return read preference of this transaction or None."""
@@ -784,15 +902,15 @@ class ClientSession(object):
             return self._transaction.opts.read_preference
         return None
 
-    def _apply_to(self, command, is_retryable, read_preference):
+    def _apply_to(self, command, is_retryable, read_preference, sock_info):
         self._check_ended()
 
+        if self.options.snapshot:
+            self._update_read_concern(command, sock_info)
+
         self._server_session.last_use = monotonic.time()
         command['lsid'] = self._server_session.session_id
 
-        if not self.in_transaction:
-            self._transaction.reset()
-
         if is_retryable:
             command['txnNumber'] = self._server_session.transaction_id
             return
@@ -810,15 +928,9 @@ class ClientSession(object):
 
                 if self._transaction.opts.read_concern:
                     rc = self._transaction.opts.read_concern.document
-                else:
-                    rc = {}
-
-                if (self.options.causal_consistency
-                        and self.operation_time is not None):
-                    rc['afterClusterTime'] = self.operation_time
-
-                if rc:
-                    command['readConcern'] = rc
+                    if rc:
+                        command['readConcern'] = rc
+                self._update_read_concern(command, sock_info)
 
             command['txnNumber'] = self._server_session.transaction_id
             command['autocommit'] = False
@@ -827,6 +939,20 @@ class ClientSession(object):
         self._check_ended()
         self._server_session.inc_transaction_id()
 
+    def _update_read_concern(self, cmd, sock_info):
+        if (self.options.causal_consistency
+                and self.operation_time is not None):
+            cmd.setdefault('readConcern', {})[
+                'afterClusterTime'] = self.operation_time
+        if self.options.snapshot:
+            if sock_info.max_wire_version < 13:
+                raise ConfigurationError(
+                    'Snapshot reads require MongoDB 5.0 or later')
+            rc = cmd.setdefault('readConcern', {})
+            rc['level'] = 'snapshot'
+            if self._snapshot_time is not None:
+                rc['atClusterTime'] = self._snapshot_time
+
 
 class _ServerSession(object):
     def __init__(self, generation):
@@ -896,9 +1022,11 @@ class _ServerSessionPool(collections.deque):
         return _ServerSession(self.generation)
 
     def return_server_session(self, server_session, session_timeout_minutes):
-        self._clear_stale(session_timeout_minutes)
-        if not server_session.timed_out(session_timeout_minutes):
-            self.return_server_session_no_lock(server_session)
+        if session_timeout_minutes is not None:
+            self._clear_stale(session_timeout_minutes)
+            if server_session.timed_out(session_timeout_minutes):
+                return
+        self.return_server_session_no_lock(server_session)
 
     def return_server_session_no_lock(self, server_session):
         # Discard sessions from an old pool to avoid duplicate sessions in the

pymongo/collection.py

@@ -154,7 +154,7 @@ class Collection(common.BaseObject):
         .. versionadded:: 2.1
            uuid_subtype attribute
 
-        .. mongodoc:: collections
+        .. seealso:: The MongoDB documentation on `collections <https://dochub.mongodb.org/core/collections>`_.
         """
         super(Collection, self).__init__(
             codec_options or database.codec_options,
@@ -197,7 +197,7 @@ class Collection(common.BaseObject):
     def _socket_for_writes(self, session):
         return self.__database.client._socket_for_writes(session)
 
-    def _command(self, sock_info, command, slave_ok=False,
+    def _command(self, sock_info, command, secondary_ok=False,
                  read_preference=None,
                  codec_options=None, check=True, allowable_errors=None,
                  read_concern=None,
@@ -211,7 +211,7 @@ class Collection(common.BaseObject):
         :Parameters:
           - `sock_info` - A SocketInfo instance.
           - `command` - The command itself, as a SON instance.
-          - `slave_ok`: whether to set the SlaveOkay wire protocol bit.
+          - `secondary_ok`: whether to set the secondaryOkay wire protocol bit.
           - `codec_options` (optional) - An instance of
             :class:`~bson.codec_options.CodecOptions`.
           - `check`: raise OperationFailure if there are errors
@@ -238,7 +238,7 @@ class Collection(common.BaseObject):
             return sock_info.command(
                 self.__database.name,
                 command,
-                slave_ok,
+                secondary_ok,
                 read_preference or self._read_preference_for(session),
                 codec_options or self.codec_options,
                 check,
@@ -303,6 +303,9 @@ class Collection(common.BaseObject):
     def __ne__(self, other):
         return not self == other
 
+    def __hash__(self):
+        return hash((self.__database, self.__name))
+
     @property
     def full_name(self):
         """The full name of this :class:`Collection`.
@@ -524,7 +527,8 @@ class Collection(common.BaseObject):
         if publish:
             duration = datetime.datetime.now() - start
             listeners.publish_command_start(
-                cmd, self.__database.name, rqst_id, sock_info.address, op_id)
+                cmd, self.__database.name, rqst_id, sock_info.address, op_id,
+                sock_info.service_id)
             start = datetime.datetime.now()
         try:
             result = sock_info.legacy_write(rqst_id, msg, max_size, False)
@@ -538,12 +542,14 @@ class Collection(common.BaseObject):
                         reply = message._convert_write_result(
                             name, cmd, details)
                         listeners.publish_command_success(
-                            dur, reply, name, rqst_id, sock_info.address, op_id)
+                            dur, reply, name, rqst_id, sock_info.address,
+                            op_id, sock_info.service_id)
                         raise
                 else:
                     details = message._convert_exception(exc)
                 listeners.publish_command_failure(
-                    dur, details, name, rqst_id, sock_info.address, op_id)
+                    dur, details, name, rqst_id, sock_info.address, op_id,
+                    sock_info.service_id)
             raise
         if publish:
             if result is not None:
@@ -553,7 +559,8 @@ class Collection(common.BaseObject):
                 reply = {'ok': 1}
             duration = (datetime.datetime.now() - start) + duration
             listeners.publish_command_success(
-                duration, reply, name, rqst_id, sock_info.address, op_id)
+                duration, reply, name, rqst_id, sock_info.address, op_id,
+                sock_info.service_id)
         return result
 
     def _insert_one(
@@ -605,7 +612,7 @@ class Collection(common.BaseObject):
         if not isinstance(doc, RawBSONDocument):
             return doc.get('_id')
 
-    def _insert(self, docs, ordered=True, check_keys=True,
+    def _insert(self, docs, ordered=True, check_keys=False,
                 manipulate=False, write_concern=None, op_id=None,
                 bypass_doc_val=False, session=None):
         """Internal insert helper."""
@@ -742,7 +749,9 @@ class Collection(common.BaseObject):
 
         .. versionadded:: 3.0
         """
-        if not isinstance(documents, abc.Iterable) or not documents:
+        if (not isinstance(documents, abc.Iterable)
+                or isinstance(documents, abc.Mapping)
+                or not documents):
             raise TypeError("documents must be a non-empty list")
         inserted_ids = []
         def gen():
@@ -762,7 +771,7 @@ class Collection(common.BaseObject):
         return InsertManyResult(inserted_ids, write_concern.acknowledged)
 
     def _update(self, sock_info, criteria, document, upsert=False,
-                check_keys=True, multi=False, manipulate=False,
+                check_keys=False, multi=False, manipulate=False,
                 write_concern=None, op_id=None, ordered=True,
                 bypass_doc_val=False, collation=None, array_filters=None,
                 hint=None, session=None, retryable_write=False):
@@ -851,7 +860,7 @@ class Collection(common.BaseObject):
 
     def _update_retryable(
             self, criteria, document, upsert=False,
-            check_keys=True, multi=False, manipulate=False,
+            check_keys=False, multi=False, manipulate=False,
             write_concern=None, op_id=None, ordered=True,
             bypass_doc_val=False, collation=None, array_filters=None,
             hint=None, session=None):
@@ -1517,8 +1526,9 @@ class Collection(common.BaseObject):
 
         .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500
 
-        .. mongodoc:: find
+        .. seealso:: The MongoDB documentation on `find <https://dochub.mongodb.org/core/find>`_.
 
+        .. seealso:: The MongoDB documentation on `find <https://dochub.mongodb.org/core/find>`_.
         """
         return Cursor(self, *args, **kwargs)
 
@@ -1538,17 +1548,16 @@ class Collection(common.BaseObject):
           >>> for batch in cursor:
           ...     print(bson.decode_all(batch))
 
-        .. note:: find_raw_batches does not support sessions or auto
-           encryption.
+        .. note:: find_raw_batches does not support auto encryption.
+
+        .. versionchanged:: 3.12
+           Instead of ignoring the user-specified read concern, this method
+           now sends it to the server when connected to MongoDB 3.6+.
+
+           Added session support.
 
         .. versionadded:: 3.6
         """
-        # OP_MSG with document stream returns is required to support
-        # sessions.
-        if "session" in kwargs:
-            raise ConfigurationError(
-                "find_raw_batches does not support sessions")
-
         # OP_MSG is required to support encryption.
         if self.__database.client._encrypter:
             raise InvalidOperation(
@@ -1621,13 +1630,13 @@ class Collection(common.BaseObject):
                    ('numCursors', num_cursors)])
         cmd.update(kwargs)
 
-        with self._socket_for_reads(session) as (sock_info, slave_ok):
+        with self._socket_for_reads(session) as (sock_info, secondary_ok):
             # We call sock_info.command here directly, instead of
             # calling self._command to avoid using an implicit session.
             result = sock_info.command(
                 self.__database.name,
                 cmd,
-                slave_ok,
+                secondary_ok,
                 self._read_preference_for(session),
                 self.codec_options,
                 read_concern=self.read_concern,
@@ -1643,38 +1652,49 @@ class Collection(common.BaseObject):
 
         return cursors
 
+    def _count_cmd(self, session, sock_info, secondary_ok, cmd, collation):
+        """Internal count command helper."""
+        # XXX: "ns missing" checks can be removed when we drop support for
+        # MongoDB 3.0, see SERVER-17051.
+        res = self._command(
+            sock_info,
+            cmd,
+            secondary_ok,
+            allowable_errors=["ns missing"],
+            codec_options=self.__write_response_codec_options,
+            read_concern=self.read_concern,
+            collation=collation,
+            session=session)
+        if res.get("errmsg", "") == "ns missing":
+            return 0
+        return int(res["n"])
+
     def _count(self, cmd, collation=None, session=None):
         """Internal count helper."""
         # XXX: "ns missing" checks can be removed when we drop support for
         # MongoDB 3.0, see SERVER-17051.
-        def _cmd(session, server, sock_info, slave_ok):
-            res = self._command(
-                sock_info,
-                cmd,
-                slave_ok,
-                allowable_errors=["ns missing"],
-                codec_options=self.__write_response_codec_options,
-                read_concern=self.read_concern,
-                collation=collation,
-                session=session)
-            if res.get("errmsg", "") == "ns missing":
-                return 0
-            return int(res["n"])
+        def _cmd(session, server, sock_info, secondary_ok):
+            return self._count_cmd(
+                session, sock_info, secondary_ok, cmd, collation)
 
         return self.__database.client._retryable_read(
             _cmd, self._read_preference_for(session), session)
 
     def _aggregate_one_result(
-            self, sock_info, slave_ok, cmd, collation=None, session=None):
+            self, sock_info, secondary_ok, cmd, collation, session):
         """Internal helper to run an aggregate that returns a single result."""
         result = self._command(
             sock_info,
             cmd,
-            slave_ok,
+            secondary_ok,
+            allowable_errors=[26],  # Ignore NamespaceNotFound.
             codec_options=self.__write_response_codec_options,
             read_concern=self.read_concern,
             collation=collation,
             session=session)
+        # cursor will not be present for NamespaceNotFound errors.
+        if 'cursor' not in result:
+            return None
         batch = result['cursor']['firstBatch']
         return batch[0] if batch else None
 
@@ -1699,9 +1719,31 @@ class Collection(common.BaseObject):
         if 'session' in kwargs:
             raise ConfigurationError(
                 'estimated_document_count does not support sessions')
-        cmd = SON([('count', self.__name)])
-        cmd.update(kwargs)
-        return self._count(cmd)
+
+        def _cmd(session, server, sock_info, secondary_ok):
+            if sock_info.max_wire_version >= 12:
+                # MongoDB 4.9+
+                pipeline = [
+                    {'$collStats': {'count': {}}},
+                    {'$group': {'_id': 1, 'n': {'$sum': '$count'}}},
+                ]
+                cmd = SON([('aggregate', self.__name),
+                           ('pipeline', pipeline),
+                           ('cursor', {})])
+                cmd.update(kwargs)
+                result = self._aggregate_one_result(
+                    sock_info, secondary_ok, cmd, collation=None, session=session)
+                if not result:
+                    return 0
+                return int(result['n'])
+            else:
+                # MongoDB < 4.9
+                cmd = SON([('count', self.__name)])
+                cmd.update(kwargs)
+                return self._count_cmd(None, sock_info, secondary_ok, cmd, None)
+
+        return self.__database.client._retryable_read(
+            _cmd, self.read_preference, None)
 
     def count_documents(self, filter, session=None, **kwargs):
         """Count the number of documents in this collection.
@@ -1775,9 +1817,9 @@ class Collection(common.BaseObject):
         collation = validate_collation_or_none(kwargs.pop('collation', None))
         cmd.update(kwargs)
 
-        def _cmd(session, server, sock_info, slave_ok):
+        def _cmd(session, server, sock_info, secondary_ok):
             result = self._aggregate_one_result(
-                sock_info, slave_ok, cmd, collation, session)
+                sock_info, secondary_ok, cmd, collation, session)
             if not result:
                 return 0
             return result['n']
@@ -2048,7 +2090,7 @@ class Collection(common.BaseObject):
            :meth:`create_index` no longer caches index names. Removed support
            for the drop_dups and bucket_size aliases.
 
-        .. mongodoc:: indexes
+        .. seealso:: The MongoDB documentation on `indexes <https://dochub.mongodb.org/core/indexes>`_.
 
         .. _wildcard index: https://docs.mongodb.com/master/core/index-wildcard/#wildcard-index-core
         """
@@ -2253,12 +2295,12 @@ class Collection(common.BaseObject):
         read_pref = ((session and session._txn_read_preference())
                      or ReadPreference.PRIMARY)
 
-        def _cmd(session, server, sock_info, slave_ok):
+        def _cmd(session, server, sock_info, secondary_ok):
             cmd = SON([("listIndexes", self.__name), ("cursor", {})])
             if sock_info.max_wire_version > 2:
                 with self.__database.client._tmp_session(session, False) as s:
                     try:
-                        cursor = self._command(sock_info, cmd, slave_ok,
+                        cursor = self._command(sock_info, cmd, secondary_ok,
                                                read_pref,
                                                codec_options,
                                                session=s)["cursor"]
@@ -2268,19 +2310,21 @@ class Collection(common.BaseObject):
                         if exc.code != 26:
                             raise
                         cursor = {'id': 0, 'firstBatch': []}
-                return CommandCursor(coll, cursor, sock_info.address,
-                                     session=s,
-                                     explicit_session=session is not None)
+                cmd_cursor = CommandCursor(
+                    coll, cursor, sock_info.address, session=s,
+                    explicit_session=session is not None)
             else:
                 res = message._first_batch(
                     sock_info, self.__database.name, "system.indexes",
-                    {"ns": self.__full_name}, 0, slave_ok, codec_options,
+                    {"ns": self.__full_name}, 0, secondary_ok, codec_options,
                     read_pref, cmd,
                     self.database.client._event_listeners)
                 cursor = res["cursor"]
                 # Note that a collection can only have 64 indexes, so there
                 # will never be a getMore call.
-                return CommandCursor(coll, cursor, sock_info.address)
+                cmd_cursor = CommandCursor(coll, cursor, sock_info.address)
+            cmd_cursor._maybe_pin_connection(sock_info)
+            return cmd_cursor
 
         return self.__database.client._retryable_read(
             _cmd, read_pref, session)
@@ -2380,24 +2424,6 @@ class Collection(common.BaseObject):
         """Perform an aggregation using the aggregation framework on this
         collection.
 
-        All optional `aggregate command`_ parameters should be passed as
-        keyword arguments to this method. Valid options include, but are not
-        limited to:
-
-          - `allowDiskUse` (bool): Enables writing to temporary files. When set
-            to True, aggregation stages can write data to the _tmp subdirectory
-            of the --dbpath directory. The default is False.
-          - `maxTimeMS` (int): The maximum amount of time to allow the operation
-            to run in milliseconds.
-          - `batchSize` (int): The maximum number of documents to return per
-            batch. Ignored if the connected mongod or mongos does not support
-            returning aggregate results using a cursor, or `useCursor` is
-            ``False``.
-          - `collation` (optional): An instance of
-            :class:`~pymongo.collation.Collation`. This option is only supported
-            on MongoDB 3.4 and above.
-          - `useCursor` (bool): Deprecated. Will be removed in PyMongo 4.0.
-
         The :meth:`aggregate` method obeys the :attr:`read_preference` of this
         :class:`Collection`, except when ``$out`` or ``$merge`` are used, in
         which case  :attr:`~pymongo.read_preferences.ReadPreference.PRIMARY`
@@ -2415,7 +2441,30 @@ class Collection(common.BaseObject):
           - `pipeline`: a list of aggregation pipeline stages
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
-          - `**kwargs` (optional): See list of options above.
+          - `**kwargs` (optional): extra `aggregate command`_ parameters.
+
+        All optional `aggregate command`_ parameters should be passed as
+        keyword arguments to this method. Valid options include, but are not
+        limited to:
+
+          - `allowDiskUse` (bool): Enables writing to temporary files. When set
+            to True, aggregation stages can write data to the _tmp subdirectory
+            of the --dbpath directory. The default is False.
+          - `maxTimeMS` (int): The maximum amount of time to allow the operation
+            to run in milliseconds.
+          - `batchSize` (int): The maximum number of documents to return per
+            batch. Ignored if the connected mongod or mongos does not support
+            returning aggregate results using a cursor, or `useCursor` is
+            ``False``.
+          - `collation` (optional): An instance of
+            :class:`~pymongo.collation.Collation`. This option is only supported
+            on MongoDB 3.4 and above.
+          - `let` (dict): A dict of parameter names and values. Values must be
+            constant or closed expressions that do not reference document
+            fields. Parameters can then be accessed as variables in an
+            aggregate expression context (e.g. ``"$$var"``). This option is
+            only supported on MongoDB >= 5.0.
+          - `useCursor` (bool): Deprecated. Will be removed in PyMongo 4.0.
 
         :Returns:
           A :class:`~pymongo.command_cursor.CommandCursor` over the result
@@ -2457,7 +2506,7 @@ class Collection(common.BaseObject):
                                    explicit_session=session is not None,
                                    **kwargs)
 
-    def aggregate_raw_batches(self, pipeline, **kwargs):
+    def aggregate_raw_batches(self, pipeline, session=None, **kwargs):
         """Perform an aggregation and retrieve batches of raw BSON.
 
         Similar to the :meth:`aggregate` method but returns a
@@ -2474,28 +2523,25 @@ class Collection(common.BaseObject):
           >>> for batch in cursor:
           ...     print(bson.decode_all(batch))
 
-        .. note:: aggregate_raw_batches does not support sessions or auto
-           encryption.
+        .. note:: aggregate_raw_batches does not support auto encryption.
+
+        .. versionchanged:: 3.12
+           Added session support.
 
         .. versionadded:: 3.6
         """
-        # OP_MSG with document stream returns is required to support
-        # sessions.
-        if "session" in kwargs:
-            raise ConfigurationError(
-                "aggregate_raw_batches does not support sessions")
-
         # OP_MSG is required to support encryption.
         if self.__database.client._encrypter:
             raise InvalidOperation(
                 "aggregate_raw_batches does not support auto encryption")
 
-        return self._aggregate(_CollectionRawAggregationCommand,
-                               pipeline,
-                               RawBatchCommandCursor,
-                               session=None,
-                               explicit_session=False,
-                               **kwargs)
+        with self.__database.client._tmp_session(session, close=False) as s:
+            return self._aggregate(_CollectionRawAggregationCommand,
+                                   pipeline,
+                                   RawBatchCommandCursor,
+                                   session=s,
+                                   explicit_session=session is not None,
+                                   **kwargs)
 
     def watch(self, pipeline=None, full_document=None, resume_after=None,
               max_await_time_ms=None, batch_size=None, collation=None,
@@ -2591,7 +2637,7 @@ class Collection(common.BaseObject):
 
         .. versionadded:: 3.6
 
-        .. mongodoc:: changeStreams
+        .. seealso:: The MongoDB documentation on `changeStreams <https://dochub.mongodb.org/core/changeStreams>`_.
 
         .. _change streams specification:
             https://github.com/mongodb/specifications/blob/master/source/change-streams/change-streams.rst
@@ -2636,8 +2682,8 @@ class Collection(common.BaseObject):
         collation = validate_collation_or_none(kwargs.pop('collation', None))
         cmd.update(kwargs)
 
-        with self._socket_for_reads(session=None) as (sock_info, slave_ok):
-            return self._command(sock_info, cmd, slave_ok,
+        with self._socket_for_reads(session=None) as (sock_info, secondary_ok):
+            return self._command(sock_info, cmd, secondary_ok,
                                  collation=collation,
                                  user_fields={'retval': 1})["retval"]
 
@@ -2740,9 +2786,9 @@ class Collection(common.BaseObject):
             kwargs["query"] = filter
         collation = validate_collation_or_none(kwargs.pop('collation', None))
         cmd.update(kwargs)
-        def _cmd(session, server, sock_info, slave_ok):
+        def _cmd(session, server, sock_info, secondary_ok):
             return self._command(
-                sock_info, cmd, slave_ok, read_concern=self.read_concern,
+                sock_info, cmd, secondary_ok, read_concern=self.read_concern,
                 collation=collation, session=session,
                 user_fields={"values": 1})["values"]
 
@@ -2769,7 +2815,7 @@ class Collection(common.BaseObject):
                      or read_pref)
 
         with self.__database.client._socket_for_reads(read_pref, session) as (
-                sock_info, slave_ok):
+                sock_info, secondary_ok):
             if (sock_info.max_wire_version >= 4 and
                     ('readConcern' not in cmd) and
                     inline):
@@ -2782,7 +2828,7 @@ class Collection(common.BaseObject):
                 write_concern = None
 
             return self._command(
-                sock_info, cmd, slave_ok, read_pref,
+                sock_info, cmd, secondary_ok, read_pref,
                 read_concern=read_concern,
                 write_concern=write_concern,
                 collation=collation, session=session,
@@ -2840,7 +2886,7 @@ class Collection(common.BaseObject):
 
         .. _map reduce command: http://docs.mongodb.org/manual/reference/command/mapReduce/
 
-        .. mongodoc:: mapreduce
+        .. seealso:: The MongoDB documentation on `mapreduce <https://dochub.mongodb.org/core/mapreduce>`_.
 
         """
         if not isinstance(out, (string_type, abc.Mapping)):

pymongo/command_cursor.py

@@ -16,14 +16,16 @@
 
 from collections import deque
 
+from bson import _convert_raw_document_lists_to_streams
 from bson.py3compat import integer_types
+from pymongo.cursor import _SocketManager, _CURSOR_CLOSED_ERRORS
 from pymongo.errors import (ConnectionFailure,
                             InvalidOperation,
-                            NotMasterError,
                             OperationFailure)
 from pymongo.message import (_CursorAddress,
                              _GetMore,
                              _RawBatchGetMore)
+from pymongo.response import PinnedResponse
 
 
 class CommandCursor(object):
@@ -37,6 +39,7 @@ class CommandCursor(object):
 
         The parameter 'retrieved' is unused.
         """
+        self.__sock_mgr = None
         self.__collection = collection
         self.__id = cursor_info['id']
         self.__data = deque(cursor_info['firstBatch'])
@@ -62,8 +65,7 @@ class CommandCursor(object):
             raise TypeError("max_await_time_ms must be an integer or None")
 
     def __del__(self):
-        if self.__id and not self.__killed:
-            self.__die()
+        self.__die()
 
     def __die(self, synchronous=False):
         """Closes this cursor.
@@ -71,16 +73,23 @@ class CommandCursor(object):
         already_killed = self.__killed
         self.__killed = True
         if self.__id and not already_killed:
+            cursor_id = self.__id
             address = _CursorAddress(
-                self.__address, self.__collection.full_name)
-            if synchronous:
-                self.__collection.database.client._close_cursor_now(
-                    self.__id, address, session=self.__session)
-            else:
-                # The cursor will be closed later in a different session.
-                self.__collection.database.client._close_cursor(
-                    self.__id, address)
-        self.__end_session(synchronous)
+                self.__address, self.__ns)
+        else:
+            # Skip killCursors.
+            cursor_id = 0
+            address = None
+        self.__collection.database.client._cleanup_cursor(
+            synchronous,
+            cursor_id,
+            address,
+            self.__sock_mgr,
+            self.__session,
+            self.__explicit_session)
+        if not self.__explicit_session:
+            self.__session = None
+        self.__sock_mgr = None
 
     def __end_session(self, synchronous):
         if self.__session and not self.__explicit_session:
@@ -127,52 +136,59 @@ class CommandCursor(object):
         changeStream aggregate or getMore."""
         return self.__postbatchresumetoken
 
+    def _maybe_pin_connection(self, sock_info):
+        client = self.__collection.database.client
+        if not client._should_pin_cursor(self.__session):
+            return
+        if not self.__sock_mgr:
+            sock_info.pin_cursor()
+            sock_mgr = _SocketManager(sock_info, False)
+            # Ensure the connection gets returned when the entire result is
+            # returned in the first batch.
+            if self.__id == 0:
+                sock_mgr.close()
+            else:
+                self.__sock_mgr = sock_mgr
+
     def __send_message(self, operation):
         """Send a getmore message and handle the response.
         """
-        def kill():
-            self.__killed = True
-            self.__end_session(True)
-
         client = self.__collection.database.client
         try:
-            response = client._run_operation_with_response(
+            response = client._run_operation(
                 operation, self._unpack_response, address=self.__address)
-        except OperationFailure:
-            kill()
-            raise
-        except NotMasterError:
-            # Don't send kill cursors to another server after a "not master"
-            # error. It's completely pointless.
-            kill()
+        except OperationFailure as exc:
+            if exc.code in _CURSOR_CLOSED_ERRORS:
+                # Don't send killCursors because the cursor is already closed.
+                self.__killed = True
+            # Return the session and pinned connection, if necessary.
+            self.close()
             raise
         except ConnectionFailure:
-            # Don't try to send kill cursors on another socket
-            # or to another server. It can cause a _pinValue
-            # assertion on some server releases if we get here
-            # due to a socket timeout.
-            kill()
+            # Don't send killCursors because the cursor is already closed.
+            self.__killed = True
+            # Return the session and pinned connection, if necessary.
+            self.close()
             raise
         except Exception:
-            # Close the cursor
-            self.__die()
+            self.close()
             raise
 
-        from_command = response.from_command
-        reply = response.data
-        docs = response.docs
-
-        if from_command:
-            cursor = docs[0]['cursor']
+        if isinstance(response, PinnedResponse):
+            if not self.__sock_mgr:
+                self.__sock_mgr = _SocketManager(response.socket_info,
+                                                 response.more_to_come)
+        if response.from_command:
+            cursor = response.docs[0]['cursor']
             documents = cursor['nextBatch']
             self.__postbatchresumetoken = cursor.get('postBatchResumeToken')
             self.__id = cursor['id']
         else:
-            documents = docs
-            self.__id = reply.cursor_id
+            documents = response.docs
+            self.__id = response.data.cursor_id
 
         if self.__id == 0:
-            kill()
+            self.close()
         self.__data = deque(documents)
 
     def _unpack_response(self, response, cursor_id, codec_options,
@@ -203,10 +219,9 @@ class CommandCursor(object):
                                     self.__session,
                                     self.__collection.database.client,
                                     self.__max_await_time_ms,
-                                    False))
+                                    self.__sock_mgr, False))
         else:  # Cursor id is zero nothing else to return
-            self.__killed = True
-            self.__end_session(True)
+            self.__die(True)
 
         return len(self.__data)
 
@@ -293,7 +308,7 @@ class RawBatchCommandCursor(CommandCursor):
         see :meth:`~pymongo.collection.Collection.aggregate_raw_batches`
         instead.
 
-        .. mongodoc:: cursors
+        .. seealso:: The MongoDB documentation on `cursors <https://dochub.mongodb.org/core/cursors>`_.
         """
         assert not cursor_info.get('firstBatch')
         super(RawBatchCommandCursor, self).__init__(
@@ -302,7 +317,13 @@ class RawBatchCommandCursor(CommandCursor):
 
     def _unpack_response(self, response, cursor_id, codec_options,
                          user_fields=None, legacy_response=False):
-        return response.raw_response(cursor_id)
+        raw_response = response.raw_response(
+            cursor_id, user_fields=user_fields)
+        if not legacy_response:
+            # OP_MSG returns firstBatch/nextBatch documents as a BSON array
+            # Re-assemble the array of documents into a document stream
+            _convert_raw_document_lists_to_streams(raw_response[0])
+        return raw_response
 
     def __getitem__(self, index):
         raise InvalidOperation("Cannot call __getitem__ on RawBatchCursor")

pymongo/common.py

@@ -27,6 +27,7 @@ from pymongo.auth import MECHANISMS
 from pymongo.compression_support import (validate_compressors,
                                          validate_zlib_compression_level)
 from pymongo.driver_info import DriverInfo
+from pymongo.server_api import ServerApi
 from pymongo.encryption_options import validate_auto_encryption_opts_or_none
 from pymongo.errors import ConfigurationError
 from pymongo.monitoring import _validate_event_listeners
@@ -57,9 +58,9 @@ MAX_WRITE_BATCH_SIZE = 1000
 # What this version of PyMongo supports.
 MIN_SUPPORTED_SERVER_VERSION = "2.6"
 MIN_SUPPORTED_WIRE_VERSION = 2
-MAX_SUPPORTED_WIRE_VERSION = 9
+MAX_SUPPORTED_WIRE_VERSION = 13
 
-# Frequency to call ismaster on servers, in seconds.
+# Frequency to call hello on servers, in seconds.
 HEARTBEAT_FREQUENCY = 10
 
 # Frequency to process kill-cursors, in seconds. See MongoClient.close_cursor.
@@ -74,7 +75,7 @@ EVENTS_QUEUE_FREQUENCY = 1
 # longest it is willing to wait for a new primary to be found.
 SERVER_SELECTION_TIMEOUT = 30
 
-# Spec requires at least 500ms between ismaster calls.
+# Spec requires at least 500ms between hello calls.
 MIN_HEARTBEAT_INTERVAL = 0.5
 
 # Spec requires at least 60s between SRV rescans.
@@ -131,13 +132,13 @@ def partition_node(node):
 
 
 def clean_node(node):
-    """Split and normalize a node name from an ismaster response."""
+    """Split and normalize a node name from a hello response."""
     host, port = partition_node(node)
 
     # Normalize hostname to lowercase, since DNS is case-insensitive:
     # http://tools.ietf.org/html/rfc4343
     # This prevents useless rediscovery if "foo.com" is in the seed list but
-    # "FOO.com" is in the ismaster response.
+    # "FOO.com" is in the hello response.
     return host.lower(), port
 
 
@@ -525,6 +526,15 @@ def validate_driver_or_none(option, value):
     return value
 
 
+def validate_server_api_or_none(option, value):
+    """Validate the server_api keyword arg."""
+    if value is None:
+        return value
+    if not isinstance(value, ServerApi):
+        raise TypeError("%s must be an instance of ServerApi" % (option,))
+    return value
+
+
 def validate_is_callable_or_none(option, value):
     """Validates that 'value' is a callable."""
     if value is None:
@@ -617,18 +627,21 @@ URI_OPTIONS_VALIDATOR_MAP = {
     'replicaset': validate_string_or_none,
     'retryreads': validate_boolean_or_string,
     'retrywrites': validate_boolean_or_string,
+    'loadbalanced': validate_boolean_or_string,
     'serverselectiontimeoutms': validate_timeout_or_zero,
     'sockettimeoutms': validate_timeout_or_none_or_zero,
     'ssl_keyfile': validate_readable,
     'tls': validate_boolean_or_string,
     'tlsallowinvalidcertificates': validate_allow_invalid_certs,
     'ssl_cert_reqs': validate_cert_reqs,
+    # Normalized to ssl_match_hostname which is the logical inverse of tlsallowinvalidhostnames
     'tlsallowinvalidhostnames': lambda *x: not validate_boolean_or_string(*x),
     'ssl_match_hostname': validate_boolean_or_string,
     'tlscafile': validate_readable,
     'tlscertificatekeyfile': validate_readable,
     'tlscertificatekeyfilepassword': validate_string_or_none,
-    'tlsdisableocspendpointcheck': validate_boolean_or_string,
+    # Normalized to ssl_check_ocsp_endpoint which is the logical inverse of tlsdisableocspendpointcheck
+    'tlsdisableocspendpointcheck': lambda *x: not validate_boolean_or_string(*x),
     'tlsinsecure': validate_boolean_or_string,
     'w': validate_non_negative_int_or_basestring,
     'wtimeoutms': validate_non_negative_integer,
@@ -640,6 +653,7 @@ URI_OPTIONS_VALIDATOR_MAP = {
 NONSPEC_OPTIONS_VALIDATOR_MAP = {
     'connect': validate_boolean_or_string,
     'driver': validate_driver_or_none,
+    'server_api': validate_server_api_or_none,
     'fsync': validate_boolean_or_string,
     'minpoolsize': validate_non_negative_integer,
     'socketkeepalive': validate_boolean_or_string,
@@ -699,6 +713,14 @@ URI_OPTIONS_DEPRECATION_MAP = {
     'ssl_match_hostname': ('renamed', 'tlsAllowInvalidHostnames'),
     'ssl_crlfile': ('renamed', 'tlsCRLFile'),
     'ssl_ca_certs': ('renamed', 'tlsCAFile'),
+    'ssl_certfile': ('removed', (
+        'Instead of using ssl_certfile to specify the certificate file, '
+        'use tlsCertificateKeyFile to pass a single file containing both '
+        'the client certificate and the private key')),
+    'ssl_keyfile': ('removed', (
+        'Instead of using ssl_keyfile to specify the private keyfile, '
+        'use tlsCertificateKeyFile to pass a single file containing both '
+        'the client certificate and the private key')),
     'ssl_pem_passphrase': ('renamed', 'tlsCertificateKeyFilePassword'),
     'waitqueuemultiple': ('removed', (
         'Instead of using waitQueueMultiple to bound queuing, limit the size '
@@ -957,4 +979,4 @@ class _CaseInsensitiveDictionary(abc.MutableMapping):
                 self[key] = other[key]
 
     def cased_key(self, key):
-        return self.__casedkeys[key.lower()]
+        return self.__casedkeys[key.lower()]

pymongo/compression_support.py

@@ -34,10 +34,11 @@ try:
 except ImportError:
     _HAVE_ZSTD = False
 
+from pymongo.hello_compat import HelloCompat
 from pymongo.monitoring import _SENSITIVE_COMMANDS
 
 _SUPPORTED_COMPRESSORS = set(["snappy", "zlib", "zstd"])
-_NO_COMPRESSION = set(['ismaster'])
+_NO_COMPRESSION = set([HelloCompat.CMD, HelloCompat.LEGACY_CMD])
 _NO_COMPRESSION.update(_SENSITIVE_COMMANDS)
 
 

pymongo/cursor.py

@@ -15,11 +15,12 @@
 """Cursor class to iterate over Mongo query results."""
 
 import copy
+import threading
 import warnings
 
 from collections import deque
 
-from bson import RE_TYPE
+from bson import RE_TYPE, _convert_raw_document_lists_to_streams
 from bson.code import Code
 from bson.py3compat import (iteritems,
                             integer_types,
@@ -30,19 +31,47 @@ from pymongo.common import validate_boolean, validate_is_mapping
 from pymongo.collation import validate_collation_or_none
 from pymongo.errors import (ConnectionFailure,
                             InvalidOperation,
-                            NotMasterError,
                             OperationFailure)
 from pymongo.message import (_CursorAddress,
                              _GetMore,
                              _RawBatchGetMore,
                              _Query,
                              _RawBatchQuery)
-from pymongo.monitoring import ConnectionClosedReason
+from pymongo.response import PinnedResponse
 
+# These errors mean that the server has already killed the cursor so there is
+# no need to send killCursors.
+_CURSOR_CLOSED_ERRORS = frozenset([
+    43,   # CursorNotFound
+    50,   # MaxTimeMSExpired
+    175,  # QueryPlanKilled
+    237,  # CursorKilled
+
+    # On a tailable cursor, the following errors mean the capped collection
+    # rolled over.
+    # MongoDB 2.6:
+    # {'$err': 'Runner killed during getMore', 'code': 28617, 'ok': 0}
+    28617,
+    # MongoDB 3.0:
+    # {'$err': 'getMore executor error: UnknownError no details available',
+    #  'code': 17406, 'ok': 0}
+    17406,
+    # MongoDB 3.2 + 3.4:
+    # {'ok': 0.0, 'errmsg': 'GetMore command executor error:
+    #  CappedPositionLost: CollectionScan died due to failure to restore
+    #  tailable cursor position. Last seen record id: RecordId(3)',
+    #  'code': 96}
+    96,
+    # MongoDB 3.6+:
+    # {'ok': 0.0, 'errmsg': 'errmsg: "CollectionScan died due to failure to
+    #  restore tailable cursor position. Last seen record id: RecordId(3)"',
+    #  'code': 136, 'codeName': 'CappedPositionLost'}
+    136,
+])
 
 _QUERY_OPTIONS = {
     "tailable_cursor": 2,
-    "slave_okay": 4,
+    "secondary_okay": 4,
     "oplog_replay": 8,
     "no_timeout": 16,
     "await_data": 32,
@@ -79,26 +108,25 @@ class CursorType(object):
     """
 
 
-# This has to be an old style class due to
-# http://bugs.jython.org/issue1057
-class _SocketManager:
+class _SocketManager(object):
     """Used with exhaust cursors to ensure the socket is returned.
     """
-    def __init__(self, sock, pool):
+    def __init__(self, sock, more_to_come):
         self.sock = sock
-        self.pool = pool
-        self.__closed = False
+        self.more_to_come = more_to_come
+        self.closed = False
+        self.lock = threading.Lock()
 
-    def __del__(self):
-        self.close()
+    def update_exhaust(self, more_to_come):
+        self.more_to_come = more_to_come
 
     def close(self):
         """Return this instance's socket to the connection pool.
         """
-        if not self.__closed:
-            self.__closed = True
-            self.pool.return_socket(self.sock)
-            self.sock, self.pool = None, None
+        if not self.closed:
+            self.closed = True
+            self.sock.unpin()
+            self.sock = None
 
 
 class Cursor(object):
@@ -113,21 +141,22 @@ class Cursor(object):
                  sort=None, allow_partial_results=False, oplog_replay=False,
                  modifiers=None, batch_size=0, manipulate=True,
                  collation=None, hint=None, max_scan=None, max_time_ms=None,
-                 max=None, min=None, return_key=False, show_record_id=False,
-                 snapshot=False, comment=None, session=None,
+                 max=None, min=None, return_key=None, show_record_id=None,
+                 snapshot=None, comment=None, session=None,
                  allow_disk_use=None):
         """Create a new cursor.
 
         Should not be called directly by application developers - see
         :meth:`~pymongo.collection.Collection.find` instead.
 
-        .. mongodoc:: cursors
+        .. seealso:: The MongoDB documentation on `cursors <https://dochub.mongodb.org/core/cursors>`_.
         """
         # Initialize all attributes used in __del__ before possibly raising
         # an error to avoid attribute errors during garbage collection.
+        self.__collection = collection
         self.__id = None
         self.__exhaust = False
-        self.__exhaust_mgr = None
+        self.__sock_mgr = None
         self.__killed = False
 
         if session:
@@ -147,6 +176,14 @@ class Cursor(object):
         if not isinstance(limit, int):
             raise TypeError("limit must be an instance of int")
         validate_boolean("no_cursor_timeout", no_cursor_timeout)
+        if no_cursor_timeout and not self.__explicit_session:
+            warnings.warn("use an explicit session with no_cursor_timeout=True "
+                          "otherwise the cursor may still timeout after "
+                          "30 minutes, for more info see "
+                          "https://docs.mongodb.com/v4.4/reference/method/"
+                          "cursor.noCursorTimeout/"
+                          "#session-idle-timeout-overrides-nocursortimeout",
+                          UserWarning, stacklevel=2)
         if cursor_type not in (CursorType.NON_TAILABLE, CursorType.TAILABLE,
                                CursorType.TAILABLE_AWAIT, CursorType.EXHAUST):
             raise ValueError("not a valid value for cursor_type")
@@ -169,7 +206,6 @@ class Cursor(object):
                 projection = {"_id": 1}
             projection = helpers._fields_list_to_dict(projection, "projection")
 
-        self.__collection = collection
         self.__spec = spec
         self.__projection = projection
         self.__skip = skip
@@ -255,6 +291,7 @@ class Cursor(object):
         be sent to the server, even if the resultant data has already been
         retrieved by this cursor.
         """
+        self.close()
         self.__data = deque()
         self.__id = None
         self.__address = None
@@ -311,27 +348,23 @@ class Cursor(object):
 
         self.__killed = True
         if self.__id and not already_killed:
-            if self.__exhaust and self.__exhaust_mgr:
-                # If this is an exhaust cursor and we haven't completely
-                # exhausted the result set we *must* close the socket
-                # to stop the server from sending more data.
-                self.__exhaust_mgr.sock.close_socket(
-                    ConnectionClosedReason.ERROR)
-            else:
-                address = _CursorAddress(
-                    self.__address, self.__collection.full_name)
-                if synchronous:
-                    self.__collection.database.client._close_cursor_now(
-                        self.__id, address, session=self.__session)
-                else:
-                    # The cursor will be closed later in a different session.
-                    self.__collection.database.client._close_cursor(
-                        self.__id, address)
-        if self.__exhaust and self.__exhaust_mgr:
-            self.__exhaust_mgr.close()
-        if self.__session and not self.__explicit_session:
-            self.__session._end_session(lock=synchronous)
+            cursor_id = self.__id
+            address = _CursorAddress(
+                self.__address, "%s.%s" % (self.__dbname, self.__collname))
+        else:
+            # Skip killCursors.
+            cursor_id = 0
+            address = None
+        self.__collection.database.client._cleanup_cursor(
+            synchronous,
+            cursor_id,
+            address,
+            self.__sock_mgr,
+            self.__session,
+            self.__explicit_session)
+        if not self.__explicit_session:
             self.__session = None
+        self.__sock_mgr = None
 
     def close(self):
         """Explicitly close / kill this cursor.
@@ -358,19 +391,19 @@ class Cursor(object):
             operators["$max"] = self.__max
         if self.__min:
             operators["$min"] = self.__min
-        if self.__return_key:
+        if self.__return_key is not None:
             operators["$returnKey"] = self.__return_key
-        if self.__show_record_id:
+        if self.__show_record_id is not None:
             # This is upgraded to showRecordId for MongoDB 3.2+ "find" command.
             operators["$showDiskLoc"] = self.__show_record_id
-        if self.__snapshot:
+        if self.__snapshot is not None:
             operators["$snapshot"] = self.__snapshot
 
         if operators:
             # Make a shallow copy so we can cleanly rewind or clone.
             spec = self.__spec.copy()
 
-            # White-listed commands must be wrapped in $query.
+            # Allow-listed commands must be wrapped in $query.
             if "$query" not in spec:
                 # $query has to come first
                 spec = SON([("$query", spec)])
@@ -470,7 +503,7 @@ class Cursor(object):
         :Parameters:
           - `limit`: the number of results to return
 
-        .. mongodoc:: limit
+        .. seealso:: The MongoDB documentation on `limit <https://dochub.mongodb.org/core/limit>`_.
         """
         if not isinstance(limit, integer_types):
             raise TypeError("limit must be an integer")
@@ -583,6 +616,18 @@ class Cursor(object):
     def __getitem__(self, index):
         """Get a single document or a slice of documents from this cursor.
 
+        .. warning:: A :class:`~Cursor` is not a Python :class:`list`. Each
+          index access or slice requires that a new query be run using skip
+          and limit. Do not iterate the cursor using index accesses.
+          The following example is **extremely inefficient** and may return
+          surprising results::
+
+            cursor = db.collection.find()
+            # Warning: This runs a new query for each document.
+            # Don't do this!
+            for idx in range(10):
+                print(cursor[idx])
+
         Raises :class:`~pymongo.errors.InvalidOperation` if this
         cursor has already been used.
 
@@ -862,7 +907,7 @@ class Cursor(object):
           :meth:`~pymongo.database.Database.command` to run the explain
           command directly.
 
-        .. mongodoc:: explain
+        .. seealso:: The MongoDB documentation on `explain <https://dochub.mongodb.org/core/explain>`_.
         """
         c = self.clone()
         c.__explain = True
@@ -996,49 +1041,35 @@ class Cursor(object):
                 "exhaust cursors do not support auto encryption")
 
         try:
-            response = client._run_operation_with_response(
-                operation, self._unpack_response, exhaust=self.__exhaust,
-                address=self.__address)
-        except OperationFailure:
-            self.__killed = True
-
-            # Make sure exhaust socket is returned immediately, if necessary.
-            self.__die()
-
+            response = client._run_operation(
+                operation, self._unpack_response, address=self.__address)
+        except OperationFailure as exc:
+            if exc.code in _CURSOR_CLOSED_ERRORS or self.__exhaust:
+                # Don't send killCursors because the cursor is already closed.
+                self.__killed = True
+            self.close()
             # If this is a tailable cursor the error is likely
             # due to capped collection roll over. Setting
             # self.__killed to True ensures Cursor.alive will be
             # False. No need to re-raise.
-            if self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]:
+            if (exc.code in _CURSOR_CLOSED_ERRORS and
+                    self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]):
                 return
             raise
-        except NotMasterError:
-            # Don't send kill cursors to another server after a "not master"
-            # error. It's completely pointless.
-            self.__killed = True
-
-            # Make sure exhaust socket is returned immediately, if necessary.
-            self.__die()
-
-            raise
         except ConnectionFailure:
-            # Don't try to send kill cursors on another socket
-            # or to another server. It can cause a _pinValue
-            # assertion on some server releases if we get here
-            # due to a socket timeout.
+            # Don't send killCursors because the cursor is already closed.
             self.__killed = True
-            self.__die()
+            self.close()
             raise
         except Exception:
-            # Close the cursor
-            self.__die()
+            self.close()
             raise
 
         self.__address = response.address
-        if self.__exhaust and not self.__exhaust_mgr:
-            # 'response' is an ExhaustResponse.
-            self.__exhaust_mgr = _SocketManager(response.socket_info,
-                                                response.pool)
+        if isinstance(response, PinnedResponse):
+            if not self.__sock_mgr:
+                self.__sock_mgr = _SocketManager(response.socket_info,
+                                                 response.more_to_come)
 
         cmd_name = operation.name
         docs = response.docs
@@ -1066,13 +1097,12 @@ class Cursor(object):
             self.__retrieved += response.data.number_returned
 
         if self.__id == 0:
-            self.__killed = True
             # Don't wait for garbage collection to call __del__, return the
             # socket and the session to the pool now.
-            self.__die()
+            self.close()
 
         if self.__limit and self.__id and self.__limit <= self.__retrieved:
-            self.__die()
+            self.close()
 
     def _unpack_response(self, response, cursor_id, codec_options,
                          user_fields=None, legacy_response=False):
@@ -1120,7 +1150,8 @@ class Cursor(object):
                                   self.__collation,
                                   self.__session,
                                   self.__collection.database.client,
-                                  self.__allow_disk_use)
+                                  self.__allow_disk_use,
+                                  self.__exhaust)
             self.__send_message(q)
         elif self.__id:  # Get More
             if self.__limit:
@@ -1129,7 +1160,6 @@ class Cursor(object):
                     limit = min(limit, self.__batch_size)
             else:
                 limit = self.__batch_size
-
             # Exhaust cursors don't send getMore messages.
             g = self._getmore_class(self.__dbname,
                                     self.__collname,
@@ -1140,7 +1170,8 @@ class Cursor(object):
                                     self.__session,
                                     self.__collection.database.client,
                                     self.__max_await_time_ms,
-                                    self.__exhaust_mgr)
+                                    self.__sock_mgr,
+                                    self.__exhaust)
             self.__send_message(g)
 
         return len(self.__data)
@@ -1282,7 +1313,7 @@ class RawBatchCursor(Cursor):
         see :meth:`~pymongo.collection.Collection.find_raw_batches`
         instead.
 
-        .. mongodoc:: cursors
+        .. seealso:: The MongoDB documentation on `cursors <https://dochub.mongodb.org/core/cursors>`_.
         """
         manipulate = kwargs.get('manipulate')
         kwargs['manipulate'] = False
@@ -1295,12 +1326,18 @@ class RawBatchCursor(Cursor):
 
     def _unpack_response(self, response, cursor_id, codec_options,
                          user_fields=None, legacy_response=False):
-        return response.raw_response(cursor_id)
+        raw_response = response.raw_response(
+            cursor_id, user_fields=user_fields)
+        if not legacy_response:
+            # OP_MSG returns firstBatch/nextBatch documents as a BSON array
+            # Re-assemble the array of documents into a document stream
+            _convert_raw_document_lists_to_streams(raw_response[0])
+        return raw_response
 
     def explain(self):
         """Returns an explain plan record for this cursor.
 
-        .. mongodoc:: explain
+        .. seealso:: The MongoDB documentation on `explain <https://dochub.mongodb.org/core/explain>`_.
         """
         clone = self._clone(deepcopy=True, base=Cursor(self.collection))
         return clone.explain()

pymongo/daemon.py

@@ -23,11 +23,14 @@ import os
 import subprocess
 import sys
 import time
+import warnings
+
 
 # The maximum amount of time to wait for the intermediate subprocess.
 _WAIT_TIMEOUT = 10
 _THIS_FILE = os.path.realpath(__file__)
 
+
 if sys.version_info[0] < 3:
     def _popen_wait(popen, timeout):
         """Implement wait timeout support for Python 2."""
@@ -66,7 +69,9 @@ def _silence_resource_warning(popen):
     # "ResourceWarning: subprocess XXX is still running".
     # See https://bugs.python.org/issue38890 and
     # https://bugs.python.org/issue26741.
-    popen.returncode = 0
+    # popen is None when mongocryptd spawning fails
+    if popen is not None:
+        popen.returncode = 0
 
 
 if sys.platform == 'win32':
@@ -75,12 +80,17 @@ if sys.platform == 'win32':
 
     def _spawn_daemon(args):
         """Spawn a daemon process (Windows)."""
-        with open(os.devnull, 'r+b') as devnull:
-            popen = subprocess.Popen(
-                args,
-                creationflags=_DETACHED_PROCESS,
-                stdin=devnull, stderr=devnull, stdout=devnull)
-            _silence_resource_warning(popen)
+        try:
+            with open(os.devnull, 'r+b') as devnull:
+                popen = subprocess.Popen(
+                    args,
+                    creationflags=_DETACHED_PROCESS,
+                    stdin=devnull, stderr=devnull, stdout=devnull)
+                _silence_resource_warning(popen)
+        except FileNotFoundError as exc:
+            warnings.warn('Failed to start %s: is it on your $PATH?\n'
+                          'Original exception: %s' % (args[0], exc),
+                          RuntimeWarning, stacklevel=2)
 else:
     # On Unix we spawn the daemon process with a double Popen.
     # 1) The first Popen runs this file as a Python script using the current
@@ -95,12 +105,16 @@ else:
     # we spawn the mongocryptd daemon process.
     def _spawn(args):
         """Spawn the process and silence stdout/stderr."""
-        with open(os.devnull, 'r+b') as devnull:
-            return subprocess.Popen(
-                args,
-                close_fds=True,
-                stdin=devnull, stderr=devnull, stdout=devnull)
-
+        try:
+            with open(os.devnull, 'r+b') as devnull:
+                return subprocess.Popen(
+                    args,
+                    close_fds=True,
+                    stdin=devnull, stderr=devnull, stdout=devnull)
+        except FileNotFoundError as exc:
+            warnings.warn('Failed to start %s: is it on your $PATH?\n'
+                          'Original exception: %s' % (args[0], exc),
+                          RuntimeWarning, stacklevel=2)
 
     def _spawn_daemon_double_popen(args):
         """Spawn a daemon process using a double subprocess.Popen."""

pymongo/database.py

@@ -30,6 +30,7 @@ from pymongo.errors import (CollectionInvalid,
                             ConfigurationError,
                             InvalidName,
                             OperationFailure)
+from pymongo.hello_compat import HelloCompat
 from pymongo.message import _first_batch
 from pymongo.read_preferences import ReadPreference
 from pymongo.son_manipulator import SONManipulator
@@ -80,7 +81,7 @@ class Database(common.BaseObject):
             :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the
             default) client.read_concern is used.
 
-        .. mongodoc:: databases
+        .. seealso:: The MongoDB documentation on `databases <https://dochub.mongodb.org/core/databases>`_.
 
         .. versionchanged:: 3.2
            Added the read_concern option.
@@ -272,6 +273,9 @@ class Database(common.BaseObject):
     def __ne__(self, other):
         return not self == other
 
+    def __hash__(self):
+        return hash((self.__client, self.__name))
+
     def __repr__(self):
         return "Database(%r, %r)" % (self.__client, self.__name)
 
@@ -353,18 +357,6 @@ class Database(common.BaseObject):
         creation. :class:`~pymongo.errors.CollectionInvalid` will be
         raised if the collection already exists.
 
-        Options should be passed as keyword arguments to this method. Supported
-        options vary with MongoDB release. Some examples include:
-
-          - "size": desired initial size for the collection (in
-            bytes). For capped collections this size is the max
-            size of the collection.
-          - "capped": if True, this is a capped collection
-          - "max": maximum number of objects if capped (optional)
-
-        See the MongoDB documentation for a full list of supported options by
-        server version.
-
         :Parameters:
           - `name`: the name of the collection to create
           - `codec_options` (optional): An instance of
@@ -387,7 +379,21 @@ class Database(common.BaseObject):
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
           - `**kwargs` (optional): additional keyword arguments will
-            be passed as options for the create collection command
+            be passed as options for the `create collection command`_
+
+        All optional `create collection command`_ parameters should be passed
+        as keyword arguments to this method. Valid options include, but are not
+        limited to:
+
+          - ``size``: desired initial size for the collection (in
+            bytes). For capped collections this size is the max
+            size of the collection.
+          - ``capped``: if True, this is a capped collection
+          - ``max``: maximum number of objects if capped (optional)
+          - ``timeseries``: a document specifying configuration options for
+            timeseries collections
+          - ``expireAfterSeconds``: the number of seconds after which a
+            document in a timeseries collection expires
 
         .. versionchanged:: 3.11
            This method is now supported inside multi-document transactions
@@ -404,6 +410,9 @@ class Database(common.BaseObject):
 
         .. versionchanged:: 2.2
            Removed deprecated argument: options
+
+        .. _create collection command:
+            https://docs.mongodb.com/manual/reference/command/create
         """
         with self.__client._tmp_session(session) as s:
             # Skip this check in a transaction where listCollections is not
@@ -468,21 +477,6 @@ class Database(common.BaseObject):
                for operation in cursor:
                    print(operation)
 
-        All optional `aggregate command`_ parameters should be passed as
-        keyword arguments to this method. Valid options include, but are not
-        limited to:
-
-          - `allowDiskUse` (bool): Enables writing to temporary files. When set
-            to True, aggregation stages can write data to the _tmp subdirectory
-            of the --dbpath directory. The default is False.
-          - `maxTimeMS` (int): The maximum amount of time to allow the operation
-            to run in milliseconds.
-          - `batchSize` (int): The maximum number of documents to return per
-            batch. Ignored if the connected mongod or mongos does not support
-            returning aggregate results using a cursor.
-          - `collation` (optional): An instance of
-            :class:`~pymongo.collation.Collation`.
-
         The :meth:`aggregate` method obeys the :attr:`read_preference` of this
         :class:`Database`, except when ``$out`` or ``$merge`` are used, in
         which case  :attr:`~pymongo.read_preferences.ReadPreference.PRIMARY`
@@ -498,7 +492,27 @@ class Database(common.BaseObject):
           - `pipeline`: a list of aggregation pipeline stages
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
-          - `**kwargs` (optional): See list of options above.
+          - `**kwargs` (optional): extra `aggregate command`_ parameters.
+
+        All optional `aggregate command`_ parameters should be passed as
+        keyword arguments to this method. Valid options include, but are not
+        limited to:
+
+          - `allowDiskUse` (bool): Enables writing to temporary files. When set
+            to True, aggregation stages can write data to the _tmp subdirectory
+            of the --dbpath directory. The default is False.
+          - `maxTimeMS` (int): The maximum amount of time to allow the operation
+            to run in milliseconds.
+          - `batchSize` (int): The maximum number of documents to return per
+            batch. Ignored if the connected mongod or mongos does not support
+            returning aggregate results using a cursor.
+          - `collation` (optional): An instance of
+            :class:`~pymongo.collation.Collation`.
+          - `let` (dict): A dict of parameter names and values. Values must be
+            constant or closed expressions that do not reference document
+            fields. Parameters can then be accessed as variables in an
+            aggregate expression context (e.g. ``"$$var"``). This option is
+            only supported on MongoDB >= 5.0.
 
         :Returns:
           A :class:`~pymongo.command_cursor.CommandCursor` over the result
@@ -602,7 +616,7 @@ class Database(common.BaseObject):
 
         .. versionadded:: 3.7
 
-        .. mongodoc:: changeStreams
+        .. seealso:: The MongoDB documentation on `changeStreams <https://dochub.mongodb.org/core/changeStreams>`_.
 
         .. _change streams specification:
             https://github.com/mongodb/specifications/blob/master/source/change-streams/change-streams.rst
@@ -612,8 +626,9 @@ class Database(common.BaseObject):
             batch_size, collation, start_at_operation_time, session,
             start_after)
 
-    def _command(self, sock_info, command, slave_ok=False, value=1, check=True,
-                 allowable_errors=None, read_preference=ReadPreference.PRIMARY,
+    def _command(self, sock_info, command, secondary_ok=False, value=1,
+                 check=True, allowable_errors=None,
+                 read_preference=ReadPreference.PRIMARY,
                  codec_options=DEFAULT_CODEC_OPTIONS,
                  write_concern=None,
                  parse_write_concern_error=False, session=None, **kwargs):
@@ -626,7 +641,7 @@ class Database(common.BaseObject):
             return sock_info.command(
                 self.__name,
                 command,
-                slave_ok,
+                secondary_ok,
                 read_preference,
                 codec_options,
                 check,
@@ -701,6 +716,12 @@ class Database(common.BaseObject):
         .. note:: :meth:`command` does **not** apply any custom TypeDecoders
            when decoding the command response.
 
+        .. note:: If this client has been configured to use MongoDB Versioned
+           API (see :ref:`versioned-api-ref`), then :meth:`command` will
+           automactically add API versioning options to the given command.
+           Explicitly adding API versioning options in the command and
+           declaring an API version on the client is not supported.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 
@@ -728,14 +749,14 @@ class Database(common.BaseObject):
 
         .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500
 
-        .. mongodoc:: commands
+        .. seealso:: The MongoDB documentation on `commands <https://dochub.mongodb.org/core/commands>`_.
         """
         if read_preference is None:
             read_preference = ((session and session._txn_read_preference())
                                or ReadPreference.PRIMARY)
         with self.__client._socket_for_reads(
-                read_preference, session) as (sock_info, slave_ok):
-            return self._command(sock_info, command, slave_ok, value,
+                read_preference, session) as (sock_info, secondary_ok):
+            return self._command(sock_info, command, secondary_ok, value,
                                  check, allowable_errors, read_preference,
                                  codec_options, session=session, **kwargs)
 
@@ -747,15 +768,15 @@ class Database(common.BaseObject):
             read_preference = ((session and session._txn_read_preference())
                                or ReadPreference.PRIMARY)
 
-        def _cmd(session, server, sock_info, slave_ok):
-            return self._command(sock_info, command, slave_ok, value,
+        def _cmd(session, server, sock_info, secondary_ok):
+            return self._command(sock_info, command, secondary_ok, value,
                                  check, allowable_errors, read_preference,
                                  codec_options, session=session, **kwargs)
 
         return self.__client._retryable_read(
             _cmd, read_preference, session)
 
-    def _list_collections(self, sock_info, slave_okay, session,
+    def _list_collections(self, sock_info, secondary_okay, session,
                           read_preference, **kwargs):
         """Internal listCollections helper."""
 
@@ -768,10 +789,10 @@ class Database(common.BaseObject):
             with self.__client._tmp_session(
                     session, close=False) as tmp_session:
                 cursor = self._command(
-                    sock_info, cmd, slave_okay,
+                    sock_info, cmd, secondary_okay,
                     read_preference=read_preference,
                     session=tmp_session)["cursor"]
-                return CommandCursor(
+                cmd_cursor = CommandCursor(
                     coll,
                     cursor,
                     sock_info.address,
@@ -790,11 +811,13 @@ class Database(common.BaseObject):
             cmd = SON([("aggregate", "system.namespaces"),
                        ("pipeline", pipeline),
                        ("cursor", kwargs.get("cursor", {}))])
-            cursor = self._command(sock_info, cmd, slave_okay)["cursor"]
-            return CommandCursor(coll, cursor, sock_info.address)
+            cursor = self._command(sock_info, cmd, secondary_okay)["cursor"]
+            cmd_cursor = CommandCursor(coll, cursor, sock_info.address)
+        cmd_cursor._maybe_pin_connection(sock_info)
+        return cmd_cursor
 
     def list_collections(self, session=None, filter=None, **kwargs):
-        """Get a cursor over the collectons of this database.
+        """Get a cursor over the collections of this database.
 
         :Parameters:
           - `session` (optional): a
@@ -817,9 +840,9 @@ class Database(common.BaseObject):
         read_pref = ((session and session._txn_read_preference())
                      or ReadPreference.PRIMARY)
 
-        def _cmd(session, server, sock_info, slave_okay):
+        def _cmd(session, server, sock_info, secondary_okay):
             return self._list_collections(
-                sock_info, slave_okay, session, read_preference=read_pref,
+                sock_info, secondary_okay, session, read_preference=read_pref,
                 **kwargs)
 
         return self.__client._retryable_read(
@@ -1058,7 +1081,18 @@ class Database(common.BaseObject):
         return self._current_op(include_all, session)
 
     def profiling_level(self, session=None):
-        """Get the database's current profiling level.
+        """**DEPRECATED**: Get the database's current profiling level.
+
+        Starting with PyMongo 3.12, this helper is obsolete. Instead, users
+        can run the `profile command`_, using the :meth:`command`
+        helper to get the current profiler level. Running the
+        `profile command`_ with the level set to ``-1`` returns the current
+        profiler information without changing it::
+
+           res = db.command("profile", -1)
+           profiling_level = res["was"]
+
+        The format of ``res`` depends on the version of MongoDB in use.
 
         Returns one of (:data:`~pymongo.OFF`,
         :data:`~pymongo.SLOW_ONLY`, :data:`~pymongo.ALL`).
@@ -1067,18 +1101,32 @@ class Database(common.BaseObject):
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
 
+        .. versionchanged:: 3.12
+           Deprecated.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 
-        .. mongodoc:: profiling
+        .. seealso:: The MongoDB documentation on `profiling <https://dochub.mongodb.org/core/profiling>`_.
+        .. _profile command: https://docs.mongodb.com/manual/reference/command/profile/
         """
+        warnings.warn("profiling_level() is deprecated. See the documentation "
+                      "for more information",
+                      DeprecationWarning, stacklevel=2)
         result = self.command("profile", -1, session=session)
 
         assert result["was"] >= 0 and result["was"] <= 2
         return result["was"]
 
-    def set_profiling_level(self, level, slow_ms=None, session=None):
-        """Set the database's profiling level.
+    def set_profiling_level(self, level, slow_ms=None, session=None,
+                            sample_rate=None, filter=None):
+        """**DEPRECATED**: Set the database's profiling level.
+
+        Starting with PyMongo 3.12, this helper is obsolete. Instead, users
+        can directly run the `profile command`_, using the :meth:`command`
+        helper, e.g.::
+
+           res = db.command("profile", 2, filter={"op": "query"})
 
         :Parameters:
           - `level`: Specifies a profiling level, see list of possible values
@@ -1088,6 +1136,10 @@ class Database(common.BaseObject):
             slower than the `slow_ms` level will get written to the logs.
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
+          - `sample_rate` (optional): The fraction of slow operations that
+            should be profiled or logged expressed as a float between 0 and 1.
+          - `filter` (optional): A filter expression that controls which
+            operations are profiled and logged.
 
         Possible `level` values:
 
@@ -1105,34 +1157,68 @@ class Database(common.BaseObject):
         (:data:`~pymongo.OFF`, :data:`~pymongo.SLOW_ONLY`,
         :data:`~pymongo.ALL`).
 
+        .. versionchanged:: 3.12
+           Added the ``sample_rate`` and ``filter`` parameters.
+           Deprecated.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 
-        .. mongodoc:: profiling
+        .. seealso:: The MongoDB documentation on `profiling <https://dochub.mongodb.org/core/profiling>`_.
+        .. _profile command: https://docs.mongodb.com/manual/reference/command/profile/
         """
+        warnings.warn("set_profiling_level() is deprecated. See the "
+                      "documentation for more information",
+                      DeprecationWarning, stacklevel=2)
+
         if not isinstance(level, int) or level < 0 or level > 2:
             raise ValueError("level must be one of (OFF, SLOW_ONLY, ALL)")
 
         if slow_ms is not None and not isinstance(slow_ms, int):
             raise TypeError("slow_ms must be an integer")
 
+        if sample_rate is not None and not isinstance(sample_rate, float):
+            raise TypeError(
+                "sample_rate must be a float, not %r" % (sample_rate,))
+
+        cmd = SON(profile=level)
         if slow_ms is not None:
-            self.command("profile", level, slowms=slow_ms, session=session)
-        else:
-            self.command("profile", level, session=session)
+            cmd['slowms'] = slow_ms
+        if sample_rate is not None:
+            cmd['sampleRate'] = sample_rate
+        if filter is not None:
+            cmd['filter'] = filter
+        self.command(cmd, session=session)
 
     def profiling_info(self, session=None):
-        """Returns a list containing current profiling information.
+        """**DEPRECATED**: Returns a list containing current profiling
+        information.
+
+        Starting with PyMongo 3.12, this helper is obsolete. Instead, users
+        can view the database profiler output by running
+        :meth:`~pymongo.collection.Collection.find` against the
+        ``system.profile`` collection as detailed in the `profiler output`_
+        documentation::
+
+           profiling_info = list(db["system.profile"].find())
 
         :Parameters:
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
 
+        .. versionchanged:: 3.12
+           Deprecated.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 
-        .. mongodoc:: profiling
+        .. seealso:: The MongoDB documentation on `profiling <https://dochub.mongodb.org/core/profiling>`_.
+        .. _profiler output: https://docs.mongodb.com/manual/reference/database-profiler/
         """
+        warnings.warn("profiling_info() is deprecated. See the "
+                      "documentation for more information",
+                      DeprecationWarning, stacklevel=2)
+
         return list(self["system.profile"].find(session=session))
 
     def error(self):
@@ -1152,7 +1238,7 @@ class Database(common.BaseObject):
         error_msg = error.get("err", "")
         if error_msg is None:
             return None
-        if error_msg.startswith("not master"):
+        if error_msg.startswith(HelloCompat.LEGACY_ERROR):
             # Reset primary server and request check, if another thread isn't
             # doing so already.
             primary = self.__client.primary
@@ -1463,7 +1549,7 @@ class Database(common.BaseObject):
            authentication fails due to invalid credentials or configuration
            issues.
 
-        .. mongodoc:: authenticate
+        .. seealso:: The MongoDB documentation on `authenticate <https://dochub.mongodb.org/core/authenticate>`_.
         """
         if name is not None and not isinstance(name, string_type):
             raise TypeError("name must be an "

pymongo/encryption.py

@@ -49,15 +49,19 @@ from pymongo.errors import (ConfigurationError,
 from pymongo.mongo_client import MongoClient
 from pymongo.pool import _configured_socket, PoolOptions
 from pymongo.read_concern import ReadConcern
-from pymongo.ssl_support import get_ssl_context
+from pymongo.ssl_support import get_ssl_context, HAVE_SSL
 from pymongo.uri_parser import parse_host
 from pymongo.write_concern import WriteConcern
 from pymongo.daemon import _spawn_daemon
 
+if HAVE_SSL:
+    from ssl import CERT_REQUIRED
+else:
+    CERT_REQUIRED = None
 
 _HTTPS_PORT = 443
 _KMS_CONNECT_TIMEOUT = 10  # TODO: CDRIVER-3262 will define this value.
-_MONGOCRYPTD_TIMEOUT_MS = 1000
+_MONGOCRYPTD_TIMEOUT_MS = 10000
 
 _DATA_KEY_OPTS = CodecOptions(document_class=SON, uuid_representation=STANDARD)
 # Use RawBSONDocument codec options to avoid needlessly decoding
@@ -107,7 +111,17 @@ class _EncryptionIO(MongoCryptCallback):
         endpoint = kms_context.endpoint
         message = kms_context.message
         host, port = parse_host(endpoint, _HTTPS_PORT)
-        ctx = get_ssl_context(None, None, None, None, None, None, True, True)
+        # Enable strict certificate verification, OCSP, match hostname, and
+        # SNI using the system default CA certificates.
+        ctx = get_ssl_context(
+            None,  # certfile
+            None,  # keyfile
+            None,  # passphrase
+            None,  # ca_certs
+            CERT_REQUIRED,  # cert_reqs
+            None,  # crlfile
+            True,  # match_hostname
+            True)  # check_ocsp_endpoint
         opts = PoolOptions(connect_timeout=_KMS_CONNECT_TIMEOUT,
                            socket_timeout=_KMS_CONNECT_TIMEOUT,
                            ssl_context=ctx)
@@ -116,6 +130,8 @@ class _EncryptionIO(MongoCryptCallback):
             conn.sendall(message)
             while kms_context.bytes_needed > 0:
                 data = conn.recv(kms_context.bytes_needed)
+                if not data:
+                    raise OSError('KMS connection closed')
                 kms_context.feed(data)
         finally:
             conn.close()
@@ -233,23 +249,57 @@ class _EncryptionIO(MongoCryptCallback):
 
 
 class _Encrypter(object):
-    def __init__(self, io_callbacks, opts):
-        """Encrypts and decrypts MongoDB commands.
+    """Encrypts and decrypts MongoDB commands.
 
-        This class is used to support automatic encryption and decryption of
-        MongoDB commands.
+    This class is used to support automatic encryption and decryption of
+    MongoDB commands."""
+    def __init__(self, client, opts):
+        """Create a _Encrypter for a client.
 
         :Parameters:
-          - `io_callbacks`: A :class:`MongoCryptCallback`.
+          - `client`: The encrypted MongoClient.
           - `opts`: The encrypted client's :class:`AutoEncryptionOpts`.
         """
         if opts._schema_map is None:
             schema_map = None
         else:
             schema_map = _dict_to_bson(opts._schema_map, False, _DATA_KEY_OPTS)
+        self._bypass_auto_encryption = opts._bypass_auto_encryption
+        self._internal_client = None
+
+        def _get_internal_client(encrypter, mongo_client):
+            if mongo_client.max_pool_size is None:
+                # Unlimited pool size, use the same client.
+                return mongo_client
+            # Else - limited pool size, use an internal client.
+            if encrypter._internal_client is not None:
+                return encrypter._internal_client
+            internal_client = mongo_client._duplicate(
+                minPoolSize=0, auto_encryption_opts=None)
+            encrypter._internal_client = internal_client
+            return internal_client
+
+        if opts._key_vault_client is not None:
+            key_vault_client = opts._key_vault_client
+        else:
+            key_vault_client = _get_internal_client(self, client)
+
+        if opts._bypass_auto_encryption:
+            metadata_client = None
+        else:
+            metadata_client = _get_internal_client(self, client)
+
+        db, coll = opts._key_vault_namespace.split('.', 1)
+        key_vault_coll = key_vault_client[db][coll]
+
+        mongocryptd_client = MongoClient(
+            opts._mongocryptd_uri, connect=False,
+            serverSelectionTimeoutMS=_MONGOCRYPTD_TIMEOUT_MS)
+
+        io_callbacks = _EncryptionIO(
+            metadata_client, key_vault_coll, mongocryptd_client, opts)
         self._auto_encrypter = AutoEncrypter(io_callbacks, MongoCryptOptions(
             opts._kms_providers, schema_map))
-        self._bypass_auto_encryption = opts._bypass_auto_encryption
         self._closed = False
 
     def encrypt(self, database, cmd, check_keys, codec_options):
@@ -299,29 +349,9 @@ class _Encrypter(object):
         """Cleanup resources."""
         self._closed = True
         self._auto_encrypter.close()
-
-    @staticmethod
-    def create(client, opts):
-        """Create a _CommandEncyptor for a client.
-
-        :Parameters:
-          - `client`: The encrypted MongoClient.
-          - `opts`: The encrypted client's :class:`AutoEncryptionOpts`.
-
-        :Returns:
-          A :class:`_CommandEncrypter` for this client.
-        """
-        key_vault_client = opts._key_vault_client or client
-        db, coll = opts._key_vault_namespace.split('.', 1)
-        key_vault_coll = key_vault_client[db][coll]
-
-        mongocryptd_client = MongoClient(
-            opts._mongocryptd_uri, connect=False,
-            serverSelectionTimeoutMS=_MONGOCRYPTD_TIMEOUT_MS)
-
-        io_callbacks = _EncryptionIO(
-            client, key_vault_coll, mongocryptd_client, opts)
-        return _Encrypter(io_callbacks, opts)
+        if self._internal_client:
+            self._internal_client.close()
+            self._internal_client = None
 
 
 class Algorithm(object):
@@ -357,7 +387,8 @@ class ClientEncryption(object):
 
               - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
                 These are the AWS access key ID and AWS secret access key used
-                to generate KMS messages.
+                to generate KMS messages. An optional "sessionToken" may be
+                included to support temporary AWS credentials.
               - `azure`: Map with "tenantId", "clientId", and "clientSecret" as
                 strings. Additionally, "identityPlatformEndpoint" may also be
                 specified as a string (defaults to 'login.microsoftonline.com').
@@ -462,7 +493,7 @@ class ClientEncryption(object):
               client_encryption.create_data_key("local", keyAltNames=["name1"])
               # reference the key with the alternate name
               client_encryption.encrypt("457-55-5462", keyAltName="name1",
-                                        algorithm=Algorithm.Random)
+                                        algorithm=Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Random)
 
         :Returns:
           The ``_id`` of the created data key document as a

pymongo/encryption_options.py

@@ -29,7 +29,8 @@ class AutoEncryptionOpts(object):
     """Options to configure automatic client-side field level encryption."""
 
     def __init__(self, kms_providers, key_vault_namespace,
-                 key_vault_client=None, schema_map=None,
+                 key_vault_client=None,
+                 schema_map=None,
                  bypass_auto_encryption=False,
                  mongocryptd_uri='mongodb://localhost:27020',
                  mongocryptd_bypass_spawn=False,
@@ -58,7 +59,8 @@ class AutoEncryptionOpts(object):
 
               - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
                 These are the AWS access key ID and AWS secret access key used
-                to generate KMS messages.
+                to generate KMS messages. An optional "sessionToken" may be
+                included to support temporary AWS credentials.
               - `azure`: Map with "tenantId", "clientId", and "clientSecret" as
                 strings. Additionally, "identityPlatformEndpoint" may also be
                 specified as a string (defaults to 'login.microsoftonline.com').

pymongo/errors.py

@@ -109,7 +109,22 @@ def _format_detailed_error(message, details):
 
 
 class NotMasterError(AutoReconnect):
-    """The server responded "not master" or "node is recovering".
+    """**DEPRECATED** - The server responded "not master" or
+    "node is recovering".
+
+    This exception has been deprecated and will be removed in PyMongo 4.0.
+    Use :exc:`~pymongo.errors.NotPrimaryError` instead.
+
+    .. versionchanged:: 3.12
+      Deprecated. Use :exc:`~pymongo.errors.NotPrimaryError` instead.
+    """
+    def __init__(self, message='', errors=None):
+        super(NotMasterError, self).__init__(
+            _format_detailed_error(message, errors), errors=errors)
+
+
+class NotPrimaryError(NotMasterError):
+    """The server responded "not primary" or "node is recovering".
 
     These errors result from a query, write, or command. The operation failed
     because the client thought it was using the primary but the primary has
@@ -120,10 +135,11 @@ class NotMasterError(AutoReconnect):
     its view of the server as soon as possible after throwing this exception.
 
     Subclass of :exc:`~pymongo.errors.AutoReconnect`.
+
+    .. versionadded:: 3.12
     """
     def __init__(self, message='', errors=None):
-        super(NotMasterError, self).__init__(
-            _format_detailed_error(message, errors), errors=errors)
+        super(NotPrimaryError, self).__init__(message, errors=errors)
 
 
 class ServerSelectionTimeoutError(AutoReconnect):
@@ -240,8 +256,9 @@ class BulkWriteError(OperationFailure):
     def __init__(self, results):
         super(BulkWriteError, self).__init__(
             "batch op errors occurred", 65, results)
-        # For pickle support
-        self.args = (results,)
+
+    def __reduce__(self):
+        return self.__class__, (self.details,)
 
 
 class InvalidOperation(PyMongoError):

pymongo/hello.py

@@ -0,0 +1,199 @@
+# Copyright 2021-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for the 'hello' and legacy hello commands."""
+
+import itertools
+
+from bson.py3compat import imap
+from pymongo import common
+from pymongo.hello_compat import HelloCompat
+from pymongo.server_type import SERVER_TYPE
+
+
+def _get_server_type(doc):
+    """Determine the server type from a hello response."""
+    if not doc.get('ok'):
+        return SERVER_TYPE.Unknown
+
+    if doc.get('serviceId'):
+        return SERVER_TYPE.LoadBalancer
+    elif doc.get('isreplicaset'):
+        return SERVER_TYPE.RSGhost
+    elif doc.get('setName'):
+        if doc.get('hidden'):
+            return SERVER_TYPE.RSOther
+        elif doc.get(HelloCompat.PRIMARY):
+            return SERVER_TYPE.RSPrimary
+        elif doc.get(HelloCompat.LEGACY_PRIMARY):
+            return SERVER_TYPE.RSPrimary
+        elif doc.get('secondary'):
+            return SERVER_TYPE.RSSecondary
+        elif doc.get('arbiterOnly'):
+            return SERVER_TYPE.RSArbiter
+        else:
+            return SERVER_TYPE.RSOther
+    elif doc.get('msg') == 'isdbgrid':
+        return SERVER_TYPE.Mongos
+    else:
+        return SERVER_TYPE.Standalone
+
+
+class Hello(object):
+    """Parse a hello response from the server."""
+    __slots__ = ('_doc', '_server_type', '_is_writable', '_is_readable',
+                 '_awaitable')
+
+    def __init__(self, doc, awaitable=False):
+        self._server_type = _get_server_type(doc)
+        self._doc = doc
+        self._is_writable = self._server_type in (
+            SERVER_TYPE.RSPrimary,
+            SERVER_TYPE.Standalone,
+            SERVER_TYPE.Mongos,
+            SERVER_TYPE.LoadBalancer)
+
+        self._is_readable = (
+            self.server_type == SERVER_TYPE.RSSecondary
+            or self._is_writable)
+        self._awaitable = awaitable
+
+    @property
+    def document(self):
+        """The complete hello command response document.
+
+        .. versionadded:: 3.4
+        """
+        return self._doc.copy()
+
+    @property
+    def server_type(self):
+        return self._server_type
+
+    @property
+    def all_hosts(self):
+        """List of hosts, passives, and arbiters known to this server."""
+        return set(imap(common.clean_node, itertools.chain(
+            self._doc.get('hosts', []),
+            self._doc.get('passives', []),
+            self._doc.get('arbiters', []))))
+
+    @property
+    def tags(self):
+        """Replica set member tags or empty dict."""
+        return self._doc.get('tags', {})
+
+    @property
+    def primary(self):
+        """This server's opinion about who the primary is, or None."""
+        if self._doc.get('primary'):
+            return common.partition_node(self._doc['primary'])
+        else:
+            return None
+
+    @property
+    def replica_set_name(self):
+        """Replica set name or None."""
+        return self._doc.get('setName')
+
+    @property
+    def max_bson_size(self):
+        return self._doc.get('maxBsonObjectSize', common.MAX_BSON_SIZE)
+
+    @property
+    def max_message_size(self):
+        return self._doc.get('maxMessageSizeBytes', 2 * self.max_bson_size)
+
+    @property
+    def max_write_batch_size(self):
+        return self._doc.get('maxWriteBatchSize', common.MAX_WRITE_BATCH_SIZE)
+
+    @property
+    def min_wire_version(self):
+        return self._doc.get('minWireVersion', common.MIN_WIRE_VERSION)
+
+    @property
+    def max_wire_version(self):
+        return self._doc.get('maxWireVersion', common.MAX_WIRE_VERSION)
+
+    @property
+    def set_version(self):
+        return self._doc.get('setVersion')
+
+    @property
+    def election_id(self):
+        return self._doc.get('electionId')
+
+    @property
+    def cluster_time(self):
+        return self._doc.get('$clusterTime')
+
+    @property
+    def logical_session_timeout_minutes(self):
+        return self._doc.get('logicalSessionTimeoutMinutes')
+
+    @property
+    def is_writable(self):
+        return self._is_writable
+
+    @property
+    def is_readable(self):
+        return self._is_readable
+
+    @property
+    def me(self):
+        me = self._doc.get('me')
+        if me:
+            return common.clean_node(me)
+
+    @property
+    def last_write_date(self):
+        return self._doc.get('lastWrite', {}).get('lastWriteDate')
+
+    @property
+    def compressors(self):
+        return self._doc.get('compression')
+
+    @property
+    def sasl_supported_mechs(self):
+        """Supported authentication mechanisms for the current user.
+
+        For example::
+
+            >>> hello.sasl_supported_mechs
+            ["SCRAM-SHA-1", "SCRAM-SHA-256"]
+
+        """
+        return self._doc.get('saslSupportedMechs', [])
+
+    @property
+    def speculative_authenticate(self):
+        """The speculativeAuthenticate field."""
+        return self._doc.get('speculativeAuthenticate')
+
+    @property
+    def topology_version(self):
+        return self._doc.get('topologyVersion')
+
+    @property
+    def awaitable(self):
+        return self._awaitable
+
+    @property
+    def service_id(self):
+        return self._doc.get('serviceId')
+
+    @property
+    def hello_ok(self):
+        return self._doc.get('helloOk', False)

pymongo/hello_compat.py

@@ -0,0 +1,23 @@
+# Copyright 2021-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compatibility enum of working with hello and legay hello."""
+
+class HelloCompat:
+    CMD = 'hello'
+    LEGACY_CMD = 'ismaster'
+    PRIMARY = 'isWritablePrimary'
+    LEGACY_PRIMARY = 'ismaster'
+    LEGACY_ERROR = 'not master'
+

pymongo/helpers.py

@@ -23,25 +23,27 @@ from pymongo import ASCENDING
 from pymongo.errors import (CursorNotFound,
                             DuplicateKeyError,
                             ExecutionTimeout,
-                            NotMasterError,
+                            NotPrimaryError,
                             OperationFailure,
                             WriteError,
                             WriteConcernError,
                             WTimeoutError)
+from pymongo.hello_compat import HelloCompat
 
 # From the SDAM spec, the "node is shutting down" codes.
 _SHUTDOWN_CODES = frozenset([
     11600,  # InterruptedAtShutdown
     91,     # ShutdownInProgress
 ])
-# From the SDAM spec, the "not master" error codes are combined with the
+# From the SDAM spec, the "not primary" error codes are combined with the
 # "node is recovering" error codes (of which the "node is shutting down"
 # errors are a subset).
 _NOT_MASTER_CODES = frozenset([
-    10107,  # NotMaster
-    13435,  # NotMasterNoSlaveOk
+    10058,  # LegacyNotPrimary <=3.2 "not primary" error code
+    10107,  # NotWritablePrimary
+    13435,  # NotPrimaryNoSecondaryOk
     11602,  # InterruptedDueToReplStateChange
-    13436,  # NotMasterOrSecondary
+    13436,  # NotPrimaryOrSecondary
     189,    # PrimarySteppedDown
 ]) | _SHUTDOWN_CODES
 # From the retryable writes spec.
@@ -115,7 +117,11 @@ def _check_command_response(response, max_wire_version,
                                max_wire_version)
 
     if parse_write_concern_error and 'writeConcernError' in response:
-        _raise_write_concern_error(response['writeConcernError'])
+        _error = response["writeConcernError"]
+        _labels = response.get("errorLabels")
+        if _labels:
+            _error.update({'errorLabels': _labels})
+        _raise_write_concern_error(_error)
 
     if response["ok"]:
         return
@@ -142,11 +148,12 @@ def _check_command_response(response, max_wire_version,
         elif errmsg in allowable_errors:
             return
 
-    # Server is "not master" or "recovering"
-    if code in _NOT_MASTER_CODES:
-        raise NotMasterError(errmsg, response)
-    elif "not master" in errmsg or "node is recovering" in errmsg:
-        raise NotMasterError(errmsg, response)
+    # Server is "not primary" or "recovering"
+    if code is not None:
+        if code in _NOT_MASTER_CODES:
+            raise NotPrimaryError(errmsg, response)
+    elif HelloCompat.LEGACY_ERROR in errmsg or "node is recovering" in errmsg:
+        raise NotPrimaryError(errmsg, response)
 
     # Other errors
     # findAndModify with upsert can raise duplicate key error
@@ -177,8 +184,8 @@ def _check_gle_response(result, max_wire_version):
     if error_msg is None:
         return result
 
-    if error_msg.startswith("not master"):
-        raise NotMasterError(error_msg, result)
+    if error_msg.startswith(HelloCompat.LEGACY_ERROR):
+        raise NotPrimaryError(error_msg, result)
 
     details = result
 
@@ -213,6 +220,18 @@ def _raise_write_concern_error(error):
         error.get("errmsg"), error.get("code"), error)
 
 
+def _get_wce_doc(result):
+    """Return the writeConcernError or None."""
+    wce = result.get("writeConcernError")
+    if wce:
+        # The server reports errorLabels at the top level but it's more
+        # convenient to attach it to the writeConcernError doc itself.
+        error_labels = result.get("errorLabels")
+        if error_labels:
+            wce["errorLabels"] = error_labels
+    return wce
+
+
 def _check_write_command_response(result):
     """Backward compatibility helper for write command error handling.
     """
@@ -221,9 +240,9 @@ def _check_write_command_response(result):
     if write_errors:
         _raise_last_write_error(write_errors)
 
-    error = result.get("writeConcernError")
-    if error:
-        _raise_write_concern_error(error)
+    wce = _get_wce_doc(result)
+    if wce:
+        _raise_write_concern_error(wce)
 
 
 def _raise_last_error(bulk_write_result):

pymongo/ismaster.py

@@ -12,174 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Parse a response to the 'ismaster' command."""
+"""**DEPRECATED** Parse a response to the 'ismaster' command.
 
-import itertools
+.. versionchanged:: 3.12
+  This module is deprecated and will be removed in PyMongo 4.0.
+"""
 
-from bson.py3compat import imap
-from pymongo import common
-from pymongo.server_type import SERVER_TYPE
+from pymongo.hello import *
 
+class IsMaster(Hello):
+    """**DEPRECATED** A hello response from the server.
 
-def _get_server_type(doc):
-    """Determine the server type from an ismaster response."""
-    if not doc.get('ok'):
-        return SERVER_TYPE.Unknown
-
-    if doc.get('isreplicaset'):
-        return SERVER_TYPE.RSGhost
-    elif doc.get('setName'):
-        if doc.get('hidden'):
-            return SERVER_TYPE.RSOther
-        elif doc.get('ismaster'):
-            return SERVER_TYPE.RSPrimary
-        elif doc.get('secondary'):
-            return SERVER_TYPE.RSSecondary
-        elif doc.get('arbiterOnly'):
-            return SERVER_TYPE.RSArbiter
-        else:
-            return SERVER_TYPE.RSOther
-    elif doc.get('msg') == 'isdbgrid':
-        return SERVER_TYPE.Mongos
-    else:
-        return SERVER_TYPE.Standalone
-
-
-class IsMaster(object):
-    __slots__ = ('_doc', '_server_type', '_is_writable', '_is_readable',
-                 '_awaitable')
-
-    def __init__(self, doc, awaitable=False):
-        """Parse an ismaster response from the server."""
-        self._server_type = _get_server_type(doc)
-        self._doc = doc
-        self._is_writable = self._server_type in (
-            SERVER_TYPE.RSPrimary,
-            SERVER_TYPE.Standalone,
-            SERVER_TYPE.Mongos)
-
-        self._is_readable = (
-            self.server_type == SERVER_TYPE.RSSecondary
-            or self._is_writable)
-        self._awaitable = awaitable
-
-    @property
-    def document(self):
-        """The complete ismaster command response document.
-
-        .. versionadded:: 3.4
-        """
-        return self._doc.copy()
-
-    @property
-    def server_type(self):
-        return self._server_type
-
-    @property
-    def all_hosts(self):
-        """List of hosts, passives, and arbiters known to this server."""
-        return set(imap(common.clean_node, itertools.chain(
-            self._doc.get('hosts', []),
-            self._doc.get('passives', []),
-            self._doc.get('arbiters', []))))
-
-    @property
-    def tags(self):
-        """Replica set member tags or empty dict."""
-        return self._doc.get('tags', {})
-
-    @property
-    def primary(self):
-        """This server's opinion about who the primary is, or None."""
-        if self._doc.get('primary'):
-            return common.partition_node(self._doc['primary'])
-        else:
-            return None
-
-    @property
-    def replica_set_name(self):
-        """Replica set name or None."""
-        return self._doc.get('setName')
-
-    @property
-    def max_bson_size(self):
-        return self._doc.get('maxBsonObjectSize', common.MAX_BSON_SIZE)
-
-    @property
-    def max_message_size(self):
-        return self._doc.get('maxMessageSizeBytes', 2 * self.max_bson_size)
-
-    @property
-    def max_write_batch_size(self):
-        return self._doc.get('maxWriteBatchSize', common.MAX_WRITE_BATCH_SIZE)
-
-    @property
-    def min_wire_version(self):
-        return self._doc.get('minWireVersion', common.MIN_WIRE_VERSION)
-
-    @property
-    def max_wire_version(self):
-        return self._doc.get('maxWireVersion', common.MAX_WIRE_VERSION)
-
-    @property
-    def set_version(self):
-        return self._doc.get('setVersion')
-
-    @property
-    def election_id(self):
-        return self._doc.get('electionId')
-
-    @property
-    def cluster_time(self):
-        return self._doc.get('$clusterTime')
-
-    @property
-    def logical_session_timeout_minutes(self):
-        return self._doc.get('logicalSessionTimeoutMinutes')
-
-    @property
-    def is_writable(self):
-        return self._is_writable
-
-    @property
-    def is_readable(self):
-        return self._is_readable
-
-    @property
-    def me(self):
-        me = self._doc.get('me')
-        if me:
-            return common.clean_node(me)
-
-    @property
-    def last_write_date(self):
-        return self._doc.get('lastWrite', {}).get('lastWriteDate')
-
-    @property
-    def compressors(self):
-        return self._doc.get('compression')
-
-    @property
-    def sasl_supported_mechs(self):
-        """Supported authentication mechanisms for the current user.
-
-        For example::
-
-            >>> ismaster.sasl_supported_mechs
-            ["SCRAM-SHA-1", "SCRAM-SHA-256"]
-
-        """
-        return self._doc.get('saslSupportedMechs', [])
-
-    @property
-    def speculative_authenticate(self):
-        """The speculativeAuthenticate field."""
-        return self._doc.get('speculativeAuthenticate')
-
-    @property
-    def topology_version(self):
-        return self._doc.get('topologyVersion')
-
-    @property
-    def awaitable(self):
-        return self._awaitable
+    .. versionchanged:: 3.12
+       Deprecated. Use :class:`~pymongo.hello.Hello` instead to parse
+       server hello responses.
+    """
+    pass

pymongo/message.py

@@ -12,12 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Tools for creating `messages
+"""**DEPRECATED** Tools for creating `messages
 <http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol>`_ to be sent to
 MongoDB.
 
 .. note:: This module is for internal use and is generally not needed by
    application developers.
+
+.. versionchanged:: 3.12
+  This module is deprecated and will be removed in PyMongo 4.0.
 """
 
 import datetime
@@ -28,10 +31,13 @@ import bson
 from bson import (CodecOptions,
                   decode,
                   encode,
+                  _decode_selective,
                   _dict_to_bson,
                   _make_c_string)
 from bson.codec_options import DEFAULT_CODEC_OPTIONS
-from bson.raw_bson import _inflate_bson, DEFAULT_RAW_BSON_OPTIONS
+from bson.int64 import Int64
+from bson.raw_bson import (_inflate_bson, DEFAULT_RAW_BSON_OPTIONS,
+                           RawBSONDocument)
 from bson.py3compat import b, StringIO
 from bson.son import SON
 
@@ -45,9 +51,10 @@ from pymongo.errors import (ConfigurationError,
                             DocumentTooLarge,
                             ExecutionTimeout,
                             InvalidOperation,
-                            NotMasterError,
+                            NotPrimaryError,
                             OperationFailure,
                             ProtocolError)
+from pymongo.hello_compat import HelloCompat
 from pymongo.read_concern import DEFAULT_READ_CONCERN
 from pymongo.read_preferences import ReadPreference
 from pymongo.write_concern import WriteConcern
@@ -100,7 +107,7 @@ def _maybe_add_read_preference(spec, read_preference):
     # problems with mongos versions that don't support read preferences. Also,
     # for maximum backwards compatibility, don't add $readPreference for
     # secondaryPreferred unless tags or maxStalenessSeconds are in use (setting
-    # the slaveOkay bit has the same effect).
+    # the secondaryOkay bit has the same effect).
     if mode and (
             mode != ReadPreference.SECONDARY_PREFERRED.mode or
             len(document) > 1):
@@ -234,16 +241,17 @@ class _Query(object):
     __slots__ = ('flags', 'db', 'coll', 'ntoskip', 'spec',
                  'fields', 'codec_options', 'read_preference', 'limit',
                  'batch_size', 'name', 'read_concern', 'collation',
-                 'session', 'client', 'allow_disk_use', '_as_command')
+                 'session', 'client', 'allow_disk_use', '_as_command',
+                 'exhaust')
 
     # For compatibility with the _GetMore class.
-    exhaust_mgr = None
+    sock_mgr = None
     cursor_id = None
 
     def __init__(self, flags, db, coll, ntoskip, spec, fields,
                  codec_options, read_preference, limit,
                  batch_size, read_concern, collation, session, client,
-                 allow_disk_use):
+                 allow_disk_use, exhaust):
         self.flags = flags
         self.db = db
         self.coll = coll
@@ -261,15 +269,18 @@ class _Query(object):
         self.allow_disk_use = allow_disk_use
         self.name = 'find'
         self._as_command = None
+        self.exhaust = exhaust
 
     def namespace(self):
         return _UJOIN % (self.db, self.coll)
 
-    def use_command(self, sock_info, exhaust):
+    def use_command(self, sock_info):
         use_find_cmd = False
-        if sock_info.max_wire_version >= 4:
-            if not exhaust:
-                use_find_cmd = True
+        if sock_info.max_wire_version >= 4 and not self.exhaust:
+            use_find_cmd = True
+        elif sock_info.max_wire_version >= 8:
+            # OP_MSG supports exhaust on MongoDB 4.2+
+            use_find_cmd = True
         elif not self.read_concern.ok_for_legacy:
             raise ConfigurationError(
                 'read concern level of %s is not valid '
@@ -307,15 +318,12 @@ class _Query(object):
             self.name = 'explain'
             cmd = SON([('explain', cmd)])
         session = self.session
+        sock_info.add_server_api(cmd)
         if session:
-            session._apply_to(cmd, False, self.read_preference)
+            session._apply_to(cmd, False, self.read_preference, sock_info)
             # Explain does not support readConcern.
-            if (not explain and session.options.causal_consistency
-                    and session.operation_time is not None
-                    and not session.in_transaction):
-                cmd.setdefault(
-                    'readConcern', {})[
-                    'afterClusterTime'] = session.operation_time
+            if not explain and not session.in_transaction:
+                session._update_read_concern(cmd, sock_info)
         sock_info.send_cluster_time(cmd, session, self.client)
         # Support auto encryption
         client = self.client
@@ -326,10 +334,10 @@ class _Query(object):
         self._as_command = cmd, self.db
         return self._as_command
 
-    def get_message(self, set_slave_ok, sock_info, use_cmd=False):
-        """Get a query message, possibly setting the slaveOk bit."""
-        if set_slave_ok:
-            # Set the slaveOk bit.
+    def get_message(self, set_secondary_ok, sock_info, use_cmd=False):
+        """Get a query message, possibly setting the secondaryOk bit."""
+        if set_secondary_ok:
+            # Set the secondaryOk bit.
             flags = self.flags | 4
         else:
             flags = self.flags
@@ -342,7 +350,7 @@ class _Query(object):
             if sock_info.op_msg_enabled:
                 request_id, msg, size, _ = _op_msg(
                     0, spec, self.db, self.read_preference,
-                    set_slave_ok, False, self.codec_options,
+                    set_secondary_ok, False, self.codec_options,
                     ctx=sock_info.compression_context)
                 return request_id, msg, size
             ns = _UJOIN % (self.db, "$cmd")
@@ -372,13 +380,13 @@ class _GetMore(object):
 
     __slots__ = ('db', 'coll', 'ntoreturn', 'cursor_id', 'max_await_time_ms',
                  'codec_options', 'read_preference', 'session', 'client',
-                 'exhaust_mgr', '_as_command')
+                 'sock_mgr', '_as_command', 'exhaust')
 
     name = 'getMore'
 
     def __init__(self, db, coll, ntoreturn, cursor_id, codec_options,
                  read_preference, session, client, max_await_time_ms,
-                 exhaust_mgr):
+                 sock_mgr, exhaust):
         self.db = db
         self.coll = coll
         self.ntoreturn = ntoreturn
@@ -388,15 +396,23 @@ class _GetMore(object):
         self.session = session
         self.client = client
         self.max_await_time_ms = max_await_time_ms
-        self.exhaust_mgr = exhaust_mgr
+        self.sock_mgr = sock_mgr
         self._as_command = None
+        self.exhaust = exhaust
 
     def namespace(self):
         return _UJOIN % (self.db, self.coll)
 
-    def use_command(self, sock_info, exhaust):
+    def use_command(self, sock_info):
+        use_cmd = False
+        if sock_info.max_wire_version >= 4 and not self.exhaust:
+            use_cmd = True
+        elif sock_info.max_wire_version >= 8:
+            # OP_MSG supports exhaust on MongoDB 4.2+
+            use_cmd = True
+
         sock_info.validate_session(self.client, self.session)
-        return sock_info.max_wire_version >= 4 and not exhaust
+        return use_cmd
 
     def as_command(self, sock_info):
         """Return a getMore command document for this query."""
@@ -409,7 +425,8 @@ class _GetMore(object):
                                     self.max_await_time_ms)
 
         if self.session:
-            self.session._apply_to(cmd, False, self.read_preference)
+            self.session._apply_to(cmd, False, self.read_preference, sock_info)
+        sock_info.add_server_api(cmd)
         sock_info.send_cluster_time(cmd, self.session, self.client)
         # Support auto encryption
         client = self.client
@@ -429,8 +446,12 @@ class _GetMore(object):
         if use_cmd:
             spec = self.as_command(sock_info)[0]
             if sock_info.op_msg_enabled:
+                if self.sock_mgr:
+                    flags = _OpMsg.EXHAUST_ALLOWED
+                else:
+                    flags = 0
                 request_id, msg, size, _ = _op_msg(
-                    0, spec, self.db, None,
+                    flags, spec, self.db, None,
                     False, False, self.codec_options,
                     ctx=sock_info.compression_context)
                 return request_id, msg, size
@@ -440,29 +461,29 @@ class _GetMore(object):
         return get_more(ns, self.ntoreturn, self.cursor_id, ctx)
 
 
-# TODO: Use OP_MSG once the server is able to respond with document streams.
 class _RawBatchQuery(_Query):
-    def use_command(self, socket_info, exhaust):
+    def use_command(self, sock_info):
         # Compatibility checks.
-        super(_RawBatchQuery, self).use_command(socket_info, exhaust)
-
+        super(_RawBatchQuery, self).use_command(sock_info)
+        if sock_info.max_wire_version >= 8:
+            # MongoDB 4.2+ supports exhaust over OP_MSG
+            return True
+        elif sock_info.op_msg_enabled and not self.exhaust:
+            return True
         return False
 
-    def get_message(self, set_slave_ok, sock_info, use_cmd=False):
-        # Always pass False for use_cmd.
-        return super(_RawBatchQuery, self).get_message(
-            set_slave_ok, sock_info, False)
-
 
 class _RawBatchGetMore(_GetMore):
-    def use_command(self, socket_info, exhaust):
+    def use_command(self, sock_info):
+        # Compatibility checks.
+        super(_RawBatchGetMore, self).use_command(sock_info)
+        if sock_info.max_wire_version >= 8:
+            # MongoDB 4.2+ supports exhaust over OP_MSG
+            return True
+        elif sock_info.op_msg_enabled and not self.exhaust:
+            return True
         return False
 
-    def get_message(self, set_slave_ok, sock_info, use_cmd=False):
-        # Always pass False for use_cmd.
-        return super(_RawBatchGetMore, self).get_message(
-            set_slave_ok, sock_info, False)
-
 
 class _CursorAddress(tuple):
     """The server address (host, port) of a cursor, with namespace property."""
@@ -581,7 +602,11 @@ if _use_c:
 
 def insert(collection_name, docs, check_keys,
            safe, last_error_args, continue_on_error, opts, ctx=None):
-    """Get an **insert** message."""
+    """**DEPRECATED** Get an **insert** message.
+
+    .. versionchanged:: 3.12
+      This function is deprecated and will be removed in PyMongo 4.0.
+    """
     if ctx:
         return _insert_compressed(
             collection_name, docs, check_keys, continue_on_error, opts, ctx)
@@ -631,7 +656,11 @@ if _use_c:
 
 def update(collection_name, upsert, multi, spec,
            doc, safe, last_error_args, check_keys, opts, ctx=None):
-    """Get an **update** message."""
+    """**DEPRECATED** Get an **update** message.
+
+    .. versionchanged:: 3.12
+      This function is deprecated and will be removed in PyMongo 4.0.
+    """
     if ctx:
         return _update_compressed(
             collection_name, upsert, multi, spec, doc, check_keys, opts, ctx)
@@ -689,13 +718,13 @@ if _use_c:
     _op_msg_uncompressed = _cmessage._op_msg
 
 
-def _op_msg(flags, command, dbname, read_preference, slave_ok, check_keys,
+def _op_msg(flags, command, dbname, read_preference, secondary_ok, check_keys,
             opts, ctx=None):
     """Get a OP_MSG message."""
     command['$db'] = dbname
     # getMore commands do not send $readPreference.
     if read_preference is not None and "$readPreference" not in command:
-        if slave_ok and not read_preference.mode:
+        if secondary_ok and not read_preference.mode:
             command["$readPreference"] = (
                 ReadPreference.PRIMARY_PREFERRED.document)
         else:
@@ -774,7 +803,11 @@ if _use_c:
 
 def query(options, collection_name, num_to_skip, num_to_return,
           query, field_selector, opts, check_keys=False, ctx=None):
-    """Get a **query** message."""
+    """**DEPRECATED** Get a **query** message.
+
+    .. versionchanged:: 3.12
+      This function is deprecated and will be removed in PyMongo 4.0.
+    """
     if ctx:
         return _query_compressed(options, collection_name, num_to_skip,
                                  num_to_return, query, field_selector,
@@ -811,7 +844,11 @@ if _use_c:
 
 
 def get_more(collection_name, num_to_return, cursor_id, ctx=None):
-    """Get a **getMore** message."""
+    """**DEPRECATED** Get a **getMore** message.
+
+    .. versionchanged:: 3.12
+      This function is deprecated and will be removed in PyMongo 4.0.
+    """
     if ctx:
         return _get_more_compressed(
             collection_name, num_to_return, cursor_id, ctx)
@@ -848,12 +885,15 @@ def _delete_uncompressed(
 
 def delete(
         collection_name, spec, safe, last_error_args, opts, flags=0, ctx=None):
-    """Get a **delete** message.
+    """**DEPRECATED** Get a **delete** message.
 
     `opts` is a CodecOptions. `flags` is a bit vector that may contain
     the SingleRemove flag or not:
 
     http://docs.mongodb.org/meta-driver/latest/legacy/mongodb-wire-protocol/#op-delete
+
+    .. versionchanged:: 3.12
+      This function is deprecated and will be removed in PyMongo 4.0.
     """
     if ctx:
         return _delete_compressed(collection_name, spec, opts, flags, ctx)
@@ -862,7 +902,10 @@ def delete(
 
 
 def kill_cursors(cursor_ids):
-    """Get a **killCursors** message.
+    """**DEPRECATED** Get a **killCursors** message.
+
+    .. versionchanged:: 3.12
+      This function is deprecated and will be removed in PyMongo 4.0.
     """
     num_cursors = len(cursor_ids)
     pack = struct.Struct("<ii" + ("q" * num_cursors)).pack
@@ -873,55 +916,55 @@ def kill_cursors(cursor_ids):
 class _BulkWriteContext(object):
     """A wrapper around SocketInfo for use with write splitting functions."""
 
-    __slots__ = ('db_name', 'command', 'sock_info', 'op_id',
+    __slots__ = ('db_name', 'sock_info', 'op_id',
                  'name', 'field', 'publish', 'start_time', 'listeners',
-                 'session', 'compress', 'op_type', 'codec')
+                 'session', 'compress', 'op_type', 'codec', 'cmd_legacy')
 
-    def __init__(self, database_name, command, sock_info, operation_id,
-                 listeners, session, op_type, codec):
+    def __init__(self, database_name, cmd_name, sock_info, operation_id,
+                 listeners, session, op_type, codec, cmd_legacy=None):
         self.db_name = database_name
-        self.command = command
         self.sock_info = sock_info
         self.op_id = operation_id
         self.listeners = listeners
         self.publish = listeners.enabled_for_commands
-        self.name = next(iter(command))
+        self.name = cmd_name
         self.field = _FIELD_MAP[self.name]
         self.start_time = datetime.datetime.now() if self.publish else None
         self.session = session
         self.compress = True if sock_info.compression_context else False
         self.op_type = op_type
         self.codec = codec
+        self.cmd_legacy = cmd_legacy
 
-    def _batch_command(self, docs):
+    def _batch_command(self, cmd, docs):
         namespace = self.db_name + '.$cmd'
         request_id, msg, to_send = _do_bulk_write_command(
-            namespace, self.op_type, self.command, docs, self.check_keys,
+            namespace, self.op_type, cmd, docs, self.check_keys,
             self.codec, self)
         if not to_send:
             raise InvalidOperation("cannot do an empty bulk write")
         return request_id, msg, to_send
 
-    def execute(self, docs, client):
-        request_id, msg, to_send = self._batch_command(docs)
-        result = self.write_command(request_id, msg, to_send)
+    def execute(self, cmd, docs, client):
+        request_id, msg, to_send = self._batch_command(cmd, docs)
+        result = self.write_command(cmd, request_id, msg, to_send)
         client._process_response(result, self.session)
         return result, to_send
 
-    def execute_unack(self, docs, client):
-        request_id, msg, to_send = self._batch_command(docs)
+    def execute_unack(self, cmd, docs, client):
+        request_id, msg, to_send = self._batch_command(cmd, docs)
         # Though this isn't strictly a "legacy" write, the helper
         # handles publishing commands and sending our message
         # without receiving a result. Send 0 for max_doc_size
         # to disable size checking. Size checking is handled while
         # the documents are encoded to BSON.
-        self.legacy_write(request_id, msg, 0, False, to_send)
+        self.legacy_write(cmd, request_id, msg, 0, False, to_send)
         return to_send
 
     @property
     def check_keys(self):
         """Should we check keys for this operation type?"""
-        return self.op_type == _INSERT
+        return False
 
     @property
     def max_bson_size(self):
@@ -952,14 +995,16 @@ class _BulkWriteContext(object):
             request_id, msg = _compress(
                 2002, msg, self.sock_info.compression_context)
         return self.legacy_write(
-            request_id, msg, max_doc_size, acknowledged, docs)
+            self.cmd_legacy.copy(), request_id, msg, max_doc_size,
+            acknowledged, docs)
 
-    def legacy_write(self, request_id, msg, max_doc_size, acknowledged, docs):
+    def legacy_write(self, cmd, request_id, msg, max_doc_size, acknowledged,
+                     docs):
         """A proxy for SocketInfo.legacy_write that handles event publishing.
         """
         if self.publish:
             duration = datetime.datetime.now() - self.start_time
-            cmd = self._start(request_id, docs)
+            cmd = self._start(cmd, request_id, docs)
             start = datetime.datetime.now()
         try:
             result = self.sock_info.legacy_write(
@@ -978,7 +1023,7 @@ class _BulkWriteContext(object):
                 if isinstance(exc, OperationFailure):
                     failure = _convert_write_result(
                         self.name, cmd, exc.details)
-                elif isinstance(exc, NotMasterError):
+                elif isinstance(exc, NotPrimaryError):
                     failure = exc.details
                 else:
                     failure = _convert_exception(exc)
@@ -988,12 +1033,12 @@ class _BulkWriteContext(object):
             self.start_time = datetime.datetime.now()
         return result
 
-    def write_command(self, request_id, msg, docs):
+    def write_command(self, cmd, request_id, msg, docs):
         """A proxy for SocketInfo.write_command that handles event publishing.
         """
         if self.publish:
             duration = datetime.datetime.now() - self.start_time
-            self._start(request_id, docs)
+            self._start(cmd, request_id, docs)
             start = datetime.datetime.now()
         try:
             reply = self.sock_info.write_command(request_id, msg)
@@ -1003,7 +1048,7 @@ class _BulkWriteContext(object):
         except Exception as exc:
             if self.publish:
                 duration = (datetime.datetime.now() - start) + duration
-                if isinstance(exc, (NotMasterError, OperationFailure)):
+                if isinstance(exc, (NotPrimaryError, OperationFailure)):
                     failure = exc.details
                 else:
                     failure = _convert_exception(exc)
@@ -1013,26 +1058,28 @@ class _BulkWriteContext(object):
             self.start_time = datetime.datetime.now()
         return reply
 
-    def _start(self, request_id, docs):
+    def _start(self, cmd, request_id, docs):
         """Publish a CommandStartedEvent."""
-        cmd = self.command.copy()
         cmd[self.field] = docs
         self.listeners.publish_command_start(
             cmd, self.db_name,
-            request_id, self.sock_info.address, self.op_id)
+            request_id, self.sock_info.address, self.op_id,
+            self.sock_info.service_id)
         return cmd
 
     def _succeed(self, request_id, reply, duration):
         """Publish a CommandSucceededEvent."""
         self.listeners.publish_command_success(
             duration, reply, self.name,
-            request_id, self.sock_info.address, self.op_id)
+            request_id, self.sock_info.address, self.op_id,
+            self.sock_info.service_id)
 
     def _fail(self, request_id, failure, duration):
         """Publish a CommandFailedEvent."""
         self.listeners.publish_command_failure(
             duration, failure, self.name,
-            request_id, self.sock_info.address, self.op_id)
+            request_id, self.sock_info.address, self.op_id,
+            self.sock_info.service_id)
 
 
 # From the Client Side Encryption spec:
@@ -1045,10 +1092,10 @@ _MAX_SPLIT_SIZE_ENC = 2097152
 class _EncryptedBulkWriteContext(_BulkWriteContext):
     __slots__ = ()
 
-    def _batch_command(self, docs):
+    def _batch_command(self, cmd, docs):
         namespace = self.db_name + '.$cmd'
         msg, to_send = _encode_batched_write_command(
-            namespace, self.op_type, self.command, docs, self.check_keys,
+            namespace, self.op_type, cmd, docs, self.check_keys,
             self.codec, self)
         if not to_send:
             raise InvalidOperation("cannot do an empty bulk write")
@@ -1059,17 +1106,18 @@ class _EncryptedBulkWriteContext(_BulkWriteContext):
                             DEFAULT_RAW_BSON_OPTIONS)
         return cmd, to_send
 
-    def execute(self, docs, client):
-        cmd, to_send = self._batch_command(docs)
+    def execute(self, cmd, docs, client):
+        batched_cmd, to_send = self._batch_command(cmd, docs)
         result = self.sock_info.command(
-            self.db_name, cmd, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS,
+            self.db_name, batched_cmd,
+            codec_options=_UNICODE_REPLACE_CODEC_OPTIONS,
             session=self.session, client=client)
         return result, to_send
 
-    def execute_unack(self, docs, client):
-        cmd, to_send = self._batch_command(docs)
+    def execute_unack(self, cmd, docs, client):
+        batched_cmd, to_send = self._batch_command(cmd, docs)
         self.sock_info.command(
-            self.db_name, cmd, write_concern=WriteConcern(w=0),
+            self.db_name, batched_cmd, write_concern=WriteConcern(w=0),
             session=self.session, client=client)
         return to_send
 
@@ -1485,16 +1533,16 @@ class _OpReply(object):
 
     def __init__(self, flags, cursor_id, number_returned, documents):
         self.flags = flags
-        self.cursor_id = cursor_id
+        self.cursor_id = Int64(cursor_id)
         self.number_returned = number_returned
         self.documents = documents
 
-    def raw_response(self, cursor_id=None):
+    def raw_response(self, cursor_id=None, user_fields=None):
         """Check the response header from the database, without decoding BSON.
 
         Check the response for errors and unpack.
 
-        Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
+        Can raise CursorNotFound, NotPrimaryError, ExecutionTimeout, or
         OperationFailure.
 
         :Parameters:
@@ -1516,8 +1564,8 @@ class _OpReply(object):
             error_object = bson.BSON(self.documents).decode()
             # Fake the ok field if it doesn't exist.
             error_object.setdefault("ok", 0)
-            if error_object["$err"].startswith("not master"):
-                raise NotMasterError(error_object["$err"], error_object)
+            if error_object["$err"].startswith(HelloCompat.LEGACY_ERROR):
+                raise NotPrimaryError(error_object["$err"], error_object)
             elif error_object.get("code") == 50:
                 raise ExecutionTimeout(error_object.get("$err"),
                                        error_object.get("code"),
@@ -1526,7 +1574,9 @@ class _OpReply(object):
                                    error_object.get("$err"),
                                    error_object.get("code"),
                                    error_object)
-        return [self.documents]
+        if self.documents:
+            return [self.documents]
+        return []
 
     def unpack_response(self, cursor_id=None,
                         codec_options=_UNICODE_REPLACE_CODEC_OPTIONS,
@@ -1536,7 +1586,7 @@ class _OpReply(object):
         Check the response for errors and unpack, returning a dictionary
         containing the response data.
 
-        Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
+        Can raise CursorNotFound, NotPrimaryError, ExecutionTimeout, or
         OperationFailure.
 
         :Parameters:
@@ -1597,8 +1647,15 @@ class _OpMsg(object):
         self.flags = flags
         self.payload_document = payload_document
 
-    def raw_response(self, cursor_id=None):
-        raise NotImplementedError
+    def raw_response(self, cursor_id=None, user_fields={}):
+        """
+        cursor_id is ignored
+        user_fields is used to determine which fields must not be decoded
+        """
+        inflated_response = _decode_selective(
+            RawBSONDocument(self.payload_document), user_fields,
+            DEFAULT_RAW_BSON_OPTIONS)
+        return [inflated_response]
 
     def unpack_response(self, cursor_id=None,
                         codec_options=_UNICODE_REPLACE_CODEC_OPTIONS,
@@ -1662,24 +1719,25 @@ _UNPACK_REPLY = {
 
 
 def _first_batch(sock_info, db, coll, query, ntoreturn,
-                 slave_ok, codec_options, read_preference, cmd, listeners):
+                 secondary_ok, codec_options, read_preference, cmd, listeners):
     """Simple query helper for retrieving a first (and possibly only) batch."""
     query = _Query(
         0, db, coll, 0, query, None, codec_options,
         read_preference, ntoreturn, 0, DEFAULT_READ_CONCERN, None, None,
-        None, None)
+        None, None, False)
 
     name = next(iter(cmd))
     publish = listeners.enabled_for_commands
     if publish:
         start = datetime.datetime.now()
 
-    request_id, msg, max_doc_size = query.get_message(slave_ok, sock_info)
+    request_id, msg, max_doc_size = query.get_message(secondary_ok, sock_info)
 
     if publish:
         encoding_duration = datetime.datetime.now() - start
         listeners.publish_command_start(
-            cmd, db, request_id, sock_info.address)
+            cmd, db, request_id, sock_info.address,
+            service_id=sock_info.service_id)
         start = datetime.datetime.now()
 
     sock_info.send_message(msg, max_doc_size)
@@ -1689,12 +1747,13 @@ def _first_batch(sock_info, db, coll, query, ntoreturn,
     except Exception as exc:
         if publish:
             duration = (datetime.datetime.now() - start) + encoding_duration
-            if isinstance(exc, (NotMasterError, OperationFailure)):
+            if isinstance(exc, (NotPrimaryError, OperationFailure)):
                 failure = exc.details
             else:
                 failure = _convert_exception(exc)
             listeners.publish_command_failure(
-                duration, failure, name, request_id, sock_info.address)
+                duration, failure, name, request_id, sock_info.address,
+                service_id=sock_info.service_id)
         raise
     # listIndexes
     if 'cursor' in cmd:
@@ -1713,6 +1772,7 @@ def _first_batch(sock_info, db, coll, query, ntoreturn,
     if publish:
         duration = (datetime.datetime.now() - start) + encoding_duration
         listeners.publish_command_success(
-            duration, result, name, request_id, sock_info.address)
+            duration, result, name, request_id, sock_info.address,
+            service_id=sock_info.service_id)
 
     return result

pymongo/mongo_client.py

@@ -59,10 +59,11 @@ from pymongo.errors import (AutoReconnect,
                             ConfigurationError,
                             ConnectionFailure,
                             InvalidOperation,
-                            NotMasterError,
+                            NotPrimaryError,
                             OperationFailure,
                             PyMongoError,
                             ServerSelectionTimeoutError)
+from pymongo.pool import ConnectionClosedReason
 from pymongo.read_preferences import ReadPreference
 from pymongo.server_selectors import (writable_preferred_server_selector,
                                       writable_server_selector)
@@ -73,7 +74,8 @@ from pymongo.topology_description import TOPOLOGY_TYPE
 from pymongo.settings import TopologySettings
 from pymongo.uri_parser import (_handle_option_deprecations,
                                 _handle_security_options,
-                                _normalize_options)
+                                _normalize_options,
+                                _check_options)
 from pymongo.write_concern import DEFAULT_WRITE_CONCERN
 
 
@@ -173,8 +175,8 @@ class MongoClient(common.BaseObject):
             from pymongo.errors import ConnectionFailure
             client = MongoClient()
             try:
-                # The ismaster command is cheap and does not require auth.
-                client.admin.command('ismaster')
+                # The ping command is cheap and does not require auth.
+                client.admin.command('ping')
             except ConnectionFailure:
                 print("Server not available")
 
@@ -195,9 +197,6 @@ class MongoClient(common.BaseObject):
           - `port` (optional): port number on which to connect
           - `document_class` (optional): default class to use for
             documents returned from queries on this client
-          - `type_registry` (optional): instance of
-            :class:`~bson.codec_options.TypeRegistry` to enable encoding
-            and decoding of custom types.
           - `tz_aware` (optional): if ``True``,
             :class:`~datetime.datetime` instances returned as values
             in a document by this :class:`MongoClient` will be timezone
@@ -205,15 +204,18 @@ class MongoClient(common.BaseObject):
           - `connect` (optional): if ``True`` (the default), immediately
             begin connecting to MongoDB in the background. Otherwise connect
             on the first operation.
+          - `type_registry` (optional): instance of
+            :class:`~bson.codec_options.TypeRegistry` to enable encoding
+            and decoding of custom types.
+
+          | **Other optional parameters can be passed as keyword arguments:**
+
           - `directConnection` (optional): if ``True``, forces this client to
              connect directly to the specified MongoDB host as a standalone.
              If ``false``, the client connects to the entire replica set of
              which the given MongoDB host(s) is a part. If this is ``True``
              and a mongodb+srv:// URI or a URI containing multiple seeds is
              provided, an exception will be raised.
-
-          | **Other optional parameters can be passed as keyword arguments:**
-
           - `maxPoolSize` (optional): The maximum allowable number of
             concurrent connections to each connected server. Requests to a
             server will block if there are `maxPoolSize` outstanding
@@ -340,10 +342,14 @@ class MongoClient(common.BaseObject):
             speed. 9 is best compression. Defaults to -1.
           - `uuidRepresentation`: The BSON representation to use when encoding
             from and decoding to instances of :class:`~uuid.UUID`. Valid
-            values are `pythonLegacy` (the default), `javaLegacy`,
-            `csharpLegacy`, `standard` and `unspecified`. New applications
-            should consider setting this to `standard` for cross language
+            values are the strings: "pythonLegacy" (the default), "javaLegacy",
+            "csharpLegacy", "standard" and "unspecified". New applications
+            should consider setting this to "standard" for cross language
             compatibility. See :ref:`handling-uuid-data-example` for details.
+          - `unicode_decode_error_handler`: The error handler to apply when
+            a Unicode-related error occurs during BSON decoding that would
+            otherwise raise :exc:`UnicodeDecodeError`. Valid options include
+            'strict', 'replace', and 'ignore'. Defaults to 'strict'.
 
           | **Write Concern options:**
           | (Only set if passed. No default values.)
@@ -446,39 +452,29 @@ class MongoClient(common.BaseObject):
             ``tlsAllowInvalidCertificates=False`` implies ``tls=True``.
             Defaults to ``False``. Think very carefully before setting this
             to ``True`` as that could make your application vulnerable to
-            man-in-the-middle attacks.
+            on-path attackers.
           - `tlsAllowInvalidHostnames`: (boolean) If ``True``, disables TLS
             hostname verification. ``tlsAllowInvalidHostnames=False`` implies
             ``tls=True``. Defaults to ``False``. Think very carefully before
             setting this to ``True`` as that could make your application
-            vulnerable to man-in-the-middle attacks.
+            vulnerable to on-path attackers.
           - `tlsCAFile`: A file containing a single or a bundle of
             "certification authority" certificates, which are used to validate
             certificates passed from the other end of the connection.
             Implies ``tls=True``. Defaults to ``None``.
           - `tlsCertificateKeyFile`: A file containing the client certificate
-            and private key. If you want to pass the certificate and private
-            key as separate files, use the ``ssl_certfile`` and ``ssl_keyfile``
-            options instead. Implies ``tls=True``. Defaults to ``None``.
+            and private key. Implies ``tls=True``. Defaults to ``None``.
           - `tlsCRLFile`: A file containing a PEM or DER formatted
             certificate revocation list. Only supported by python 2.7.9+
             (pypy 2.5.1+). Implies ``tls=True``. Defaults to ``None``.
           - `tlsCertificateKeyFilePassword`: The password or passphrase for
-            decrypting the private key in ``tlsCertificateKeyFile`` or
-            ``ssl_keyfile``. Only necessary if the private key is encrypted.
-            Only supported by python 2.7.9+ (pypy 2.5.1+) and 3.3+. Defaults
-            to ``None``.
+            decrypting the private key in ``tlsCertificateKeyFile``. Only
+            necessary if the private key is encrypted. Only supported by
+            python 2.7.9+ (pypy 2.5.1+) and 3.3+. Defaults to ``None``.
           - `tlsDisableOCSPEndpointCheck`: (boolean) If ``True``, disables
             certificate revocation status checking via the OCSP responder
             specified on the server certificate. Defaults to ``False``.
           - `ssl`: (boolean) Alias for ``tls``.
-          - `ssl_certfile`: The certificate file used to identify the local
-            connection against mongod. Implies ``tls=True``. Defaults to
-            ``None``.
-          - `ssl_keyfile`: The private keyfile used to identify the local
-            connection against mongod. Can be omitted if the keyfile is
-            included with the ``tlsCertificateKeyFile``. Implies ``tls=True``.
-            Defaults to ``None``.
 
           | **Read Concern options:**
           | (If not set explicitly, this will use the server default)
@@ -497,8 +493,32 @@ class MongoClient(common.BaseObject):
             configures this client to automatically encrypt collection commands
             and automatically decrypt results. See
             :ref:`automatic-client-side-encryption` for an example.
+            If a :class:`MongoClient` is configured with
+            ``auto_encryption_opts`` and a non-None ``maxPoolSize``, a
+            separate internal ``MongoClient`` is created if any of the
+            following are true:
 
-        .. mongodoc:: connections
+              - A ``key_vault_client`` is not passed to
+                :class:`~pymongo.encryption_options.AutoEncryptionOpts`
+              - ``bypass_auto_encrpytion=False`` is passed to
+                :class:`~pymongo.encryption_options.AutoEncryptionOpts`
+
+          | **Versioned API options:**
+          | (If not set explicitly, Versioned API will not be enabled.)
+
+          - `server_api`: A
+            :class:`~pymongo.server_api.ServerApi` which configures this
+            client to use Versioned API. See :ref:`versioned-api-ref` for
+            details.
+
+        .. seealso:: The MongoDB documentation on `connections <https://dochub.mongodb.org/core/connections>`_.
+
+        .. versionchanged:: 3.12
+           Added the ``server_api`` keyword argument.
+           The following keyword arguments were deprecated:
+
+             - ``ssl_certfile`` and ``ssl_keyfile`` were deprecated in favor
+               of ``tlsCertificateKeyFile``.
 
         .. versionchanged:: 3.11
            Added the following keyword arguments and URI options:
@@ -604,6 +624,14 @@ class MongoClient(common.BaseObject):
 
                client.__my_database__
         """
+        self.__init_kwargs = {'host': host,
+                              'port': port,
+                              'document_class': document_class,
+                              'tz_aware': tz_aware,
+                              'connect': connect,
+                              'type_registry': type_registry}
+        self.__init_kwargs.update(kwargs)
+
         if host is None:
             host = self.HOST
         if isinstance(host, string_type):
@@ -627,10 +655,13 @@ class MongoClient(common.BaseObject):
         username = None
         password = None
         dbase = None
-        opts = {}
+        opts = common._CaseInsensitiveDictionary()
         fqdn = None
         for entity in host:
-            if "://" in entity:
+            # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/'
+            # it must be a URI,
+            # https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
+            if "/" in entity:
                 # Determine connection timeout from kwargs.
                 timeout = keyword_opts.get("connecttimeoutms")
                 if timeout is not None:
@@ -672,11 +703,7 @@ class MongoClient(common.BaseObject):
         opts = _handle_security_options(opts)
         # Normalize combined options.
         opts = _normalize_options(opts)
-
-        # Ensure directConnection was not True if there are multiple seeds.
-        if len(seeds) > 1 and opts.get('directconnection'):
-            raise ConfigurationError(
-                "Cannot specify multiple hosts with directConnection=true")
+        _check_options(seeds, opts)
 
         # Username and password passed as kwargs override user info in URI.
         username = opts.get("username", username)
@@ -724,7 +751,9 @@ class MongoClient(common.BaseObject):
             server_selector=options.server_selector,
             heartbeat_frequency=options.heartbeat_frequency,
             fqdn=fqdn,
-            direct_connection=options.direct_connection)
+            direct_connection=options.direct_connection,
+            load_balanced=options.load_balanced,
+        )
 
         self._topology = Topology(self._topology_settings)
 
@@ -752,9 +781,14 @@ class MongoClient(common.BaseObject):
         self._encrypter = None
         if self.__options.auto_encryption_opts:
             from pymongo.encryption import _Encrypter
-            self._encrypter = _Encrypter.create(
+            self._encrypter = _Encrypter(
                 self, self.__options.auto_encryption_opts)
 
+    def _duplicate(self, **kwargs):
+        args = self.__init_kwargs.copy()
+        args.update(kwargs)
+        return MongoClient(**args)
+
     def _cache_credentials(self, source, credentials, connect=False):
         """Save a set of authentication credentials.
 
@@ -939,7 +973,7 @@ class MongoClient(common.BaseObject):
 
         .. versionadded:: 3.7
 
-        .. mongodoc:: changeStreams
+        .. seealso:: The MongoDB documentation on `changeStreams <https://dochub.mongodb.org/core/changeStreams>`_.
 
         .. _change streams specification:
             https://github.com/mongodb/specifications/blob/master/source/change-streams/change-streams.rst
@@ -957,6 +991,28 @@ class MongoClient(common.BaseObject):
         """
         return self._event_listeners.event_listeners
 
+    @property
+    def topology_description(self):
+        """The description of the connected MongoDB deployment.
+
+        >>> client.topology_description
+        <TopologyDescription id: 605a7b04e76489833a7c6113, topology_type: ReplicaSetWithPrimary, servers: [<ServerDescription ('localhost', 27017) server_type: RSPrimary, rtt: 0.0007973677999995488>, <ServerDescription ('localhost', 27018) server_type: RSSecondary, rtt: 0.0005540556000003249>, <ServerDescription ('localhost', 27019) server_type: RSSecondary, rtt: 0.0010367483999999649>]>
+        >>> client.topology_description.topology_type_name
+        'ReplicaSetWithPrimary'
+
+        Note that the description is periodically updated in the background
+        but the returned object itself is immutable. Access this property again
+        to get a more recent
+        :class:`~pymongo.topology_description.TopologyDescription`.
+
+        :Returns:
+          An instance of
+          :class:`~pymongo.topology_description.TopologyDescription`.
+
+        .. versionadded:: 3.12
+        """
+        return self._topology.description
+
     @property
     def address(self):
         """(host, port) of the current standalone, primary, or mongos, or None.
@@ -977,7 +1033,8 @@ class MongoClient(common.BaseObject):
                 'Cannot use "address" property when load balancing among'
                 ' mongoses, use "nodes" instead.')
         if topology_type not in (TOPOLOGY_TYPE.ReplicaSetWithPrimary,
-                                 TOPOLOGY_TYPE.Single):
+                                 TOPOLOGY_TYPE.Single,
+                                 TOPOLOGY_TYPE.LoadBalanced):
             return None
         return self._server_property('address')
 
@@ -1159,7 +1216,7 @@ class MongoClient(common.BaseObject):
             # another session.
             with self._socket_for_reads(
                     ReadPreference.PRIMARY_PREFERRED,
-                    None) as (sock_info, slave_ok):
+                    None) as (sock_info, secondary_ok):
                 if not sock_info.supports_sessions:
                     return
 
@@ -1167,7 +1224,7 @@ class MongoClient(common.BaseObject):
                     spec = SON([('endSessions',
                                  session_ids[i:i + common._MAX_END_SESSIONS])])
                     sock_info.command(
-                        'admin', spec, slave_ok=slave_ok, client=self)
+                        'admin', spec, secondary_ok, client=self)
         except PyMongoError:
             # Drivers MUST ignore any errors returned by the endSessions
             # command.
@@ -1241,10 +1298,19 @@ class MongoClient(common.BaseObject):
         return self._topology
 
     @contextlib.contextmanager
-    def _get_socket(self, server, session, exhaust=False):
+    def _get_socket(self, server, session):
+        in_txn = session and session.in_transaction
         with _MongoClientErrorHandler(self, server, session) as err_handler:
+            # Reuse the pinned connection, if it exists.
+            if in_txn and session._pinned_connection:
+                yield session._pinned_connection
+                return
             with server.get_socket(
-                    self.__all_credentials, checkout=exhaust) as sock_info:
+                    self.__all_credentials, handler=err_handler) as sock_info:
+                # Pin this session to the selected server or connection.
+                if (in_txn and server.description.server_type in (
+                        SERVER_TYPE.Mongos, SERVER_TYPE.LoadBalancer)):
+                    session._pin(server, sock_info)
                 err_handler.contribute_socket(sock_info)
                 if (self._encrypter and
                         not self._encrypter._bypass_auto_encryption and
@@ -1267,6 +1333,8 @@ class MongoClient(common.BaseObject):
         """
         try:
             topology = self._get_topology()
+            if session and not session.in_transaction:
+                session._transaction.reset()
             address = address or (session and session._pinned_address)
             if address:
                 # We're running a getMore or this session is pinned to a mongos.
@@ -1276,17 +1344,12 @@ class MongoClient(common.BaseObject):
                                         % address)
             else:
                 server = topology.select_server(server_selector)
-                # Pin this session to the selected server if it's performing a
-                # sharded transaction.
-                if server.description.mongos and (session and
-                                                  session.in_transaction):
-                    session._pin_mongos(server)
             return server
         except PyMongoError as exc:
             # Server selection errors in a transaction are transient.
             if session and session.in_transaction:
                 exc._add_error_label("TransientTransactionError")
-                session._unpin_mongos()
+                session._unpin()
             raise
 
     def _socket_for_writes(self, session):
@@ -1294,82 +1357,73 @@ class MongoClient(common.BaseObject):
         return self._get_socket(server, session)
 
     @contextlib.contextmanager
-    def _slaveok_for_server(self, read_preference, server, session,
-                            exhaust=False):
+    def _secondaryok_for_server(self, read_preference, server, session):
         assert read_preference is not None, "read_preference must not be None"
         # Get a socket for a server matching the read preference, and yield
-        # sock_info, slave_ok. Server Selection Spec: "slaveOK must be sent to
-        # mongods with topology type Single. If the server type is Mongos,
-        # follow the rules for passing read preference to mongos, even for
-        # topology type Single."
+        # sock_info, secondary_ok. Server Selection Spec: "secondaryOK must be
+        # sent to mongods with topology type Single. If the server type is
+        # Mongos, follow the rules for passing read preference to mongos, even
+        # for topology type Single."
         # Thread safe: if the type is single it cannot change.
         topology = self._get_topology()
         single = topology.description.topology_type == TOPOLOGY_TYPE.Single
 
-        with self._get_socket(server, session, exhaust=exhaust) as sock_info:
-            slave_ok = (single and not sock_info.is_mongos) or (
+        with self._get_socket(server, session) as sock_info:
+            secondary_ok = (single and not sock_info.is_mongos) or (
                 read_preference != ReadPreference.PRIMARY)
-            yield sock_info, slave_ok
+            yield sock_info, secondary_ok
 
     @contextlib.contextmanager
     def _socket_for_reads(self, read_preference, session):
         assert read_preference is not None, "read_preference must not be None"
         # Get a socket for a server matching the read preference, and yield
-        # sock_info, slave_ok. Server Selection Spec: "slaveOK must be sent to
-        # mongods with topology type Single. If the server type is Mongos,
-        # follow the rules for passing read preference to mongos, even for
-        # topology type Single."
+        # sock_info, secondary_ok. Server Selection Spec: "secondaryOK must be
+        # sent to mongods with topology type Single. If the server type is
+        # Mongos, follow the rules for passing read preference to mongos, even
+        # for topology type Single."
         # Thread safe: if the type is single it cannot change.
         topology = self._get_topology()
         single = topology.description.topology_type == TOPOLOGY_TYPE.Single
         server = self._select_server(read_preference, session)
 
         with self._get_socket(server, session) as sock_info:
-            slave_ok = (single and not sock_info.is_mongos) or (
+            secondary_ok = (single and not sock_info.is_mongos) or (
                 read_preference != ReadPreference.PRIMARY)
-            yield sock_info, slave_ok
+            yield sock_info, secondary_ok
 
-    def _run_operation_with_response(self, operation, unpack_res,
-                                     exhaust=False, address=None):
+    def _should_pin_cursor(self, session):
+        return (self.__options.load_balanced and
+                not (session and session.in_transaction))
+
+    def _run_operation(self, operation, unpack_res, address=None):
         """Run a _Query/_GetMore operation and return a Response.
 
         :Parameters:
           - `operation`: a _Query or _GetMore object.
           - `unpack_res`: A callable that decodes the wire protocol response.
-          - `exhaust` (optional): If True, the socket used stays checked out.
-            It is returned along with its Pool in the Response.
           - `address` (optional): Optional address when sending a message
             to a specific server, used for getMore.
         """
-        if operation.exhaust_mgr:
+        if operation.sock_mgr:
             server = self._select_server(
                 operation.read_preference, operation.session, address=address)
 
-            with _MongoClientErrorHandler(
-                    self, server, operation.session) as err_handler:
-                err_handler.contribute_socket(operation.exhaust_mgr.sock)
-                return server.run_operation_with_response(
-                    operation.exhaust_mgr.sock,
-                    operation,
-                    True,
-                    self._event_listeners,
-                    exhaust,
-                    unpack_res)
+            with operation.sock_mgr.lock:
+                with _MongoClientErrorHandler(
+                        self, server, operation.session) as err_handler:
+                    err_handler.contribute_socket(operation.sock_mgr.sock)
+                    return server.run_operation(
+                        operation.sock_mgr.sock, operation, True,
+                        self._event_listeners, unpack_res)
 
-        def _cmd(session, server, sock_info, slave_ok):
-            return server.run_operation_with_response(
-                sock_info,
-                operation,
-                slave_ok,
-                self._event_listeners,
-                exhaust,
+        def _cmd(session, server, sock_info, secondary_ok):
+            return server.run_operation(
+                sock_info, operation, secondary_ok, self._event_listeners,
                 unpack_res)
 
         return self._retryable_read(
             _cmd, operation.read_preference, operation.session,
-            address=address,
-            retryable=isinstance(operation, message._Query),
-            exhaust=exhaust)
+            address=address, retryable=isinstance(operation, message._Query))
 
     def _retry_with_session(self, retryable, func, session, bulk):
         """Execute an operation with at most one consecutive retries
@@ -1431,7 +1485,7 @@ class MongoClient(common.BaseObject):
                 _add_retryable_write_error(exc, max_wire_version)
                 retryable_error = exc.has_error_label("RetryableWriteError")
                 if retryable_error:
-                    session._unpin_mongos()
+                    session._unpin()
                 if is_retrying() or not retryable_error:
                     raise
                 if bulk:
@@ -1441,7 +1495,7 @@ class MongoClient(common.BaseObject):
                 last_error = exc
 
     def _retryable_read(self, func, read_pref, session, address=None,
-                        retryable=True, exhaust=False):
+                        retryable=True):
         """Execute an operation with at most one consecutive retries
 
         Returns func()'s return value on success. On error retries the same
@@ -1461,14 +1515,14 @@ class MongoClient(common.BaseObject):
                     read_pref, session, address=address)
                 if not server.description.retryable_reads_supported:
                     retryable = False
-                with self._slaveok_for_server(read_pref, server, session,
-                                              exhaust=exhaust) as (sock_info,
-                                                                   slave_ok):
+                with self._secondaryok_for_server(
+                        read_pref, server, session) as (
+                                sock_info, secondary_ok):
                     if retrying and not retryable:
                         # A retry is not possible because this server does
                         # not support retryable reads, raise the last error.
                         raise last_error
-                    return func(session, server, sock_info, slave_ok)
+                    return func(session, server, sock_info, secondary_ok)
             except ServerSelectionTimeoutError:
                 if retrying:
                     # The application may think the write was never attempted
@@ -1509,6 +1563,9 @@ class MongoClient(common.BaseObject):
     def __ne__(self, other):
         return not self == other
 
+    def __hash__(self):
+        return hash(self.address)
+
     def _repr_helper(self):
         def option_repr(option, value):
             """Fix options whose __repr__ isn't usable in a constructor."""
@@ -1599,10 +1656,47 @@ class MongoClient(common.BaseObject):
         if not isinstance(cursor_id, integer_types):
             raise TypeError("cursor_id must be an instance of (int, long)")
 
-        self._close_cursor(cursor_id, address)
+        self._close_cursor_soon(cursor_id, address)
 
-    def _close_cursor(self, cursor_id, address):
-        """Send a kill cursors message with the given id.
+    def _cleanup_cursor(self, locks_allowed, cursor_id, address, sock_mgr,
+                        session, explicit_session):
+        """Cleanup a cursor from cursor.close() or __del__.
+
+        This method handles cleanup for Cursors/CommandCursors including any
+        pinned connection or implicit session attached at the time the cursor
+        was closed or garbage collected.
+
+        :Parameters:
+          - `locks_allowed`: True if we are allowed to acquire locks.
+          - `cursor_id`: The cursor id which may be 0.
+          - `address`: The _CursorAddress.
+          - `sock_mgr`: The _SocketManager for the pinned connection or None.
+          - `session`: The cursor's session.
+          - `explicit_session`: True if the session was passed explicitly.
+        """
+        if locks_allowed:
+            if cursor_id:
+                if sock_mgr and sock_mgr.more_to_come:
+                    # If this is an exhaust cursor and we haven't completely
+                    # exhausted the result set we *must* close the socket
+                    # to stop the server from sending more data.
+                    sock_mgr.sock.close_socket(
+                        ConnectionClosedReason.ERROR)
+                else:
+                    self._close_cursor_now(
+                        cursor_id, address, session=session,
+                        sock_mgr=sock_mgr)
+            if sock_mgr:
+                sock_mgr.close()
+        else:
+            # The cursor will be closed later in a different session.
+            if cursor_id or sock_mgr:
+                self._close_cursor_soon(cursor_id, address, sock_mgr)
+        if session and not explicit_session:
+            session._end_session(lock=locks_allowed)
+
+    def _close_cursor_soon(self, cursor_id, address, sock_mgr=None):
+        """Request that a cursor and/or connection be cleaned up soon
 
         What closing the cursor actually means depends on this client's
         cursor manager. If there is none, the cursor is closed asynchronously
@@ -1611,9 +1705,10 @@ class MongoClient(common.BaseObject):
         if self.__cursor_manager is not None:
             self.__cursor_manager.close(cursor_id, address)
         else:
-            self.__kill_cursors_queue.append((address, [cursor_id]))
+            self.__kill_cursors_queue.append((address, cursor_id, sock_mgr))
 
-    def _close_cursor_now(self, cursor_id, address=None, session=None):
+    def _close_cursor_now(self, cursor_id, address=None, session=None,
+                          sock_mgr=None):
         """Send a kill cursors message with the given id.
 
         What closing the cursor actually means depends on this client's
@@ -1627,11 +1722,17 @@ class MongoClient(common.BaseObject):
             self.__cursor_manager.close(cursor_id, address)
         else:
             try:
-                self._kill_cursors(
-                    [cursor_id], address, self._get_topology(), session)
+                if sock_mgr:
+                    with sock_mgr.lock:
+                        # Cursor is pinned to LB outside of a transaction.
+                        self._kill_cursor_impl(
+                            [cursor_id], address, session, sock_mgr.sock)
+                else:
+                    self._kill_cursors(
+                        [cursor_id], address, self._get_topology(), session)
             except PyMongoError:
                 # Make another attempt to kill the cursor later.
-                self.__kill_cursors_queue.append((address, [cursor_id]))
+                self._close_cursor_soon(cursor_id, address)
 
     def kill_cursors(self, cursor_ids, address=None):
         """DEPRECATED - Send a kill cursors message soon with the given ids.
@@ -1662,12 +1763,11 @@ class MongoClient(common.BaseObject):
             raise TypeError("cursor_ids must be a list")
 
         # "Atomic", needs no lock.
-        self.__kill_cursors_queue.append((address, cursor_ids))
+        for cursor_id in cursor_ids:
+            self.__kill_cursors_queue.append((address, cursor_id, None))
 
     def _kill_cursors(self, cursor_ids, address, topology, session):
         """Send a kill cursors message with the given ids."""
-        listeners = self._event_listeners
-        publish = listeners.enabled_for_commands
         if address:
             # address could be a tuple or _CursorAddress, but
             # select_server_by_address needs (host, port).
@@ -1676,62 +1776,79 @@ class MongoClient(common.BaseObject):
             # Application called close_cursor() with no address.
             server = topology.select_server(writable_server_selector)
 
+        with self._get_socket(server, session) as sock_info:
+            self._kill_cursor_impl(cursor_ids, address, session, sock_info)
+
+    def _kill_cursor_impl(self, cursor_ids, address, session, sock_info):
+        listeners = self._event_listeners
+        publish = listeners.enabled_for_commands
+
         try:
             namespace = address.namespace
             db, coll = namespace.split('.', 1)
         except AttributeError:
             namespace = None
             db = coll = "OP_KILL_CURSORS"
-
         spec = SON([('killCursors', coll), ('cursors', cursor_ids)])
-        with server.get_socket(self.__all_credentials) as sock_info:
-            if sock_info.max_wire_version >= 4 and namespace is not None:
-                sock_info.command(db, spec, session=session, client=self)
-            else:
-                if publish:
-                    start = datetime.datetime.now()
-                request_id, msg = message.kill_cursors(cursor_ids)
-                if publish:
-                    duration = datetime.datetime.now() - start
-                    # Here and below, address could be a tuple or
-                    # _CursorAddress. We always want to publish a
-                    # tuple to match the rest of the monitoring
-                    # API.
-                    listeners.publish_command_start(
-                        spec, db, request_id, tuple(address))
-                    start = datetime.datetime.now()
-
-                try:
-                    sock_info.send_message(msg, 0)
-                except Exception as exc:
-                    if publish:
-                        dur = ((datetime.datetime.now() - start) + duration)
-                        listeners.publish_command_failure(
-                            dur, message._convert_exception(exc),
-                            'killCursors', request_id,
-                            tuple(address))
-                    raise
+        if sock_info.max_wire_version >= 4 and namespace is not None:
+            sock_info.command(db, spec, session=session, client=self)
+        else:
+            if publish:
+                start = datetime.datetime.now()
+            request_id, msg = message.kill_cursors(cursor_ids)
+            if publish:
+                duration = datetime.datetime.now() - start
+                # Here and below, address could be a tuple or
+                # _CursorAddress. We always want to publish a
+                # tuple to match the rest of the monitoring
+                # API.
+                listeners.publish_command_start(
+                    spec, db, request_id, tuple(address),
+                    service_id=sock_info.service_id)
+                start = datetime.datetime.now()
 
+            try:
+                sock_info.send_message(msg, 0)
+            except Exception as exc:
                 if publish:
-                    duration = ((datetime.datetime.now() - start) + duration)
-                    # OP_KILL_CURSORS returns no reply, fake one.
-                    reply = {'cursorsUnknown': cursor_ids, 'ok': 1}
-                    listeners.publish_command_success(
-                        duration, reply, 'killCursors', request_id,
-                        tuple(address))
+                    dur = ((datetime.datetime.now() - start) + duration)
+                    listeners.publish_command_failure(
+                        dur, message._convert_exception(exc),
+                        'killCursors', request_id,
+                        tuple(address), service_id=sock_info.service_id)
+                raise
+
+            if publish:
+                duration = ((datetime.datetime.now() - start) + duration)
+                # OP_KILL_CURSORS returns no reply, fake one.
+                reply = {'cursorsUnknown': cursor_ids, 'ok': 1}
+                listeners.publish_command_success(
+                    duration, reply, 'killCursors', request_id,
+                    tuple(address), service_id=sock_info.service_id)
 
     def _process_kill_cursors(self):
         """Process any pending kill cursors requests."""
         address_to_cursor_ids = defaultdict(list)
+        pinned_cursors = []
 
         # Other threads or the GC may append to the queue concurrently.
         while True:
             try:
-                address, cursor_ids = self.__kill_cursors_queue.pop()
+                address, cursor_id, sock_mgr = self.__kill_cursors_queue.pop()
             except IndexError:
                 break
 
-            address_to_cursor_ids[address].extend(cursor_ids)
+            if sock_mgr:
+                pinned_cursors.append((address, cursor_id, sock_mgr))
+            else:
+                address_to_cursor_ids[address].append(cursor_id)
+
+        for address, cursor_id, sock_mgr in pinned_cursors:
+            try:
+                self._cleanup_cursor(True, cursor_id, address, sock_mgr,
+                                     None, False)
+            except Exception:
+                helpers._handle_exception()
 
         # Don't re-open topology if it's closed and there's no pending cursors.
         if address_to_cursor_ids:
@@ -1769,8 +1886,9 @@ class MongoClient(common.BaseObject):
             self, server_session, opts, authset, implicit)
 
     def start_session(self,
-                      causal_consistency=True,
-                      default_transaction_options=None):
+                      causal_consistency=None,
+                      default_transaction_options=None,
+                      snapshot=False):
         """Start a logical session.
 
         This method takes the same parameters as
@@ -1795,7 +1913,8 @@ class MongoClient(common.BaseObject):
         return self.__start_session(
             False,
             causal_consistency=causal_consistency,
-            default_transaction_options=default_transaction_options)
+            default_transaction_options=default_transaction_options,
+            snapshot=snapshot)
 
     def _get_server_session(self):
         """Internal: start or resume a _ServerSession."""
@@ -2229,7 +2348,7 @@ def _retryable_error_doc(exc):
         wces = exc.details['writeConcernErrors']
         wce = wces[-1] if wces else None
         return wce
-    if isinstance(exc, (NotMasterError, OperationFailure)):
+    if isinstance(exc, (NotPrimaryError, OperationFailure)):
         return exc.details
     return None
 
@@ -2254,17 +2373,18 @@ def _add_retryable_write_error(exc, max_wire_version):
             if code in helpers._RETRYABLE_ERROR_CODES:
                 exc._add_error_label("RetryableWriteError")
 
-    # Connection errors are always retryable except NotMasterError which is
+    # Connection errors are always retryable except NotPrimaryError which is
     # handled above.
     if (isinstance(exc, ConnectionFailure) and
-            not isinstance(exc, NotMasterError)):
+            not isinstance(exc, NotPrimaryError)):
         exc._add_error_label("RetryableWriteError")
 
 
 class _MongoClientErrorHandler(object):
     """Handle errors raised when executing an operation."""
     __slots__ = ('client', 'server_address', 'session', 'max_wire_version',
-                 'sock_generation', 'completed_handshake')
+                 'sock_generation', 'completed_handshake', 'service_id',
+                 'handled')
 
     def __init__(self, client, server, session):
         self.client = client
@@ -2275,22 +2395,22 @@ class _MongoClientErrorHandler(object):
         # "Note that when a network error occurs before the handshake
         # completes then the error's generation number is the generation
         # of the pool at the time the connection attempt was started."
-        self.sock_generation = server.pool.generation
+        self.sock_generation = server.pool.gen.get_overall()
         self.completed_handshake = False
+        self.service_id = None
+        self.handled = False
 
     def contribute_socket(self, sock_info):
         """Provide socket information to the error handler."""
         self.max_wire_version = sock_info.max_wire_version
         self.sock_generation = sock_info.generation
+        self.service_id = sock_info.service_id
         self.completed_handshake = True
 
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is None:
+    def handle(self, exc_type, exc_val):
+        if self.handled or exc_type is None:
             return
-
+        self.handled = True
         if self.session:
             if issubclass(exc_type, ConnectionFailure):
                 if self.session.in_transaction:
@@ -2300,9 +2420,15 @@ class _MongoClientErrorHandler(object):
             if issubclass(exc_type, PyMongoError):
                 if (exc_val.has_error_label("TransientTransactionError") or
                         exc_val.has_error_label("RetryableWriteError")):
-                    self.session._unpin_mongos()
+                    self.session._unpin()
 
         err_ctx = _ErrorContext(
             exc_val, self.max_wire_version, self.sock_generation,
-            self.completed_handshake)
+            self.completed_handshake, self.service_id)
         self.client._topology.handle_error(self.server_address, err_ctx)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self.handle(exc_type, exc_val)

pymongo/monitor.py

@@ -18,8 +18,10 @@ import atexit
 import threading
 import weakref
 
+from bson.py3compat import PY3
+
 from pymongo import common, periodic_executor
-from pymongo.errors import (NotMasterError,
+from pymongo.errors import (NotPrimaryError,
                             OperationFailure,
                             _OperationCancelled)
 from pymongo.ismaster import IsMaster
@@ -30,6 +32,14 @@ from pymongo.server_description import ServerDescription
 from pymongo.srv_resolver import _SrvResolver
 
 
+def _sanitize(error):
+    """PYTHON-2433 Clear error traceback info."""
+    if PY3:
+        error.__traceback__ = None
+        error.__context__ = None
+        error.__cause__ = None
+
+
 class MonitorBase(object):
     def __init__(self, topology, name, interval, min_interval):
         """Base class to do periodic work on a background thread.
@@ -55,7 +65,7 @@ class MonitorBase(object):
         self._executor = executor
 
         def _on_topology_gc(dummy=None):
-            # This prevents GC from waiting 10 seconds for isMaster to complete
+            # This prevents GC from waiting 10 seconds for 'hello' to complete
             # See test_cleanup_executors_on_client_del.
             monitor = self_ref()
             if monitor:
@@ -126,7 +136,7 @@ class Monitor(MonitorBase):
         self.heartbeater = None
 
     def cancel_check(self):
-        """Cancel any concurrent isMaster check.
+        """Cancel any concurrent hello check.
 
         Note: this is called from a weakref.proxy callback and MUST NOT take
         any locks.
@@ -169,6 +179,7 @@ class Monitor(MonitorBase):
             try:
                 self._server_description = self._check_server()
             except _OperationCancelled as exc:
+                _sanitize(exc)
                 # Already closed the connection, wait for the next check.
                 self._server_description = ServerDescription(
                     self._server_description.address, error=exc)
@@ -196,7 +207,7 @@ class Monitor(MonitorBase):
             self.close()
 
     def _check_server(self):
-        """Call isMaster or read the next streaming response.
+        """Call hello or read the next streaming response.
 
         Returns a ServerDescription.
         """
@@ -204,14 +215,15 @@ class Monitor(MonitorBase):
         try:
             try:
                 return self._check_once()
-            except (OperationFailure, NotMasterError) as exc:
-                # Update max cluster time even when isMaster fails.
+            except (OperationFailure, NotPrimaryError) as exc:
+                # Update max cluster time even when hello fails.
                 self._topology.receive_cluster_time(
                     exc.details.get('$clusterTime'))
                 raise
         except ReferenceError:
             raise
         except Exception as error:
+            _sanitize(error)
             sd = self._server_description
             address = sd.address
             duration = _time() - start
@@ -227,7 +239,7 @@ class Monitor(MonitorBase):
             return ServerDescription(address, error=error)
 
     def _check_once(self):
-        """A single attempt to call ismaster.
+        """A single attempt to call hello.
 
         Returns a ServerDescription, or raises an exception.
         """
@@ -251,26 +263,26 @@ class Monitor(MonitorBase):
             return sd
 
     def _check_with_socket(self, conn):
-        """Return (IsMaster, round_trip_time).
+        """Return (Hello, round_trip_time).
 
         Can raise ConnectionFailure or OperationFailure.
         """
         cluster_time = self._topology.max_cluster_time()
         start = _time()
         if conn.more_to_come:
-            # Read the next streaming isMaster (MongoDB 4.4+).
+            # Read the next streaming hello (MongoDB 4.4+).
             response = IsMaster(conn._next_reply(), awaitable=True)
         elif (conn.performed_handshake and
               self._server_description.topology_version):
-            # Initiate streaming isMaster (MongoDB 4.4+).
-            response = conn._ismaster(
+            # Initiate streaming hello (MongoDB 4.4+).
+            response = conn._hello(
                 cluster_time,
                 self._server_description.topology_version,
                 self._settings.heartbeat_frequency,
                 None)
         else:
-            # New connection handshake or polling isMaster (MongoDB <4.4).
-            response = conn._ismaster(cluster_time, None, None, None)
+            # New connection handshake or polling hello (MongoDB <4.4).
+            response = conn._hello(cluster_time, None, None, None)
         return response, _time() - start
 
 
@@ -375,12 +387,12 @@ class _RttMonitor(MonitorBase):
             self._pool.reset()
 
     def _ping(self):
-        """Run an "isMaster" command and return the RTT."""
+        """Run a "hello" command and return the RTT."""
         with self._pool.get_socket({}) as sock_info:
             if self._executor._stopped:
                 raise Exception('_RttMonitor closed')
             start = _time()
-            sock_info.ismaster()
+            sock_info.hello()
             return _time() - start
 
 

pymongo/monitoring.py

@@ -183,6 +183,7 @@ will not add that listener to existing client instances.
 from collections import namedtuple
 
 from bson.py3compat import abc
+from pymongo.hello_compat import HelloCompat
 from pymongo.helpers import _handle_exception
 
 _Listeners = namedtuple('Listeners',
@@ -498,16 +499,28 @@ _SENSITIVE_COMMANDS = set(
      "updateuser", "copydbgetnonce", "copydbsaslstart", "copydb"])
 
 
+# The "hello" command is also deemed sensitive when attempting speculative
+# authentication.
+def _is_speculative_authenticate(command_name, doc):
+    if (command_name.lower() in ('hello', HelloCompat.LEGACY_CMD) and
+            'speculativeAuthenticate' in doc):
+        return True
+    return False
+
+
 class _CommandEvent(object):
     """Base class for command events."""
 
-    __slots__ = ("__cmd_name", "__rqst_id", "__conn_id", "__op_id")
+    __slots__ = ("__cmd_name", "__rqst_id", "__conn_id", "__op_id",
+                 "__service_id")
 
-    def __init__(self, command_name, request_id, connection_id, operation_id):
+    def __init__(self, command_name, request_id, connection_id, operation_id,
+                 service_id=None):
         self.__cmd_name = command_name
         self.__rqst_id = request_id
         self.__conn_id = connection_id
         self.__op_id = operation_id
+        self.__service_id = service_id
 
     @property
     def command_name(self):
@@ -524,6 +537,14 @@ class _CommandEvent(object):
         """The address (host, port) of the server this command was sent to."""
         return self.__conn_id
 
+    @property
+    def service_id(self):
+        """The service_id this command was sent to, or ``None``.
+
+        .. versionadded:: 3.12
+        """
+        return self.__service_id
+
     @property
     def operation_id(self):
         """An id for this series of events or None."""
@@ -540,16 +561,22 @@ class CommandStartedEvent(_CommandEvent):
       - `connection_id`: The address (host, port) of the server this command
         was sent to.
       - `operation_id`: An optional identifier for a series of related events.
+      - `service_id`: The service_id this command was sent to, or ``None``.
     """
     __slots__ = ("__cmd", "__db")
 
-    def __init__(self, command, database_name, *args):
+    def __init__(self, command, database_name, request_id, connection_id,
+                 operation_id, service_id=None):
         if not command:
             raise ValueError("%r is not a valid command" % (command,))
         # Command name must be first key.
         command_name = next(iter(command))
-        super(CommandStartedEvent, self).__init__(command_name, *args)
-        if command_name.lower() in _SENSITIVE_COMMANDS:
+        super(CommandStartedEvent, self).__init__(
+            command_name, request_id, connection_id, operation_id,
+            service_id=service_id)
+        cmd_name, cmd_doc = command_name.lower(), command[command_name]
+        if (cmd_name in _SENSITIVE_COMMANDS or
+                _is_speculative_authenticate(cmd_name, command)):
             self.__cmd = {}
         else:
             self.__cmd = command
@@ -566,9 +593,12 @@ class CommandStartedEvent(_CommandEvent):
         return self.__db
 
     def __repr__(self):
-        return "<%s %s db: %r, command: %r, operation_id: %s>" % (
-            self.__class__.__name__, self.connection_id, self.database_name,
-            self.command_name, self.operation_id)
+        return (
+            "<%s %s db: %r, command: %r, operation_id: %s, "
+            "service_id: %s>") % (
+                self.__class__.__name__, self.connection_id,
+                self.database_name, self.command_name, self.operation_id,
+                self.service_id)
 
 
 class CommandSucceededEvent(_CommandEvent):
@@ -582,15 +612,19 @@ class CommandSucceededEvent(_CommandEvent):
       - `connection_id`: The address (host, port) of the server this command
         was sent to.
       - `operation_id`: An optional identifier for a series of related events.
+      - `service_id`: The service_id this command was sent to, or ``None``.
     """
     __slots__ = ("__duration_micros", "__reply")
 
     def __init__(self, duration, reply, command_name,
-                 request_id, connection_id, operation_id):
+                 request_id, connection_id, operation_id, service_id=None):
         super(CommandSucceededEvent, self).__init__(
-            command_name, request_id, connection_id, operation_id)
+            command_name, request_id, connection_id, operation_id,
+            service_id=service_id)
         self.__duration_micros = _to_micros(duration)
-        if command_name.lower() in _SENSITIVE_COMMANDS:
+        cmd_name = command_name.lower()
+        if (cmd_name in _SENSITIVE_COMMANDS or
+                _is_speculative_authenticate(cmd_name, reply)):
             self.__reply = {}
         else:
             self.__reply = reply
@@ -606,9 +640,12 @@ class CommandSucceededEvent(_CommandEvent):
         return self.__reply
 
     def __repr__(self):
-        return "<%s %s command: %r, operation_id: %s, duration_micros: %s>" % (
-            self.__class__.__name__, self.connection_id,
-            self.command_name, self.operation_id, self.duration_micros)
+        return (
+            "<%s %s command: %r, operation_id: %s, duration_micros: %s, "
+            "service_id: %s>") % (
+                self.__class__.__name__, self.connection_id,
+                self.command_name, self.operation_id, self.duration_micros,
+                self.service_id)
 
 
 class CommandFailedEvent(_CommandEvent):
@@ -622,11 +659,15 @@ class CommandFailedEvent(_CommandEvent):
       - `connection_id`: The address (host, port) of the server this command
         was sent to.
       - `operation_id`: An optional identifier for a series of related events.
+      - `service_id`: The service_id this command was sent to, or ``None``.
     """
     __slots__ = ("__duration_micros", "__failure")
 
-    def __init__(self, duration, failure, *args):
-        super(CommandFailedEvent, self).__init__(*args)
+    def __init__(self, duration, failure, command_name, request_id,
+                 connection_id, operation_id, service_id=None):
+        super(CommandFailedEvent, self).__init__(
+            command_name, request_id, connection_id, operation_id,
+            service_id=service_id)
         self.__duration_micros = _to_micros(duration)
         self.__failure = failure
 
@@ -643,9 +684,10 @@ class CommandFailedEvent(_CommandEvent):
     def __repr__(self):
         return (
             "<%s %s command: %r, operation_id: %s, duration_micros: %s, "
-            "failure: %r>" % (
+            "failure: %r, service_id: %s>") % (
                 self.__class__.__name__, self.connection_id, self.command_name,
-                self.operation_id, self.duration_micros, self.failure))
+                self.operation_id, self.duration_micros, self.failure,
+                self.service_id)
 
 
 class _PoolEvent(object):
@@ -698,10 +740,29 @@ class PoolClearedEvent(_PoolEvent):
     :Parameters:
      - `address`: The address (host, port) pair of the server this Pool is
        attempting to connect to.
+     - `service_id`: The service_id this command was sent to, or ``None``.
 
     .. versionadded:: 3.9
     """
-    __slots__ = ()
+    __slots__ = ("__service_id",)
+
+    def __init__(self, address, service_id=None):
+        super(PoolClearedEvent, self).__init__(address)
+        self.__service_id = service_id
+
+    @property
+    def service_id(self):
+        """Connections with this service_id are cleared.
+
+        When service_id is ``None``, all connections in the pool are cleared.
+
+        .. versionadded:: 3.12
+        """
+        return self.__service_id
+
+    def __repr__(self):
+        return '%s(%r, %r)' % (
+            self.__class__.__name__, self.address, self.__service_id)
 
 
 class PoolClosedEvent(_PoolEvent):
@@ -1118,7 +1179,12 @@ class ServerHeartbeatSucceededEvent(_ServerHeartbeatEvent):
 
     @property
     def reply(self):
-        """An instance of :class:`~pymongo.ismaster.IsMaster`."""
+        """An instance of :class:`~pymongo.ismaster.IsMaster`.
+
+        .. warning:: :class:`~pymongo.ismaster.IsMaster` is deprecated.
+          Starting with PyMongo 4.0 this attribute will return an instance
+          of :class:`~pymongo.hello.Hello`, which provides the same API.
+        """
         return self.__reply
 
     @property
@@ -1245,7 +1311,8 @@ class _EventListeners(object):
                 self.__topology_listeners[:])
 
     def publish_command_start(self, command, database_name,
-                              request_id, connection_id, op_id=None):
+                              request_id, connection_id, op_id=None,
+                              service_id=None):
         """Publish a CommandStartedEvent to all command listeners.
 
         :Parameters:
@@ -1256,11 +1323,13 @@ class _EventListeners(object):
           - `connection_id`: The address (host, port) of the server this
             command was sent to.
           - `op_id`: The (optional) operation id for this operation.
+          - `service_id`: The service_id this command was sent to, or ``None``.
         """
         if op_id is None:
             op_id = request_id
         event = CommandStartedEvent(
-            command, database_name, request_id, connection_id, op_id)
+            command, database_name, request_id, connection_id, op_id,
+            service_id=service_id)
         for subscriber in self.__command_listeners:
             try:
                 subscriber.started(event)
@@ -1268,7 +1337,9 @@ class _EventListeners(object):
                 _handle_exception()
 
     def publish_command_success(self, duration, reply, command_name,
-                                request_id, connection_id, op_id=None):
+                                request_id, connection_id, op_id=None,
+                                service_id=None,
+                                speculative_hello=False):
         """Publish a CommandSucceededEvent to all command listeners.
 
         :Parameters:
@@ -1279,11 +1350,18 @@ class _EventListeners(object):
           - `connection_id`: The address (host, port) of the server this
             command was sent to.
           - `op_id`: The (optional) operation id for this operation.
+          - `service_id`: The service_id this command was sent to, or ``None``.
+          - `speculative_hello`: Was the command sent with speculative auth?
         """
         if op_id is None:
             op_id = request_id
+        if speculative_hello:
+            # Redact entire response when the command started contained
+            # speculativeAuthenticate.
+            reply = {}
         event = CommandSucceededEvent(
-            duration, reply, command_name, request_id, connection_id, op_id)
+            duration, reply, command_name, request_id, connection_id, op_id,
+            service_id)
         for subscriber in self.__command_listeners:
             try:
                 subscriber.succeeded(event)
@@ -1291,7 +1369,8 @@ class _EventListeners(object):
                 _handle_exception()
 
     def publish_command_failure(self, duration, failure, command_name,
-                                request_id, connection_id, op_id=None):
+                                request_id, connection_id, op_id=None,
+                                service_id=None):
         """Publish a CommandFailedEvent to all command listeners.
 
         :Parameters:
@@ -1303,11 +1382,13 @@ class _EventListeners(object):
           - `connection_id`: The address (host, port) of the server this
             command was sent to.
           - `op_id`: The (optional) operation id for this operation.
+          - `service_id`: The service_id this command was sent to, or ``None``.
         """
         if op_id is None:
             op_id = request_id
         event = CommandFailedEvent(
-            duration, failure, command_name, request_id, connection_id, op_id)
+            duration, failure, command_name, request_id, connection_id, op_id,
+            service_id=service_id)
         for subscriber in self.__command_listeners:
             try:
                 subscriber.failed(event)
@@ -1475,10 +1556,10 @@ class _EventListeners(object):
             except Exception:
                 _handle_exception()
 
-    def publish_pool_cleared(self, address):
+    def publish_pool_cleared(self, address, service_id):
         """Publish a :class:`PoolClearedEvent` to all pool listeners.
         """
-        event = PoolClearedEvent(address)
+        event = PoolClearedEvent(address, service_id)
         for subscriber in self.__cmap_listeners:
             try:
                 subscriber.pool_cleared(event)

pymongo/network.py

@@ -27,12 +27,13 @@ from pymongo import helpers, message
 from pymongo.common import MAX_MESSAGE_SIZE
 from pymongo.compression_support import decompress, _NO_COMPRESSION
 from pymongo.errors import (AutoReconnect,
-                            NotMasterError,
+                            NotPrimaryError,
                             OperationFailure,
                             ProtocolError,
                             NetworkTimeout,
                             _OperationCancelled)
 from pymongo.message import _UNPACK_REPLY, _OpMsg
+from pymongo.monitoring import _is_speculative_authenticate
 from pymongo.monotonic import time
 from pymongo.socket_checker import _errno_from_exception
 
@@ -40,7 +41,7 @@ from pymongo.socket_checker import _errno_from_exception
 _UNPACK_HEADER = struct.Struct("<iiii").unpack
 
 
-def command(sock_info, dbname, spec, slave_ok, is_mongos,
+def command(sock_info, dbname, spec, secondary_ok, is_mongos,
             read_preference, codec_options, session, client, check=True,
             allowable_errors=None, address=None,
             check_keys=False, listeners=None, max_bson_size=None,
@@ -58,7 +59,7 @@ def command(sock_info, dbname, spec, slave_ok, is_mongos,
       - `sock`: a raw socket instance
       - `dbname`: name of the database on which to run the command
       - `spec`: a command document as an ordered dict type, eg SON.
-      - `slave_ok`: whether to set the SlaveOkay wire protocol bit
+      - `secondary_ok`: whether to set the secondaryOkay wire protocol bit
       - `is_mongos`: are we connected to a mongos?
       - `read_preference`: a read preference
       - `codec_options`: a CodecOptions instance
@@ -84,7 +85,8 @@ def command(sock_info, dbname, spec, slave_ok, is_mongos,
     """
     name = next(iter(spec))
     ns = dbname + '.$cmd'
-    flags = 4 if slave_ok else 0
+    flags = 4 if secondary_ok else 0
+    speculative_hello = False
 
     # Publish the original command document, perhaps with lsid and $clusterTime.
     orig = spec
@@ -93,16 +95,15 @@ def command(sock_info, dbname, spec, slave_ok, is_mongos,
     if read_concern and not (session and session.in_transaction):
         if read_concern.level:
             spec['readConcern'] = read_concern.document
-        if (session and session.options.causal_consistency
-                and session.operation_time is not None):
-            spec.setdefault(
-                'readConcern', {})['afterClusterTime'] = session.operation_time
+        if session:
+            session._update_read_concern(spec, sock_info)
     if collation is not None:
         spec['collation'] = collation
 
     publish = listeners is not None and listeners.enabled_for_commands
     if publish:
         start = datetime.datetime.now()
+        speculative_hello = _is_speculative_authenticate(name, spec)
 
     if compression_ctx and name.lower() in _NO_COMPRESSION:
         compression_ctx = None
@@ -118,7 +119,7 @@ def command(sock_info, dbname, spec, slave_ok, is_mongos,
         flags = _OpMsg.MORE_TO_COME if unacknowledged else 0
         flags |= _OpMsg.EXHAUST_ALLOWED if exhaust_allowed else 0
         request_id, msg, size, max_doc_size = message._op_msg(
-            flags, spec, dbname, read_preference, slave_ok, check_keys,
+            flags, spec, dbname, read_preference, secondary_ok, check_keys,
             codec_options, ctx=compression_ctx)
         # If this is an unacknowledged write then make sure the encoded doc(s)
         # are small enough, otherwise rely on the server to return an error.
@@ -137,7 +138,8 @@ def command(sock_info, dbname, spec, slave_ok, is_mongos,
 
     if publish:
         encoding_duration = datetime.datetime.now() - start
-        listeners.publish_command_start(orig, dbname, request_id, address)
+        listeners.publish_command_start(orig, dbname, request_id, address,
+                                        service_id=sock_info.service_id)
         start = datetime.datetime.now()
 
     try:
@@ -162,17 +164,20 @@ def command(sock_info, dbname, spec, slave_ok, is_mongos,
     except Exception as exc:
         if publish:
             duration = (datetime.datetime.now() - start) + encoding_duration
-            if isinstance(exc, (NotMasterError, OperationFailure)):
+            if isinstance(exc, (NotPrimaryError, OperationFailure)):
                 failure = exc.details
             else:
                 failure = message._convert_exception(exc)
             listeners.publish_command_failure(
-                duration, failure, name, request_id, address)
+                duration, failure, name, request_id, address,
+                service_id=sock_info.service_id)
         raise
     if publish:
         duration = (datetime.datetime.now() - start) + encoding_duration
         listeners.publish_command_success(
-            duration, response_doc, name, request_id, address)
+            duration, response_doc, name, request_id, address,
+            service_id=sock_info.service_id,
+            speculative_hello=speculative_hello)
 
     if client and client._encrypter and reply:
         decrypted = client._encrypter.decrypt(reply.raw_command_response())
@@ -244,7 +249,7 @@ def wait_for_read(sock_info, deadline):
                 readable = sock_info.socket_checker.select(
                     sock, read=True, timeout=timeout)
             if context.cancelled:
-                raise _OperationCancelled('isMaster cancelled')
+                raise _OperationCancelled('hello cancelled')
             if readable:
                 return
             if deadline and time() > deadline:

pymongo/pool.py

@@ -20,7 +20,7 @@ import socket
 import sys
 import threading
 import collections
-
+import weakref
 
 from pymongo.ssl_support import (
     SSLError as _SSLError,
@@ -48,9 +48,11 @@ from pymongo.errors import (AutoReconnect,
                             InvalidOperation,
                             DocumentTooLarge,
                             NetworkTimeout,
-                            NotMasterError,
+                            NotPrimaryError,
                             OperationFailure,
                             PyMongoError)
+from pymongo.hello_compat import HelloCompat
+from pymongo._ipaddress import is_ip_address
 from pymongo.ismaster import IsMaster
 from pymongo.monotonic import time as _time
 from pymongo.monitoring import (ConnectionCheckOutFailedReason,
@@ -58,56 +60,12 @@ from pymongo.monitoring import (ConnectionCheckOutFailedReason,
 from pymongo.network import (command,
                              receive_message)
 from pymongo.read_preferences import ReadPreference
+from pymongo.server_api import _add_to_command
 from pymongo.server_type import SERVER_TYPE
 from pymongo.socket_checker import SocketChecker
 # Always use our backport so we always have support for IP address matching
 from pymongo.ssl_match_hostname import match_hostname
 
-# For SNI support. According to RFC6066, section 3, IPv4 and IPv6 literals are
-# not permitted for SNI hostname.
-try:
-    from ipaddress import ip_address
-    def is_ip_address(address):
-        try:
-            ip_address(_unicode(address))
-            return True
-        except (ValueError, UnicodeError):
-            return False
-except ImportError:
-    if hasattr(socket, 'inet_pton') and socket.has_ipv6:
-        # Most *nix, recent Windows
-        def is_ip_address(address):
-            try:
-                # inet_pton rejects IPv4 literals with leading zeros
-                # (e.g. 192.168.0.01), inet_aton does not, and we
-                # can connect to them without issue. Use inet_aton.
-                socket.inet_aton(address)
-                return True
-            except socket.error:
-                try:
-                    socket.inet_pton(socket.AF_INET6, address)
-                    return True
-                except socket.error:
-                    return False
-    else:
-        # No inet_pton
-        def is_ip_address(address):
-            try:
-                socket.inet_aton(address)
-                return True
-            except socket.error:
-                if ':' in address:
-                    # ':' is not a valid character for a hostname. If we get
-                    # here a few things have to be true:
-                    #   - We're on a recent version of python 2.7 (2.7.9+).
-                    #     Older 2.7 versions don't support SNI.
-                    #   - We're on Windows XP or some unusual Unix that doesn't
-                    #     have inet_pton.
-                    #   - The application is using IPv6 literals with TLS, which
-                    #     is pretty unusual.
-                    return True
-                return False
-
 try:
     from fcntl import fcntl, F_GETFD, F_SETFD, FD_CLOEXEC
     def _set_non_inheritable_non_atomic(fd):
@@ -262,6 +220,9 @@ else:
 # main thread, to avoid the deadlock. See PYTHON-607.
 u'foo'.encode('idna')
 
+# Remove after PYTHON-2712
+_MOCK_SERVICE_ID = False
+
 
 def _raise_connection_failure(address, error, msg_prefix=None):
     """Convert a socket.error to ConnectionFailure and raise it."""
@@ -294,7 +255,7 @@ class PoolOptions(object):
                  '__wait_queue_timeout', '__wait_queue_multiple',
                  '__ssl_context', '__ssl_match_hostname', '__socket_keepalive',
                  '__event_listeners', '__appname', '__driver', '__metadata',
-                 '__compression_settings')
+                 '__compression_settings', '__server_api', '__load_balanced')
 
     def __init__(self, max_pool_size=MAX_POOL_SIZE,
                  min_pool_size=MIN_POOL_SIZE,
@@ -303,8 +264,8 @@ class PoolOptions(object):
                  wait_queue_multiple=None, ssl_context=None,
                  ssl_match_hostname=True, socket_keepalive=True,
                  event_listeners=None, appname=None, driver=None,
-                 compression_settings=None):
-
+                 compression_settings=None, server_api=None,
+                 load_balanced=None):
         self.__max_pool_size = max_pool_size
         self.__min_pool_size = min_pool_size
         self.__max_idle_time_seconds = max_idle_time_seconds
@@ -319,6 +280,8 @@ class PoolOptions(object):
         self.__appname = appname
         self.__driver = driver
         self.__compression_settings = compression_settings
+        self.__server_api = server_api
+        self.__load_balanced = load_balanced
         self.__metadata = copy.deepcopy(_METADATA)
         if appname:
             self.__metadata['application'] = {'name': appname}
@@ -442,13 +405,13 @@ class PoolOptions(object):
 
     @property
     def appname(self):
-        """The application name, for sending with ismaster in server handshake.
+        """The application name, for sending with hello in server handshake.
         """
         return self.__appname
 
     @property
     def driver(self):
-        """Driver name and version, for sending with ismaster in handshake.
+        """Driver name and version, for sending with hello in handshake.
         """
         return self.__driver
 
@@ -462,6 +425,18 @@ class PoolOptions(object):
         """
         return self.__metadata.copy()
 
+    @property
+    def server_api(self):
+        """A pymongo.server_api.ServerApi or None.
+        """
+        return self.__server_api
+
+    @property
+    def load_balanced(self):
+        """True if this Pool is configured in load balanced mode.
+        """
+        return self.__load_balanced
+
 
 def _negotiate_creds(all_credentials):
     """Return one credential that needs mechanism negotiation, if any.
@@ -506,6 +481,7 @@ class SocketInfo(object):
       - `id`: the id of this socket in it's pool
     """
     def __init__(self, sock, pool, address, id):
+        self.pool_ref = weakref.ref(pool)
         self.sock = sock
         self.address = address
         self.id = id
@@ -519,6 +495,7 @@ class SocketInfo(object):
         self.max_message_size = MAX_MESSAGE_SIZE
         self.max_write_batch_size = MAX_WRITE_BATCH_SIZE
         self.supports_sessions = False
+        self.hello_ok = None
         self.is_mongos = False
         self.op_msg_enabled = False
         self.listeners = pool.opts.event_listeners
@@ -533,7 +510,8 @@ class SocketInfo(object):
 
         # The pool's generation changes with each reset() so we can close
         # sockets created before the last reset.
-        self.generation = pool.generation
+        self.pool_gen = pool.gen
+        self.generation = self.pool_gen.get_overall()
         self.ready = False
         self.cancel_context = None
         if not pool.handshake:
@@ -541,13 +519,41 @@ class SocketInfo(object):
             self.cancel_context = _CancellationContext()
         self.opts = pool.opts
         self.more_to_come = False
+        # For load balancer support.
+        self.service_id = None
+        # When executing a transaction in load balancing mode, this flag is
+        # set to true to indicate that the session now owns the connection.
+        self.pinned_txn = False
+        self.pinned_cursor = False
+        self.active = False
 
-    def ismaster(self, all_credentials=None):
-        return self._ismaster(None, None, None, all_credentials)
+    def pin_txn(self):
+        self.pinned_txn = True
+        assert not self.pinned_cursor
 
-    def _ismaster(self, cluster_time, topology_version,
+    def pin_cursor(self):
+        self.pinned_cursor = True
+        assert not self.pinned_txn
+
+    def unpin(self):
+        pool = self.pool_ref()
+        if pool:
+            pool.return_socket(self)
+        else:
+            self.close_socket(ConnectionClosedReason.STALE)
+
+    def hello_cmd(self):
+        if self.opts.server_api or self.hello_ok:
+            return SON([(HelloCompat.CMD, 1)])
+        else:
+            return SON([(HelloCompat.LEGACY_CMD, 1), ('helloOk', True)])
+
+    def hello(self, all_credentials=None):
+        return self._hello(None, None, None, all_credentials)
+
+    def _hello(self, cluster_time, topology_version,
                   heartbeat_frequency, all_credentials):
-        cmd = SON([('ismaster', 1)])
+        cmd = self.hello_cmd()
         performing_handshake = not self.performed_handshake
         awaitable = False
         if performing_handshake:
@@ -555,6 +561,8 @@ class SocketInfo(object):
             cmd['client'] = self.opts.metadata
             if self.compression_settings:
                 cmd['compression'] = self.compression_settings.compressors
+            if self.opts.load_balanced:
+                cmd['loadBalanced'] = True
         elif topology_version is not None:
             cmd['topologyVersion'] = topology_version
             cmd['maxAwaitTimeMS'] = int(heartbeat_frequency*1000)
@@ -578,28 +586,42 @@ class SocketInfo(object):
 
         doc = self.command('admin', cmd, publish_events=False,
                            exhaust_allowed=awaitable)
-        ismaster = IsMaster(doc, awaitable=awaitable)
-        self.is_writable = ismaster.is_writable
-        self.max_wire_version = ismaster.max_wire_version
-        self.max_bson_size = ismaster.max_bson_size
-        self.max_message_size = ismaster.max_message_size
-        self.max_write_batch_size = ismaster.max_write_batch_size
+        # PYTHON-2712 will remove this topologyVersion fallback logic.
+        if self.opts.load_balanced and _MOCK_SERVICE_ID:
+            process_id = doc.get('topologyVersion', {}).get('processId')
+            doc.setdefault('serviceId', process_id)
+        if not self.opts.load_balanced:
+            doc.pop('serviceId', None)
+        hello = IsMaster(doc, awaitable=awaitable)
+        self.is_writable = hello.is_writable
+        self.max_wire_version = hello.max_wire_version
+        self.max_bson_size = hello.max_bson_size
+        self.max_message_size = hello.max_message_size
+        self.max_write_batch_size = hello.max_write_batch_size
         self.supports_sessions = (
-            ismaster.logical_session_timeout_minutes is not None)
-        self.is_mongos = ismaster.server_type == SERVER_TYPE.Mongos
+            hello.logical_session_timeout_minutes is not None)
+        self.hello_ok = hello.hello_ok
+        self.is_mongos = hello.server_type == SERVER_TYPE.Mongos
         if performing_handshake and self.compression_settings:
             ctx = self.compression_settings.get_compression_context(
-                ismaster.compressors)
+                hello.compressors)
             self.compression_context = ctx
 
-        self.op_msg_enabled = ismaster.max_wire_version >= 6
+        self.op_msg_enabled = hello.max_wire_version >= 6
         if creds:
-            self.negotiated_mechanisms[creds] = ismaster.sasl_supported_mechs
+            self.negotiated_mechanisms[creds] = hello.sasl_supported_mechs
         if auth_ctx:
-            auth_ctx.parse_response(ismaster)
+            auth_ctx.parse_response(hello)
             if auth_ctx.speculate_succeeded():
                 self.auth_ctx[auth_ctx.credentials] = auth_ctx
-        return ismaster
+        if self.opts.load_balanced:
+            if not hello.service_id:
+                raise ConfigurationError(
+                    'Driver attempted to initialize in load balancing mode,'
+                    ' but the server does not support this mode')
+            self.service_id = hello.service_id
+            self.generation = self.pool_gen.get(self.service_id)
+        return hello
 
     def _next_reply(self):
         reply = self.receive_message(None)
@@ -607,9 +629,12 @@ class SocketInfo(object):
         unpacked_docs = reply.unpack_response()
         response_doc = unpacked_docs[0]
         helpers._check_command_response(response_doc, self.max_wire_version)
+        # Remove after PYTHON-2712.
+        if not self.opts.load_balanced:
+            response_doc.pop('serviceId', None)
         return response_doc
 
-    def command(self, dbname, spec, slave_ok=False,
+    def command(self, dbname, spec, secondary_ok=False,
                 read_preference=ReadPreference.PRIMARY,
                 codec_options=DEFAULT_CODEC_OPTIONS, check=True,
                 allowable_errors=None, check_keys=False,
@@ -628,7 +653,7 @@ class SocketInfo(object):
         :Parameters:
           - `dbname`: name of the database on which to run the command
           - `spec`: a command document as a dict, SON, or mapping object
-          - `slave_ok`: whether to set the SlaveOkay wire protocol bit
+          - `secondary_ok`: whether to set the secondaryOkay wire protocol bit
           - `read_preference`: a read preference
           - `codec_options`: a CodecOptions instance
           - `check`: raise OperationFailure if there are errors
@@ -672,15 +697,17 @@ class SocketInfo(object):
             raise ConfigurationError(
                 'Must be connected to MongoDB 3.4+ to use a collation.')
 
+        self.add_server_api(spec)
         if session:
-            session._apply_to(spec, retryable_write, read_preference)
+            session._apply_to(spec, retryable_write, read_preference,
+                              self)
         self.send_cluster_time(spec, session, client)
         listeners = self.listeners if publish_events else None
         unacknowledged = write_concern and not write_concern.acknowledged
         if self.op_msg_enabled:
             self._raise_if_not_writable(unacknowledged)
         try:
-            return command(self, dbname, spec, slave_ok,
+            return command(self, dbname, spec, secondary_ok,
                            self.is_mongos, read_preference, codec_options,
                            session, client, check, allowable_errors,
                            self.address, check_keys, listeners,
@@ -692,7 +719,7 @@ class SocketInfo(object):
                            unacknowledged=unacknowledged,
                            user_fields=user_fields,
                            exhaust_allowed=exhaust_allowed)
-        except OperationFailure:
+        except (OperationFailure, NotPrimaryError):
             raise
         # Catch socket.error, KeyboardInterrupt, etc. and close ourselves.
         except BaseException as error:
@@ -726,13 +753,13 @@ class SocketInfo(object):
             self._raise_connection_failure(error)
 
     def _raise_if_not_writable(self, unacknowledged):
-        """Raise NotMasterError on unacknowledged write if this socket is not
+        """Raise NotPrimaryError on unacknowledged write if this socket is not
         writable.
         """
         if unacknowledged and not self.is_writable:
-            # Write won't succeed, bail as if we'd received a not master error.
-            raise NotMasterError("not master", {
-                "ok": 0, "errmsg": "not master", "code": 10107})
+            # Write won't succeed, bail as if we'd received a not primary error.
+            raise NotPrimaryError("not primary", {
+                "ok": 0, "errmsg": "not primary", "code": 10107})
 
     def legacy_write(self, request_id, msg, max_doc_size, with_last_error):
         """Send OP_INSERT, etc., optionally returning response as a dict.
@@ -767,7 +794,7 @@ class SocketInfo(object):
         reply = self.receive_message(request_id)
         result = reply.command_response()
 
-        # Raises NotMasterError or OperationFailure.
+        # Raises NotPrimaryError or OperationFailure.
         helpers._check_command_response(result, self.max_wire_version)
         return result
 
@@ -861,6 +888,11 @@ class SocketInfo(object):
         if self.max_wire_version >= 6 and client:
             client._send_cluster_time(command, session)
 
+    def add_server_api(self, command):
+        """Add server_api parameters."""
+        if self.opts.server_api:
+            _add_to_command(command, self.opts.server_api)
+
     def update_last_checkin_time(self):
         self.last_checkin_time = _time()
 
@@ -885,7 +917,13 @@ class SocketInfo(object):
         # ...) is called in Python code, which experiences the signal as a
         # KeyboardInterrupt from the start, rather than as an initial
         # socket.error, so we catch that, close the socket, and reraise it.
-        self.close_socket(ConnectionClosedReason.ERROR)
+        #
+        # The connection closed event will be emitted later in return_socket.
+        if self.ready:
+            reason = None
+        else:
+            reason = ConnectionClosedReason.ERROR
+        self.close_socket(reason)
         # SSLError from PyOpenSSL inherits directly from Exception.
         if isinstance(error, (IOError, OSError, _SSLError)):
             _raise_connection_failure(self.address, error)
@@ -1033,6 +1071,43 @@ class _PoolClosedError(PyMongoError):
     pass
 
 
+class _PoolGeneration(object):
+    def __init__(self):
+        # Maps service_id to generation.
+        self._generations = collections.defaultdict(int)
+        # Overall pool generation.
+        self._generation = 0
+
+    def get(self, service_id):
+        """Get the generation for the given service_id."""
+        if service_id is None:
+            return self._generation
+        return self._generations[service_id]
+
+    def get_overall(self):
+        """Get the Pool's overall generation."""
+        return self._generation
+
+    def inc(self, service_id):
+        """Increment the generation for the given service_id."""
+        self._generation += 1
+        if service_id is None:
+            for service_id in self._generations:
+                self._generations[service_id] += 1
+        else:
+            self._generations[service_id] += 1
+
+    def stale(self, gen, service_id):
+        """Return if the given generation for a given service_id is stale."""
+        return gen != self.get(service_id)
+
+
+class PoolState(object):
+    PAUSED = 1
+    READY = 2
+    CLOSED = 3
+
+
 # Do *not* explicitly inherit from object or Jython won't call __del__
 # http://bugs.jython.org/issue1057
 class Pool:
@@ -1041,7 +1116,7 @@ class Pool:
         :Parameters:
           - `address`: a (hostname, port) tuple
           - `options`: a PoolOptions instance
-          - `handshake`: whether to call ismaster for each new SocketInfo
+          - `handshake`: whether to call hello for each new SocketInfo
         """
         # Check a socket's health with socket_closed() every once in a while.
         # Can override for testing: 0 to always check, None to never check.
@@ -1060,7 +1135,8 @@ class Pool:
 
         # Keep track of resets, so we notice sockets created before the most
         # recent reset and close them.
-        self.generation = 0
+        # self.generation = 0
+        self.gen = _PoolGeneration()
         self.pid = os.getpid()
         self.address = address
         self.opts = options
@@ -1083,15 +1159,35 @@ class Pool:
         if self.enabled_for_cmap:
             self.opts.event_listeners.publish_pool_created(
                 self.address, self.opts.non_default_options)
+        # Retain references to pinned connections to prevent the CPython GC
+        # from thinking that a cursor's pinned connection can be GC'd when the
+        # cursor is GC'd (see PYTHON-2751).
+        self.__pinned_sockets = set()
+        self.ncursors = 0
+        self.ntxns = 0
 
-    def _reset(self, close):
+    def _reset(self, close, service_id=None):
         with self.lock:
             if self.closed:
                 return
-            self.generation += 1
-            self.pid = os.getpid()
-            sockets, self.sockets = self.sockets, collections.deque()
-            self.active_sockets = 0
+            self.gen.inc(service_id)
+            newpid = os.getpid()
+            if self.pid != newpid:
+                self.pid = newpid
+                self.active_sockets = 0
+            if service_id is None:
+                sockets, self.sockets = self.sockets, collections.deque()
+            else:
+                discard = collections.deque()
+                keep = collections.deque()
+                for sock_info in self.sockets:
+                    if sock_info.service_id == service_id:
+                        discard.append(sock_info)
+                    else:
+                        keep.append(sock_info)
+                sockets = discard
+                self.sockets = keep
+
             if close:
                 self.closed = True
 
@@ -1106,7 +1202,8 @@ class Pool:
                 listeners.publish_pool_closed(self.address)
         else:
             if self.enabled_for_cmap:
-                listeners.publish_pool_cleared(self.address)
+                listeners.publish_pool_cleared(self.address,
+                                               service_id=service_id)
             for sock_info in sockets:
                 sock_info.close_socket(ConnectionClosedReason.STALE)
 
@@ -1119,12 +1216,15 @@ class Pool:
             for socket in self.sockets:
                 socket.update_is_writable(self.is_writable)
 
-    def reset(self):
-        self._reset(close=False)
+    def reset(self, service_id=None):
+        self._reset(close=False, service_id=service_id)
 
     def close(self):
         self._reset(close=True)
 
+    def stale_generation(self, gen, service_id):
+        return self.gen.stale(gen, service_id)
+
     def remove_stale_sockets(self, reference_generation, all_credentials):
         """Removes stale sockets then adds new ones if pool is too small and
         has not been reset. The `reference_generation` argument specifies the
@@ -1153,7 +1253,7 @@ class Pool:
                 with self.lock:
                     # Close connection and return if the pool was reset during
                     # socket creation or while acquiring the pool lock.
-                    if self.generation != reference_generation:
+                    if self.gen.get_overall() != reference_generation:
                         sock_info.close_socket(ConnectionClosedReason.STALE)
                         break
                     self.sockets.appendleft(sock_info)
@@ -1178,7 +1278,7 @@ class Pool:
 
         try:
             sock = _configured_socket(self.address, self.opts)
-        except Exception as error:
+        except BaseException as error:
             if self.enabled_for_cmap:
                 listeners.publish_connection_closed(
                     self.address, conn_id, ConnectionClosedReason.ERROR)
@@ -1189,20 +1289,20 @@ class Pool:
             raise
 
         sock_info = SocketInfo(sock, self, self.address, conn_id)
-        if self.handshake:
-            sock_info.ismaster(all_credentials)
-            self.is_writable = sock_info.is_writable
-
         try:
+            if self.handshake:
+                sock_info.hello(all_credentials)
+                self.is_writable = sock_info.is_writable
+
             sock_info.check_auth(all_credentials)
-        except Exception:
+        except BaseException:
             sock_info.close_socket(ConnectionClosedReason.ERROR)
             raise
 
         return sock_info
 
     @contextlib.contextmanager
-    def get_socket(self, all_credentials, checkout=False):
+    def get_socket(self, all_credentials, handler=None):
         """Get a socket from the pool. Use with a "with" statement.
 
         Returns a :class:`SocketInfo` object wrapping a connected
@@ -1210,7 +1310,7 @@ class Pool:
 
         This method should always be used in a with-statement::
 
-            with pool.get_socket(credentials, checkout) as socket_info:
+            with pool.get_socket(credentials) as socket_info:
                 socket_info.send_message(msg)
                 data = socket_info.receive_message(op_code, request_id)
 
@@ -1222,26 +1322,42 @@ class Pool:
 
         :Parameters:
           - `all_credentials`: dict, maps auth source to MongoCredential.
-          - `checkout` (optional): keep socket checked out.
+          - `handler` (optional): A _MongoClientErrorHandler.
         """
         listeners = self.opts.event_listeners
         if self.enabled_for_cmap:
             listeners.publish_connection_check_out_started(self.address)
 
         sock_info = self._get_socket(all_credentials)
-
         if self.enabled_for_cmap:
             listeners.publish_connection_checked_out(
                 self.address, sock_info.id)
         try:
             yield sock_info
         except:
-            # Exception in caller. Decrement semaphore.
-            self.return_socket(sock_info)
-            raise
-        else:
-            if not checkout:
+            # Exception in caller. Ensure the connection gets returned.
+            # Note that when pinned is True, the session owns the
+            # connection and it is responsible for checking the connection
+            # back into the pool.
+            pinned = sock_info.pinned_txn or sock_info.pinned_cursor
+            if handler:
+                # Perform SDAM error handling rules while the connection is
+                # still checked out.
+                exc_type, exc_val, _ = sys.exc_info()
+                handler.handle(exc_type, exc_val)
+            if not pinned and sock_info.active:
                 self.return_socket(sock_info)
+            raise
+        if sock_info.pinned_txn:
+            with self.lock:
+                self.__pinned_sockets.add(sock_info)
+                self.ntxns += 1
+        elif sock_info.pinned_cursor:
+            with self.lock:
+                self.__pinned_sockets.add(sock_info)
+                self.ncursors += 1
+        elif sock_info.active:
+            self.return_socket(sock_info)
 
     def _get_socket(self, all_credentials):
         """Get or create a SocketInfo. Can raise ConnectionFailure."""
@@ -1283,7 +1399,7 @@ class Pool:
                     if self._perished(sock_info):
                         sock_info = None
             sock_info.check_auth(all_credentials)
-        except Exception:
+        except BaseException:
             if sock_info:
                 # We checked out a socket but authentication failed.
                 sock_info.close_socket(ConnectionClosedReason.ERROR)
@@ -1298,6 +1414,7 @@ class Pool:
                     self.address, ConnectionCheckOutFailedReason.CONN_ERROR)
             raise
 
+        sock_info.active = True
         return sock_info
 
     def return_socket(self, sock_info):
@@ -1306,6 +1423,12 @@ class Pool:
         :Parameters:
           - `sock_info`: The socket to check into the pool.
         """
+        txn = sock_info.pinned_txn
+        cursor = sock_info.pinned_cursor
+        sock_info.active = False
+        sock_info.pinned_txn = False
+        sock_info.pinned_cursor = False
+        self.__pinned_sockets.discard(sock_info)
         listeners = self.opts.event_listeners
         if self.enabled_for_cmap:
             listeners.publish_connection_checked_in(self.address, sock_info.id)
@@ -1314,11 +1437,18 @@ class Pool:
         else:
             if self.closed:
                 sock_info.close_socket(ConnectionClosedReason.POOL_CLOSED)
-            elif not sock_info.closed:
+            elif sock_info.closed:
+                # CMAP requires the closed event be emitted after the check in.
+                if self.enabled_for_cmap:
+                    listeners.publish_connection_closed(
+                        self.address, sock_info.id,
+                        ConnectionClosedReason.ERROR)
+            else:
                 with self.lock:
                     # Hold the lock to ensure this section does not race with
                     # Pool.reset().
-                    if sock_info.generation != self.generation:
+                    if self.stale_generation(sock_info.generation,
+                                             sock_info.service_id):
                         sock_info.close_socket(ConnectionClosedReason.STALE)
                     else:
                         sock_info.update_last_checkin_time()
@@ -1327,6 +1457,10 @@ class Pool:
 
         self._socket_semaphore.release()
         with self.lock:
+            if txn:
+                self.ntxns -= 1
+            elif cursor:
+                self.ncursors -= 1
             self.active_sockets -= 1
 
     def _perished(self, sock_info):
@@ -1357,7 +1491,7 @@ class Pool:
                 sock_info.close_socket(ConnectionClosedReason.ERROR)
                 return True
 
-        if sock_info.generation != self.generation:
+        if self.stale_generation(sock_info.generation, sock_info.service_id):
             sock_info.close_socket(ConnectionClosedReason.STALE)
             return True
 
@@ -1368,9 +1502,18 @@ class Pool:
         if self.enabled_for_cmap:
             listeners.publish_connection_check_out_failed(
                 self.address, ConnectionCheckOutFailedReason.TIMEOUT)
+        if self.opts.load_balanced:
+            other_ops = self.active_sockets - self.ncursors - self.ntxns
+            raise ConnectionFailure(
+                'Timeout waiting for connection from the connection pool. '
+                'maxPoolSize: %s, connections in use by cursors: %s, '
+                'connections in use by transactions: %s, connections in use '
+                'by other operations: %s, wait_queue_timeout: %s' % (
+                    self.opts.max_pool_size, self.ncursors, self.ntxns,
+                    other_ops, self.opts.wait_queue_timeout))
         raise ConnectionFailure(
-            'Timed out while checking out a connection from connection pool '
-            'with max_size %r and wait_queue_timeout %r' % (
+            'Timed out while checking out a connection from connection pool. '
+            'maxPoolSize: %s, wait_queue_timeout: %s' % (
                 self.opts.max_pool_size, self.opts.wait_queue_timeout))
 
     def __del__(self):

pymongo/read_preferences.py

@@ -268,6 +268,10 @@ class PrimaryPreferred(_ServerMode):
     * When connected to a replica set queries are sent to the primary if
       available, otherwise a secondary.
 
+    .. note:: When a :class:`~pymongo.mongo_client.MongoClient` is first
+      created reads will be routed to an available secondary until the
+      primary of the replica set is discovered.
+
     :Parameters:
       - `tag_sets`: The :attr:`~tag_sets` to use if the primary is not
         available.
@@ -346,6 +350,10 @@ class SecondaryPreferred(_ServerMode):
     * When connected to a replica set queries are distributed among
       secondaries, or the primary if no secondary is available.
 
+    .. note:: When a :class:`~pymongo.mongo_client.MongoClient` is first
+      created reads will be routed to the primary of the replica set until
+      an available secondary is discovered.
+
     :Parameters:
       - `tag_sets`: The :attr:`~tag_sets` for this read preference.
       - `max_staleness`: (integer, in seconds) The maximum estimated
@@ -510,7 +518,7 @@ class MovingAverage(object):
 
     def add_sample(self, sample):
         if sample < 0:
-            # Likely system time change while waiting for ismaster response
+            # Likely system time change while waiting for hello response
             # and not using time.monotonic. Ignore it, the next one will
             # probably be valid.
             return

pymongo/response.py

@@ -67,11 +67,12 @@ class Response(object):
         """The decoded document(s)."""
         return self._docs
 
-class ExhaustResponse(Response):
-    __slots__ = ('_socket_info', '_pool')
 
-    def __init__(self, data, address, socket_info, pool, request_id, duration,
-                 from_command, docs):
+class PinnedResponse(Response):
+    __slots__ = ('_socket_info', '_more_to_come')
+
+    def __init__(self, data, address, socket_info, request_id, duration,
+                 from_command, docs, more_to_come):
         """Represent a response to an exhaust cursor's initial query.
 
         :Parameters:
@@ -82,14 +83,17 @@ class ExhaustResponse(Response):
           - `request_id`: The request id of this operation.
           - `duration`: The duration of the operation.
           - `from_command`: If the response is the result of a db command.
+          - `docs`: List of documents.
+          - `more_to_come`: Bool indicating whether cursor is ready to be
+            exhausted.
         """
-        super(ExhaustResponse, self).__init__(data,
-                                              address,
-                                              request_id,
-                                              duration,
-                                              from_command, docs)
+        super(PinnedResponse, self).__init__(data,
+                                             address,
+                                             request_id,
+                                             duration,
+                                             from_command, docs)
         self._socket_info = socket_info
-        self._pool = pool
+        self._more_to_come = more_to_come
 
     @property
     def socket_info(self):
@@ -102,6 +106,7 @@ class ExhaustResponse(Response):
         return self._socket_info
 
     @property
-    def pool(self):
-        """The Pool from which the SocketInfo came."""
-        return self._pool
+    def more_to_come(self):
+        """If true, server is ready to send batches on the socket until the
+        result set is exhausted or there is an error."""
+        return self._more_to_come

pymongo/server.py

@@ -18,10 +18,10 @@ from datetime import datetime
 
 from bson import _decode_all_selective
 
-from pymongo.errors import NotMasterError, OperationFailure
+from pymongo.errors import NotPrimaryError, OperationFailure
 from pymongo.helpers import _check_command_response
-from pymongo.message import _convert_exception
-from pymongo.response import Response, ExhaustResponse
+from pymongo.message import _convert_exception, _OpMsg
+from pymongo.response import Response, PinnedResponse
 from pymongo.server_type import SERVER_TYPE
 
 _CURSOR_DOC_FIELDS = {'cursor': {'firstBatch': 1, 'nextBatch': 1}}
@@ -46,11 +46,12 @@ class Server(object):
 
         Multiple calls have no effect.
         """
-        self._monitor.open()
+        if not self._pool.opts.load_balanced:
+            self._monitor.open()
 
-    def reset(self):
+    def reset(self, service_id=None):
         """Clear the connection pool."""
-        self.pool.reset()
+        self.pool.reset(service_id)
 
     def close(self):
         """Clear the connection pool and stop the monitor.
@@ -67,14 +68,9 @@ class Server(object):
         """Check the server's state soon."""
         self._monitor.request_check()
 
-    def run_operation_with_response(
-            self,
-            sock_info,
-            operation,
-            set_slave_okay,
-            listeners,
-            exhaust,
-            unpack_res):
+    def run_operation(
+            self, sock_info, operation,
+            set_secondary_okay, listeners, unpack_res):
         """Run a _Query or _GetMore operation and return a Response object.
 
         This method is used only to run _Query/_GetMore operations from
@@ -83,10 +79,9 @@ class Server(object):
 
         :Parameters:
           - `operation`: A _Query or _GetMore object.
-          - `set_slave_okay`: Pass to operation.get_message.
+          - `set_secondary_okay`: Pass to operation.get_message.
           - `all_credentials`: dict, maps auth source to MongoCredential.
           - `listeners`: Instance of _EventListeners or None.
-          - `exhaust`: If True, then this is an exhaust cursor operation.
           - `unpack_res`: A callable that decodes the wire protocol response.
         """
         duration = None
@@ -94,29 +89,29 @@ class Server(object):
         if publish:
             start = datetime.now()
 
-        send_message = not operation.exhaust_mgr
-
-        if send_message:
-            use_cmd = operation.use_command(sock_info, exhaust)
-            message = operation.get_message(
-                set_slave_okay, sock_info, use_cmd)
-            request_id, data, max_doc_size = self._split_message(message)
-        else:
-            use_cmd = False
+        use_cmd = operation.use_command(sock_info)
+        more_to_come = (operation.sock_mgr
+                        and operation.sock_mgr.more_to_come)
+        if more_to_come:
             request_id = 0
+        else:
+            message = operation.get_message(
+                set_secondary_okay, sock_info, use_cmd)
+            request_id, data, max_doc_size = self._split_message(message)
 
         if publish:
             cmd, dbn = operation.as_command(sock_info)
             listeners.publish_command_start(
-                cmd, dbn, request_id, sock_info.address)
+                cmd, dbn, request_id, sock_info.address,
+                service_id=sock_info.service_id)
             start = datetime.now()
 
         try:
-            if send_message:
+            if more_to_come:
+                reply = sock_info.receive_message(None)
+            else:
                 sock_info.send_message(data, max_doc_size)
                 reply = sock_info.receive_message(request_id)
-            else:
-                reply = sock_info.receive_message(None)
 
             # Unpack and check for command errors.
             if use_cmd:
@@ -136,13 +131,14 @@ class Server(object):
         except Exception as exc:
             if publish:
                 duration = datetime.now() - start
-                if isinstance(exc, (NotMasterError, OperationFailure)):
+                if isinstance(exc, (NotPrimaryError, OperationFailure)):
                     failure = exc.details
                 else:
                     failure = _convert_exception(exc)
                 listeners.publish_command_failure(
                     duration, failure, operation.name,
-                    request_id, sock_info.address)
+                    request_id, sock_info.address,
+                    service_id=sock_info.service_id)
             raise
 
         if publish:
@@ -163,7 +159,7 @@ class Server(object):
                     res["cursor"]["nextBatch"] = docs
             listeners.publish_command_success(
                 duration, res, operation.name, request_id,
-                sock_info.address)
+                sock_info.address, service_id=sock_info.service_id)
 
         # Decrypt response.
         client = operation.client
@@ -174,16 +170,26 @@ class Server(object):
                 docs = _decode_all_selective(
                     decrypted, operation.codec_options, user_fields)
 
-        if exhaust:
-            response = ExhaustResponse(
+        if client._should_pin_cursor(operation.session) or operation.exhaust:
+            sock_info.pin_cursor()
+            if isinstance(reply, _OpMsg):
+                # In OP_MSG, the server keeps sending only if the
+                # more_to_come flag is set.
+                more_to_come = reply.more_to_come
+            else:
+                # In OP_REPLY, the server keeps sending until cursor_id is 0.
+                more_to_come = bool(operation.exhaust and reply.cursor_id)
+            if operation.sock_mgr:
+                operation.sock_mgr.update_exhaust(more_to_come)
+            response = PinnedResponse(
                 data=reply,
                 address=self._description.address,
                 socket_info=sock_info,
-                pool=self._pool,
                 duration=duration,
                 request_id=request_id,
                 from_command=use_cmd,
-                docs=docs)
+                docs=docs,
+                more_to_come=more_to_come)
         else:
             response = Response(
                 data=reply,
@@ -195,8 +201,8 @@ class Server(object):
 
         return response
 
-    def get_socket(self, all_credentials, checkout=False):
-        return self.pool.get_socket(all_credentials, checkout)
+    def get_socket(self, all_credentials, handler=None):
+        return self.pool.get_socket(all_credentials, handler)
 
     @property
     def description(self):

pymongo/server_api.py

@@ -0,0 +1,168 @@
+# Copyright 2020-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Support for MongoDB Versioned API.
+
+.. _versioned-api-ref:
+
+MongoDB Versioned API
+=====================
+
+Starting in MongoDB 5.0, applications can specify the server API version
+to use when creating a :class:`~pymongo.mongo_client.MongoClient`. Doing so
+ensures that the driver behaves in a manner compatible with that server API
+version, regardless of the server's actual release version.
+
+Declaring an API Version
+````````````````````````
+
+.. attention:: Versioned API requires MongoDB >=5.0.
+
+To configure MongoDB Versioned API, pass the ``server_api`` keyword option to
+:class:`~pymongo.mongo_client.MongoClient`::
+
+    >>> from pymongo.mongo_client import MongoClient
+    >>> from pymongo.server_api import ServerApi
+    >>>
+    >>> # Declare API version "1" for MongoClient "client"
+    >>> server_api = ServerApi('1')
+    >>> client = MongoClient(server_api=server_api)
+
+The declared API version is applied to all commands run through ``client``,
+including those sent through the generic
+:meth:`~pymongo.database.Database.command` helper.
+
+.. note:: Declaring an API version on the
+   :class:`~pymongo.mongo_client.MongoClient` **and** specifying versioned
+   API options in :meth:`~pymongo.database.Database.command` command document
+   is not supported and will lead to undefined behaviour.
+
+To run any command without declaring a server API version or using a different
+API version, create a separate :class:`~pymongo.mongo_client.MongoClient`
+instance.
+
+Strict Mode
+```````````
+
+Configuring ``strict`` mode will cause the MongoDB server to reject all
+commands that are not part of the declared :attr:`ServerApi.version`. This
+includes command options and aggregation pipeline stages.
+
+For example::
+
+    >>> server_api = ServerApi('1', strict=True)
+    >>> client = MongoClient(server_api=server_api)
+    >>> client.test.command('count', 'test')
+    Traceback (most recent call last):
+    ...
+    pymongo.errors.OperationFailure: Provided apiStrict:true, but the command count is not in API Version 1, full error: {'ok': 0.0, 'errmsg': 'Provided apiStrict:true, but the command count is not in API Version 1', 'code': 323, 'codeName': 'APIStrictError'
+
+Detecting API Deprecations
+``````````````````````````
+
+The ``deprecationErrors`` option can be used to enable command failures
+when using functionality that is deprecated from the configured
+:attr:`ServerApi.version`. For example::
+
+    >>> server_api = ServerApi('1', deprecation_errors=True)
+    >>> client = MongoClient(server_api=server_api)
+
+Note that at the time of this writing, no deprecated APIs exist.
+
+Classes
+=======
+"""
+
+
+class ServerApiVersion:
+    """An enum that defines values for :attr:`ServerApi.version`.
+
+    .. versionadded:: 3.12
+    """
+
+    V1 = "1"
+    """Server API version "1"."""
+
+
+class ServerApi(object):
+    """MongoDB Versioned API."""
+    def __init__(self, version, strict=None, deprecation_errors=None):
+        """Options to configure MongoDB Versioned API.
+
+        :Parameters:
+          - `version`: The API version string. Must be one of the values in
+            :class:`ServerApiVersion`.
+          - `strict` (optional): Set to ``True`` to enable API strict mode.
+            Defaults to ``None`` which means "use the server's default".
+          - `deprecation_errors` (optional): Set to ``True`` to enable
+            deprecation errors. Defaults to ``None`` which means "use the
+            server's default".
+
+        .. versionadded:: 3.12
+        """
+        if version != ServerApiVersion.V1:
+            raise ValueError("Unknown ServerApi version: %s" % (version,))
+        if strict is not None and not isinstance(strict, bool):
+            raise TypeError(
+                "Wrong type for ServerApi strict, value must be an instance "
+                "of bool, not %s" % (type(strict),))
+        if (deprecation_errors is not None and
+                not isinstance(deprecation_errors, bool)):
+            raise TypeError(
+                "Wrong type for ServerApi deprecation_errors, value must be "
+                "an instance of bool, not %s" % (type(deprecation_errors),))
+        self._version = version
+        self._strict = strict
+        self._deprecation_errors = deprecation_errors
+
+    @property
+    def version(self):
+        """The API version setting.
+
+        This value is sent to the server in the "apiVersion" field.
+        """
+        return self._version
+
+    @property
+    def strict(self):
+        """The API strict mode setting.
+
+        When set, this value is sent to the server in the "apiStrict" field.
+        """
+        return self._strict
+
+    @property
+    def deprecation_errors(self):
+        """The API deprecation errors setting.
+
+        When set, this value is sent to the server in the
+        "apiDeprecationErrors" field.
+        """
+        return self._deprecation_errors
+
+
+def _add_to_command(cmd, server_api):
+    """Internal helper which adds API versioning options to a command.
+
+    :Parameters:
+      - `cmd`: The command.
+      - `server_api` (optional): A :class:`ServerApi` or ``None``.
+    """
+    if not server_api:
+        return
+    cmd['apiVersion'] = server_api.version
+    if server_api.strict is not None:
+        cmd['apiStrict'] = server_api.strict
+    if server_api.deprecation_errors is not None:
+        cmd['apiDeprecationErrors'] = server_api.deprecation_errors

pymongo/server_description.py

@@ -15,8 +15,8 @@
 """Represent one server the driver is connected to."""
 
 from bson import EPOCH_NAIVE
-from pymongo.server_type import SERVER_TYPE
 from pymongo.ismaster import IsMaster
+from pymongo.server_type import SERVER_TYPE
 from pymongo.monotonic import time as _time
 
 
@@ -25,9 +25,12 @@ class ServerDescription(object):
 
     :Parameters:
       - `address`: A (host, port) pair
-      - `ismaster`: Optional IsMaster instance
+      - `ismaster`: Optional Hello instance
       - `round_trip_time`: Optional float
       - `error`: Optional, the last error attempting to connect to the server
+
+    .. warning:: The `ismaster` parameter will be renamed to `hello` in PyMongo
+      4.0.
     """
 
     __slots__ = (
@@ -46,37 +49,36 @@ class ServerDescription(object):
             round_trip_time=None,
             error=None):
         self._address = address
-        if not ismaster:
-            ismaster = IsMaster({})
+        hello = ismaster or IsMaster({})
 
-        self._server_type = ismaster.server_type
-        self._all_hosts = ismaster.all_hosts
-        self._tags = ismaster.tags
-        self._replica_set_name = ismaster.replica_set_name
-        self._primary = ismaster.primary
-        self._max_bson_size = ismaster.max_bson_size
-        self._max_message_size = ismaster.max_message_size
-        self._max_write_batch_size = ismaster.max_write_batch_size
-        self._min_wire_version = ismaster.min_wire_version
-        self._max_wire_version = ismaster.max_wire_version
-        self._set_version = ismaster.set_version
-        self._election_id = ismaster.election_id
-        self._cluster_time = ismaster.cluster_time
-        self._is_writable = ismaster.is_writable
-        self._is_readable = ismaster.is_readable
-        self._ls_timeout_minutes = ismaster.logical_session_timeout_minutes
+        self._server_type = hello.server_type
+        self._all_hosts = hello.all_hosts
+        self._tags = hello.tags
+        self._replica_set_name = hello.replica_set_name
+        self._primary = hello.primary
+        self._max_bson_size = hello.max_bson_size
+        self._max_message_size = hello.max_message_size
+        self._max_write_batch_size = hello.max_write_batch_size
+        self._min_wire_version = hello.min_wire_version
+        self._max_wire_version = hello.max_wire_version
+        self._set_version = hello.set_version
+        self._election_id = hello.election_id
+        self._cluster_time = hello.cluster_time
+        self._is_writable = hello.is_writable
+        self._is_readable = hello.is_readable
+        self._ls_timeout_minutes = hello.logical_session_timeout_minutes
         self._round_trip_time = round_trip_time
-        self._me = ismaster.me
+        self._me = hello.me
         self._last_update_time = _time()
         self._error = error
-        self._topology_version = ismaster.topology_version
+        self._topology_version = hello.topology_version
         if error:
             if hasattr(error, 'details') and isinstance(error.details, dict):
                 self._topology_version = error.details.get('topologyVersion')
 
-        if ismaster.last_write_date:
+        if hello.last_write_date:
             # Convert from datetime to seconds.
-            delta = ismaster.last_write_date - EPOCH_NAIVE
+            delta = hello.last_write_date - EPOCH_NAIVE
             self._last_write_date = delta.total_seconds()
         else:
             self._last_write_date = None
@@ -203,9 +205,10 @@ class ServerDescription(object):
     @property
     def retryable_writes_supported(self):
         """Checks if this server supports retryable writes."""
-        return (
+        return ((
             self._ls_timeout_minutes is not None and
             self._server_type in (SERVER_TYPE.Mongos, SERVER_TYPE.RSPrimary))
+                or self._server_type == SERVER_TYPE.LoadBalancer)
 
     @property
     def retryable_reads_supported(self):

pymongo/server_type.py

@@ -20,4 +20,4 @@ from collections import namedtuple
 SERVER_TYPE = namedtuple('ServerType',
                          ['Unknown', 'Mongos', 'RSPrimary', 'RSSecondary',
                           'RSArbiter', 'RSOther', 'RSGhost',
-                          'Standalone'])(*range(8))
+                          'Standalone', 'LoadBalancer'])(*range(9))

pymongo/settings.py

@@ -39,7 +39,8 @@ class TopologySettings(object):
                  heartbeat_frequency=common.HEARTBEAT_FREQUENCY,
                  server_selector=None,
                  fqdn=None,
-                 direct_connection=None):
+                 direct_connection=None,
+                 load_balanced=None):
         """Represent MongoClient's configuration.
 
         Take a list of (host, port) pairs and optional replica set name.
@@ -65,6 +66,7 @@ class TopologySettings(object):
             self._direct = (len(self._seeds) == 1 and not self.replica_set_name)
         else:
             self._direct = direct_connection
+        self._load_balanced = load_balanced
 
         self._topology_id = ObjectId()
         # Store the allocation traceback to catch unclosed clients in the
@@ -124,8 +126,15 @@ class TopologySettings(object):
         """
         return self._direct
 
+    @property
+    def load_balanced(self):
+        """True if the client was configured to connect to a load balancer."""
+        return self._load_balanced
+
     def get_topology_type(self):
-        if self.direct:
+        if self.load_balanced:
+            return TOPOLOGY_TYPE.LoadBalanced
+        elif self.direct:
             return TOPOLOGY_TYPE.Single
         elif self.replica_set_name is not None:
             return TOPOLOGY_TYPE.ReplicaSetNoPrimary

pymongo/socket_checker.py

@@ -41,7 +41,7 @@ class SocketChecker(object):
             self._poller = None
 
     def select(self, sock, read=False, write=False, timeout=0):
-        """Select for reads or writes with a timeout in seconds.
+        """Select for reads or writes with a timeout in seconds (or None).
 
         Returns True if the socket is readable/writable, False on timeout.
         """
@@ -57,7 +57,8 @@ class SocketChecker(object):
                     try:
                         # poll() timeout is in milliseconds. select()
                         # timeout is in seconds.
-                        res = self._poller.poll(timeout * 1000)
+                        timeout_ = None if timeout is None else timeout * 1000
+                        res = self._poller.poll(timeout_)
                         # poll returns a possibly-empty list containing
                         # (fd, event) 2-tuples for the descriptors that have
                         # events or errors to report. Return True if the list

pymongo/srv_resolver.py

@@ -24,6 +24,7 @@ from bson.py3compat import PY3
 
 from pymongo.common import CONNECT_TIMEOUT
 from pymongo.errors import ConfigurationError
+from pymongo._ipaddress import is_ip_address
 
 
 if PY3:
@@ -38,24 +39,39 @@ else:
         return text
 
 
+# PYTHON-2667 Lazily call dns.resolver methods for compatibility with eventlet.
+def _resolve(*args, **kwargs):
+    if hasattr(resolver, 'resolve'):
+        # dnspython >= 2
+        return resolver.resolve(*args, **kwargs)
+    # dnspython 1.X
+    return resolver.query(*args, **kwargs)
+
+_INVALID_HOST_MSG = (
+    "Invalid URI host: %s is not a valid hostname for 'mongodb+srv://'. "
+    "Did you mean to use 'mongodb://'?")
+
 class _SrvResolver(object):
     def __init__(self, fqdn, connect_timeout=None):
         self.__fqdn = fqdn
         self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT
 
         # Validate the fully qualified domain name.
+        if is_ip_address(fqdn):
+            raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",))
+
         try:
             self.__plist = self.__fqdn.split(".")[1:]
         except Exception:
-            raise ConfigurationError("Invalid URI host: %s" % (fqdn,))
+            raise ConfigurationError(_INVALID_HOST_MSG % (fqdn,))
         self.__slen = len(self.__plist)
         if self.__slen < 2:
-            raise ConfigurationError("Invalid URI host: %s" % (fqdn,))
+            raise ConfigurationError(_INVALID_HOST_MSG % (fqdn,))
 
     def get_options(self):
         try:
-            results = resolver.query(self.__fqdn, 'TXT',
-                                     lifetime=self.__connect_timeout)
+            results = _resolve(self.__fqdn, 'TXT',
+                               lifetime=self.__connect_timeout)
         except (resolver.NoAnswer, resolver.NXDOMAIN):
             # No TXT records
             return None
@@ -69,8 +85,8 @@ class _SrvResolver(object):
 
     def _resolve_uri(self, encapsulate_errors):
         try:
-            results = resolver.query('_mongodb._tcp.' + self.__fqdn, 'SRV',
-                                     lifetime=self.__connect_timeout)
+            results = _resolve('_mongodb._tcp.' + self.__fqdn, 'SRV',
+                               lifetime=self.__connect_timeout)
         except Exception as exc:
             if not encapsulate_errors:
                 # Raise the original error.

pymongo/topology.py

@@ -29,6 +29,7 @@ else:
 from pymongo import (common,
                      helpers,
                      periodic_executor)
+from pymongo.ismaster import IsMaster
 from pymongo.pool import PoolOptions
 from pymongo.topology_description import (updated_topology_description,
                                           _updated_topology_description_srv_polling,
@@ -37,9 +38,10 @@ from pymongo.topology_description import (updated_topology_description,
 from pymongo.errors import (ConnectionFailure,
                             ConfigurationError,
                             NetworkTimeout,
-                            NotMasterError,
+                            NotPrimaryError,
                             OperationFailure,
-                            ServerSelectionTimeoutError)
+                            ServerSelectionTimeoutError,
+                            WriteError)
 from pymongo.monitor import SrvMonitor
 from pymongo.monotonic import time as _time
 from pymongo.server import Server
@@ -139,7 +141,8 @@ class Topology(object):
             executor.open()
 
         self._srv_monitor = None
-        if self._settings.fqdn is not None:
+        if (self._settings.fqdn is not None and
+                not self._settings.load_balanced):
             self._srv_monitor = SrvMonitor(self, self._settings)
 
     def open(self):
@@ -273,7 +276,7 @@ class Topology(object):
         td_old = self._description
         sd_old = td_old._server_descriptions[server_description.address]
         if _is_stale_server_description(sd_old, server_description):
-            # This is a stale isMaster response. Ignore it.
+            # This is a stale hello response. Ignore it.
             return
 
         suppress_event = ((self._publish_server or self._publish_tp)
@@ -313,10 +316,10 @@ class Topology(object):
         self._condition.notify_all()
 
     def on_change(self, server_description, reset_pool=False):
-        """Process a new ServerDescription after an ismaster call completes."""
+        """Process a new ServerDescription after a hello call completes."""
         # We do no I/O holding the lock.
         with self._lock:
-            # Monitors may continue working on ismaster calls for some time
+            # Monitors may continue working on hello calls for some time
             # after a call to Topology.close, so this method may be called at
             # any time. Ensure the topology is open before processing the
             # change.
@@ -422,7 +425,7 @@ class Topology(object):
 
     def handle_getlasterror(self, address, error_msg):
         """Clear our pool for a server, mark it Unknown, and check it soon."""
-        error = NotMasterError(error_msg, {'code': 10107, 'errmsg': error_msg})
+        error = NotPrimaryError(error_msg, {'code': 10107, 'errmsg': error_msg})
         with self._lock:
             server = self._servers.get(address)
             if server:
@@ -430,15 +433,27 @@ class Topology(object):
                     ServerDescription(address, error=error), True)
                 server.request_check()
 
+    def data_bearing_servers(self):
+        """Return a list of all data-bearing servers.
+
+        This includes any server that might be selected for an operation.
+        """
+        if self._description.topology_type == TOPOLOGY_TYPE.Single:
+            return self._description.known_servers
+        return self._description.readable_servers
+
     def update_pool(self, all_credentials):
         # Remove any stale sockets and add new sockets if pool is too small.
         servers = []
         with self._lock:
-            for server in self._servers.values():
-                servers.append((server, server._pool.generation))
+            # Only update pools for data-bearing servers.
+            for sd in self.data_bearing_servers():
+                server = self._servers[sd.address]
+                servers.append((server,
+                                server.pool.gen.get_overall()))
 
         for server, generation in servers:
-            server._pool.remove_stale_sockets(generation, all_credentials)
+            server.pool.remove_stale_sockets(generation, all_credentials)
 
     def close(self):
         """Clear pools and terminate monitors. Topology reopens on demand."""
@@ -474,39 +489,46 @@ class Topology(object):
         with self._lock:
             return self._session_pool.pop_all()
 
-    def get_server_session(self):
-        """Start or resume a server session, or raise ConfigurationError."""
-        with self._lock:
-            session_timeout = self._description.logical_session_timeout_minutes
-            if session_timeout is None:
-                # Maybe we need an initial scan? Can raise ServerSelectionError.
-                if self._description.topology_type == TOPOLOGY_TYPE.Single:
-                    if not self._description.has_known_servers:
-                        self._select_servers_loop(
-                            any_server_selector,
-                            self._settings.server_selection_timeout,
-                            None)
-                elif not self._description.readable_servers:
+    def _check_session_support(self):
+        """Internal check for session support on non-load balanced clusters."""
+        session_timeout = self._description.logical_session_timeout_minutes
+        if session_timeout is None:
+            # Maybe we need an initial scan? Can raise ServerSelectionError.
+            if self._description.topology_type == TOPOLOGY_TYPE.Single:
+                if not self._description.has_known_servers:
                     self._select_servers_loop(
-                        readable_server_selector,
+                        any_server_selector,
                         self._settings.server_selection_timeout,
                         None)
+            elif not self._description.readable_servers:
+                self._select_servers_loop(
+                    readable_server_selector,
+                    self._settings.server_selection_timeout,
+                    None)
 
             session_timeout = self._description.logical_session_timeout_minutes
             if session_timeout is None:
                 raise ConfigurationError(
                     "Sessions are not supported by this MongoDB deployment")
+        return session_timeout
 
+    def get_server_session(self):
+        """Start or resume a server session, or raise ConfigurationError."""
+        with self._lock:
+            # Sessions are always supported in load balanced mode.
+            if not self._settings.load_balanced:
+                session_timeout = self._check_session_support()
+            else:
+                # Sessions never time out in load balanced mode.
+                session_timeout = float('inf')
             return self._session_pool.get_server_session(session_timeout)
 
     def return_server_session(self, server_session, lock):
         if lock:
             with self._lock:
-                session_timeout = \
-                    self._description.logical_session_timeout_minutes
-                if session_timeout is not None:
-                    self._session_pool.return_server_session(server_session,
-                                                             session_timeout)
+                self._session_pool.return_server_session(
+                    server_session,
+                    self._description.logical_session_timeout_minutes)
         else:
             # Called from a __del__ method, can't use a lock.
             self._session_pool.return_server_session_no_lock(server_session)
@@ -536,6 +558,13 @@ class Topology(object):
                                       SRV_POLLING_TOPOLOGIES):
                 self._srv_monitor.open()
 
+            if self._settings.load_balanced:
+                # Emit initial SDAM events for load balancer mode.
+                self._process_change(ServerDescription(
+                    self._seed_addresses[0],
+                    IsMaster({'ok': 1, 'serviceId': self._topology_id,
+                              'maxWireVersion': 13})))
+
         # Ensure that the monitors are open.
         for server in itervalues(self._servers):
             server.open()
@@ -546,7 +575,8 @@ class Topology(object):
             # Another thread removed this server from the topology.
             return True
 
-        if err_ctx.sock_generation != server._pool.generation:
+        if server._pool.stale_generation(
+                err_ctx.sock_generation, err_ctx.service_id):
             # This is an outdated error from a previous pool version.
             return True
 
@@ -567,6 +597,7 @@ class Topology(object):
         server = self._servers[address]
         error = err_ctx.error
         exc_type = type(error)
+        service_id = err_ctx.service_id
         if (issubclass(exc_type, NetworkTimeout) and
                 err_ctx.completed_handshake):
             # The socket has been closed. Don't reset the server.
@@ -574,9 +605,12 @@ class Topology(object):
             # operation fails because of any network error besides a socket
             # timeout...."
             return
-        elif issubclass(exc_type, NotMasterError):
+        elif issubclass(exc_type, WriteError):
+            # Ignore writeErrors.
+            return
+        elif issubclass(exc_type, NotPrimaryError):
             # As per the SDAM spec if:
-            #   - the server sees a "not master" error, and
+            #   - the server sees a "not primary" error, and
             #   - the server is not shutting down, and
             #   - the server version is >= 4.2, then
             # we keep the existing connection pool, but mark the server type
@@ -586,28 +620,32 @@ class Topology(object):
             err_code = error.details.get('code', -1)
             is_shutting_down = err_code in helpers._SHUTDOWN_CODES
             # Mark server Unknown, clear the pool, and request check.
-            self._process_change(ServerDescription(address, error=error))
+            if not self._settings.load_balanced:
+                self._process_change(ServerDescription(address, error=error))
             if is_shutting_down or (err_ctx.max_wire_version <= 7):
                 # Clear the pool.
-                server.reset()
+                server.reset(service_id)
             server.request_check()
         elif issubclass(exc_type, ConnectionFailure):
             # "Client MUST replace the server's description with type Unknown
             # ... MUST NOT request an immediate check of the server."
-            self._process_change(ServerDescription(address, error=error))
+            if not self._settings.load_balanced:
+                self._process_change(ServerDescription(address, error=error))
             # Clear the pool.
-            server.reset()
+            server.reset(service_id)
             # "When a client marks a server Unknown from `Network error when
-            # reading or writing`_, clients MUST cancel the isMaster check on
+            # reading or writing`_, clients MUST cancel the hello check on
             # that server and close the current monitoring connection."
             server._monitor.cancel_check()
         elif issubclass(exc_type, OperationFailure):
             # Do not request an immediate check since the server is likely
             # shutting down.
             if error.code in helpers._NOT_MASTER_CODES:
-                self._process_change(ServerDescription(address, error=error))
+                if not self._settings.load_balanced:
+                    self._process_change(
+                        ServerDescription(address, error=error))
                 # Clear the pool.
-                server.reset()
+                server.reset(service_id)
 
     def handle_error(self, address, err_ctx):
         """Handle an application error.
@@ -680,7 +718,9 @@ class Topology(object):
             ssl_match_hostname=options.ssl_match_hostname,
             event_listeners=options.event_listeners,
             appname=options.appname,
-            driver=options.driver)
+            driver=options.driver,
+            server_api=options.server_api,
+        )
 
         return self._settings.pool_class(address, monitor_pool_options,
                                          handshake=False)
@@ -752,11 +792,12 @@ class Topology(object):
 class _ErrorContext(object):
     """An error with context for SDAM error handling."""
     def __init__(self, error, max_wire_version, sock_generation,
-                 completed_handshake):
+                 completed_handshake, service_id):
         self.error = error
         self.max_wire_version = max_wire_version
         self.sock_generation = sock_generation
         self.completed_handshake = completed_handshake
+        self.service_id = service_id
 
 
 def _is_stale_error_topology_version(current_tv, error_tv):

pymongo/topology_description.py

@@ -25,9 +25,9 @@ from pymongo.server_type import SERVER_TYPE
 
 
 # Enumeration for various kinds of MongoDB cluster topologies.
-TOPOLOGY_TYPE = namedtuple('TopologyType', ['Single', 'ReplicaSetNoPrimary',
-                                            'ReplicaSetWithPrimary', 'Sharded',
-                                            'Unknown'])(*range(5))
+TOPOLOGY_TYPE = namedtuple('TopologyType', [
+    'Single', 'ReplicaSetNoPrimary', 'ReplicaSetWithPrimary', 'Sharded',
+    'Unknown', 'LoadBalanced'])(*range(6))
 
 # Topologies compatible with SRV record polling.
 SRV_POLLING_TOPOLOGIES = (TOPOLOGY_TYPE.Unknown, TOPOLOGY_TYPE.Sharded)
@@ -63,7 +63,28 @@ class TopologyDescription(object):
 
         # Is PyMongo compatible with all servers' wire protocols?
         self._incompatible_err = None
+        if self._topology_type != TOPOLOGY_TYPE.LoadBalanced:
+            self._init_incompatible_err()
 
+        # Server Discovery And Monitoring Spec: Whenever a client updates the
+        # TopologyDescription from a hello response, it MUST set
+        # TopologyDescription.logicalSessionTimeoutMinutes to the smallest
+        # logicalSessionTimeoutMinutes value among ServerDescriptions of all
+        # data-bearing server types. If any have a null
+        # logicalSessionTimeoutMinutes, then
+        # TopologyDescription.logicalSessionTimeoutMinutes MUST be set to null.
+        readable_servers = self.readable_servers
+        if not readable_servers:
+            self._ls_timeout_minutes = None
+        elif any(s.logical_session_timeout_minutes is None
+                 for s in readable_servers):
+            self._ls_timeout_minutes = None
+        else:
+            self._ls_timeout_minutes = min(s.logical_session_timeout_minutes
+                                           for s in readable_servers)
+
+    def _init_incompatible_err(self):
+        """Internal compatibility check for non-load balanced topologies."""
         for s in self._server_descriptions.values():
             if not s.is_server_type_known:
                 continue
@@ -98,23 +119,6 @@ class TopologyDescription(object):
 
                 break
 
-        # Server Discovery And Monitoring Spec: Whenever a client updates the
-        # TopologyDescription from an ismaster response, it MUST set
-        # TopologyDescription.logicalSessionTimeoutMinutes to the smallest
-        # logicalSessionTimeoutMinutes value among ServerDescriptions of all
-        # data-bearing server types. If any have a null
-        # logicalSessionTimeoutMinutes, then
-        # TopologyDescription.logicalSessionTimeoutMinutes MUST be set to null.
-        readable_servers = self.readable_servers
-        if not readable_servers:
-            self._ls_timeout_minutes = None
-        elif any(s.logical_session_timeout_minutes is None
-                 for s in readable_servers):
-            self._ls_timeout_minutes = None
-        else:
-            self._ls_timeout_minutes = min(s.logical_session_timeout_minutes
-                                           for s in readable_servers)
-
     def check_compatible(self):
         """Raise ConfigurationError if any server is incompatible.
 
@@ -243,8 +247,9 @@ class TopologyDescription(object):
                                              selector.min_wire_version,
                                              common_wv))
 
-        if self.topology_type == TOPOLOGY_TYPE.Single:
-            # Ignore selectors for standalone.
+        if self.topology_type in (TOPOLOGY_TYPE.Single,
+                                  TOPOLOGY_TYPE.LoadBalanced):
+            # Ignore selectors for standalone and load balancer mode.
             return self.known_servers
         elif address:
             # Ignore selectors when explicit address is requested.
@@ -298,7 +303,7 @@ class TopologyDescription(object):
             self.topology_type_name, servers)
 
 
-# If topology type is Unknown and we receive an ismaster response, what should
+# If topology type is Unknown and we receive a hello response, what should
 # the new topology type be?
 _SERVER_TYPE_TO_TOPOLOGY_TYPE = {
     SERVER_TYPE.Mongos: TOPOLOGY_TYPE.Sharded,
@@ -306,6 +311,7 @@ _SERVER_TYPE_TO_TOPOLOGY_TYPE = {
     SERVER_TYPE.RSSecondary: TOPOLOGY_TYPE.ReplicaSetNoPrimary,
     SERVER_TYPE.RSArbiter: TOPOLOGY_TYPE.ReplicaSetNoPrimary,
     SERVER_TYPE.RSOther: TOPOLOGY_TYPE.ReplicaSetNoPrimary,
+    # Note: SERVER_TYPE.LoadBalancer and Unknown are intentionally left out.
 }
 
 
@@ -315,9 +321,9 @@ def updated_topology_description(topology_description, server_description):
     :Parameters:
       - `topology_description`: the current TopologyDescription
       - `server_description`: a new ServerDescription that resulted from
-        an ismaster call
+        a hello call
 
-    Called after attempting (successfully or not) to call ismaster on the
+    Called after attempting (successfully or not) to call hello on the
     server at server_description.address. Does not modify topology_description.
     """
     address = server_description.address
@@ -355,7 +361,7 @@ def updated_topology_description(topology_description, server_description):
             topology_description._topology_settings)
 
     if topology_type == TOPOLOGY_TYPE.Unknown:
-        if server_type == SERVER_TYPE.Standalone:
+        if server_type in (SERVER_TYPE.Standalone, SERVER_TYPE.LoadBalancer):
             if len(topology_description._topology_settings.seeds) == 1:
                 topology_type = TOPOLOGY_TYPE.Single
             else:
@@ -430,7 +436,7 @@ def _updated_topology_description_srv_polling(topology_description, seedlist):
     :Parameters:
       - `topology_description`: the current TopologyDescription
       - `seedlist`: a list of new seeds new ServerDescription that resulted from
-        an ismaster call
+        a hello call
     """
     # Create a copy of the server descriptions.
     sds = topology_description.server_descriptions()
@@ -464,7 +470,7 @@ def _update_rs_from_primary(
         server_description,
         max_set_version,
         max_election_id):
-    """Update topology description from a primary's ismaster response.
+    """Update topology description from a primary's hello response.
 
     Pass in a dict of ServerDescriptions, current replica set name, the
     ServerDescription we are processing, and the TopologyDescription's

pymongo/uri_parser.py

@@ -16,6 +16,7 @@
 """Tools to parse and validate a MongoDB URI."""
 import re
 import warnings
+import sys
 
 from bson.py3compat import string_type, PY3
 
@@ -122,7 +123,7 @@ def parse_host(entity, default_port=DEFAULT_PORT):
     # Normalize hostname to lowercase, since DNS is case-insensitive:
     # http://tools.ietf.org/html/rfc4343
     # This prevents useless rediscovery if "foo.com" is in the seed list but
-    # "FOO.com" is in the ismaster response.
+    # "FOO.com" is in the hello response.
     return host.lower(), port
 
 
@@ -370,7 +371,26 @@ def split_hosts(hosts, default_port=DEFAULT_PORT):
 _BAD_DB_CHARS = re.compile('[' + re.escape(r'/ "$') + ']')
 
 _ALLOWED_TXT_OPTS = frozenset(
-    ['authsource', 'authSource', 'replicaset', 'replicaSet'])
+    ['authsource', 'authSource', 'replicaset', 'replicaSet', 'loadbalanced',
+     'loadBalanced'])
+
+
+def _check_options(nodes, options):
+    # Ensure directConnection was not True if there are multiple seeds.
+    if len(nodes) > 1 and options.get('directconnection'):
+        raise ConfigurationError(
+            'Cannot specify multiple hosts with directConnection=true')
+
+    if options.get('loadbalanced'):
+        if len(nodes) > 1:
+            raise ConfigurationError(
+                'Cannot specify multiple hosts with loadBalanced=true')
+        if options.get('directconnection'):
+            raise ConfigurationError(
+                'Cannot specify directConnection=true with loadBalanced=true')
+        if options.get('replicaset'):
+            raise ConfigurationError(
+                'Cannot specify replicaSet with loadBalanced=true')
 
 
 def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False,
@@ -425,8 +445,12 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False,
         scheme_free = uri[SCHEME_LEN:]
     elif uri.startswith(SRV_SCHEME):
         if not _HAVE_DNSPYTHON:
-            raise ConfigurationError('The "dnspython" module must be '
-                                     'installed to use mongodb+srv:// URIs')
+            python_path = sys.executable or "python"
+            raise ConfigurationError(
+                'The "dnspython" module must be '
+                'installed to use mongodb+srv:// URIs. '
+                'To fix this error install pymongo with the srv extra:\n '
+                '%s -m pip install "pymongo[srv]"' % (python_path))
         is_srv = True
         scheme_free = uri[SRV_SCHEME_LEN:]
     else:
@@ -504,7 +528,8 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False,
                 dns_options, validate, warn, normalize)
             if set(parsed_dns_options) - _ALLOWED_TXT_OPTS:
                 raise ConfigurationError(
-                    "Only authSource and replicaSet are supported from DNS")
+                    "Only authSource, replicaSet, and loadBalanced are "
+                    "supported from DNS")
             for opt, val in parsed_dns_options.items():
                 if opt not in options:
                     options[opt] = val
@@ -512,9 +537,8 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False,
             options["ssl"] = True if validate else 'true'
     else:
         nodes = split_hosts(hosts, default_port=default_port)
-        if len(nodes) > 1 and options.get('directConnection'):
-            raise ConfigurationError(
-                "Cannot specify multiple hosts with directConnection=true")
+
+    _check_options(nodes, options)
 
     return {
         'nodelist': nodes,
@@ -534,4 +558,4 @@ if __name__ == '__main__':
         pprint.pprint(parse_uri(sys.argv[1]))
     except InvalidURI as exc:
         print(exc)
-    sys.exit(0)
+    sys.exit(0)

setup.py

@@ -24,11 +24,15 @@ except ImportError:
     use_setuptools()
     from setuptools import setup, __version__ as _setuptools_version
 
-from distutils.cmd import Command
-from distutils.command.build_ext import build_ext
-from distutils.errors import CCompilerError, DistutilsOptionError
-from distutils.errors import DistutilsPlatformError, DistutilsExecError
-from distutils.core import Extension
+
+if sys.version_info[:2] < (3, 10):
+    from distutils.cmd import Command
+    from distutils.command.build_ext import build_ext
+    from distutils.core import Extension
+else:
+    from setuptools import Command
+    from setuptools.command.build_ext import build_ext
+    from setuptools.extension import Extension
 
 _HAVE_SPHINX = True
 try:
@@ -39,7 +43,7 @@ except ImportError:
     except ImportError:
         _HAVE_SPHINX = False
 
-version = "3.11.1"
+version = "3.12.4.dev0"
 
 f = open("README.rst")
 try:
@@ -89,7 +93,7 @@ class test(Command):
         if self.test_suite is None and self.test_module is None:
             self.test_module = 'test'
         elif self.test_module is not None and self.test_suite is not None:
-            raise DistutilsOptionError(
+            raise Exception(
                 "You may specify a module or suite, but not both"
             )
 
@@ -223,15 +227,6 @@ class doc(Command):
                          "   %s/\n" % (mode, path))
 
 
-if sys.platform == 'win32':
-    # distutils.msvc9compiler can raise an IOError when failing to
-    # find the compiler
-    build_errors = (CCompilerError, DistutilsExecError,
-                    DistutilsPlatformError, IOError)
-else:
-    build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
-
-
 class custom_build_ext(build_ext):
     """Allow C extension building to fail.
 
@@ -284,7 +279,7 @@ https://pymongo.readthedocs.io/en/stable/installation.html#osx
     def run(self):
         try:
             build_ext.run(self)
-        except DistutilsPlatformError:
+        except Exception:
             e = sys.exc_info()[1]
             sys.stdout.write('%s\n' % str(e))
             warnings.warn(self.warning_message % ("Extension modules",
@@ -296,7 +291,7 @@ https://pymongo.readthedocs.io/en/stable/installation.html#osx
         name = ext.name
         try:
             build_ext.build_extension(self, ext)
-        except build_errors:
+        except Exception:
             e = sys.exc_info()[1]
             sys.stdout.write('%s\n' % str(e))
             warnings.warn(self.warning_message % ("The %s extension "
@@ -322,9 +317,21 @@ ext_modules = [Extension('bson._cbson',
 # in set_default_verify_paths we should really avoid.
 # service_identity 18.1.0 introduced support for IP addr matching.
 pyopenssl_reqs = ["pyopenssl>=17.2.0", "requests<3.0.0", "service_identity>=18.1.0"]
+# PyOpenSSL is incapable of loading system CA certs on Windows
+# and mostly incapable on macOS.
+# https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_default_verify_paths
+if sys.platform == 'win32':
+    # wincertstore appears dead and only claims support for
+    # Python versions <= 3.4.
+    if sys.version_info[:2] < (3, 5):
+        pyopenssl_reqs.append("wincertstore>=0.2")
+    else:
+        pyopenssl_reqs.append("certifi")
+elif sys.platform == "darwin":
+    pyopenssl_reqs.append("certifi")
 
 extras_require = {
-    'encryption': ['pymongocrypt<2.0.0'],
+    'encryption': ['pymongocrypt>=1.1.0,<2.0.0'],
     'ocsp': pyopenssl_reqs,
     'snappy': ['python-snappy'],
     'tls': [],
@@ -347,25 +354,17 @@ if sys.version_info[0] == 2:
         for req in pyopenssl_reqs:
             extras_require['tls'].append(
                 "%s ; python_full_version < '2.7.9'" % (req,))
-        if sys.platform == 'win32':
-            extras_require['tls'].append(
-                "wincertstore>=0.2 ; python_full_version < '2.7.9'")
-        else:
-            extras_require['tls'].append(
-                "certifi ; python_full_version < '2.7.9'")
     elif sys.version_info < (2, 7, 9):
         # For installing from source or egg files on Python versions
         # older than 2.7.9, or systems that have setuptools versions
         # older than 20.10.
         extras_require['tls'].extend(pyopenssl_reqs)
-        if sys.platform == 'win32':
-            extras_require['tls'].append("wincertstore>=0.2")
-        else:
-            extras_require['tls'].append("certifi")
-    extras_require.update({'srv': ["dnspython>=1.16.0,<1.17.0"]})
     extras_require.update({'tls': ["ipaddress"]})
+
+if sys.version_info[:2] < (3, 6):
+    extras_require.update({'srv': ["dnspython>=1.16.0,<1.17.0"]})
 else:
-    extras_require.update({'srv': ["dnspython>=1.16.0,<2.0.0"]})
+    extras_require.update({'srv': ["dnspython>=1.16.0,<3.0.0"]})
 
 # GSSAPI extras
 if sys.platform == 'win32':
@@ -421,6 +420,7 @@ setup(
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Database"],

test/__init__.py

@@ -21,6 +21,7 @@ import socket
 import sys
 import threading
 import time
+import traceback
 import unittest
 import warnings
 
@@ -47,7 +48,10 @@ import pymongo.errors
 from bson.son import SON
 from pymongo import common, message
 from pymongo.common import partition_node
-from pymongo.ssl_support import HAVE_SSL, validate_cert_reqs
+from pymongo.hello_compat import HelloCompat
+from pymongo.server_api import ServerApi
+from pymongo.ssl_support import HAVE_SSL, _ssl
+from pymongo.uri_parser import parse_uri
 from test.version import Version
 
 if HAVE_SSL:
@@ -89,6 +93,29 @@ if CA_PEM:
     TLS_OPTIONS['tlsCAFile'] = CA_PEM
 
 COMPRESSORS = os.environ.get("COMPRESSORS")
+MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION")
+TEST_LOADBALANCER = bool(os.environ.get("TEST_LOADBALANCER"))
+TEST_SERVERLESS = bool(os.environ.get("TEST_SERVERLESS"))
+SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI")
+MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI")
+if TEST_LOADBALANCER:
+    # Remove after PYTHON-2712
+    from pymongo import pool
+    pool._MOCK_SERVICE_ID = True
+    res = parse_uri(SINGLE_MONGOS_LB_URI)
+    host, port = res['nodelist'][0]
+    db_user = res['username'] or db_user
+    db_pwd = res['password'] or db_pwd
+elif TEST_SERVERLESS:
+    TEST_LOADBALANCER = True
+    res = parse_uri(SINGLE_MONGOS_LB_URI)
+    host, port = res['nodelist'][0]
+    db_user = res['username'] or db_user
+    db_pwd = res['password'] or db_pwd
+    TLS_OPTIONS = {'tls': True}
+    # Spec says serverless tests must be run with compression.
+    COMPRESSORS = COMPRESSORS or 'zlib'
+
 
 def is_server_resolvable():
     """Returns True if 'server' is resolvable."""
@@ -130,6 +157,8 @@ class client_knobs(object):
         self.old_min_heartbeat_interval = None
         self.old_kill_cursor_frequency = None
         self.old_events_queue_frequency = None
+        self._enabled = True
+        self._stack = None
 
     def enable(self):
         self.old_heartbeat_frequency = common.HEARTBEAT_FREQUENCY
@@ -148,6 +177,9 @@ class client_knobs(object):
 
         if self.events_queue_frequency is not None:
             common.EVENTS_QUEUE_FREQUENCY = self.events_queue_frequency
+        self._enabled = True
+        # Store the allocation traceback to catch non-disabled client_knobs.
+        self._stack = ''.join(traceback.format_stack())
 
     def __enter__(self):
         self.enable()
@@ -157,16 +189,32 @@ class client_knobs(object):
         common.MIN_HEARTBEAT_INTERVAL = self.old_min_heartbeat_interval
         common.KILL_CURSOR_FREQUENCY = self.old_kill_cursor_frequency
         common.EVENTS_QUEUE_FREQUENCY = self.old_events_queue_frequency
+        self._enabled = False
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.disable()
 
+    def __del__(self):
+        if self._enabled:
+            msg = (
+                'ERROR: client_knobs still enabled! HEARTBEAT_FREQUENCY=%s, '
+                'MIN_HEARTBEAT_INTERVAL=%s, KILL_CURSOR_FREQUENCY=%s, '
+                'EVENTS_QUEUE_FREQUENCY=%s, stack:\n%s' % (
+                    common.HEARTBEAT_FREQUENCY,
+                    common.MIN_HEARTBEAT_INTERVAL,
+                    common.KILL_CURSOR_FREQUENCY,
+                    common.EVENTS_QUEUE_FREQUENCY,
+                    self._stack))
+            self.disable()
+            raise Exception(msg)
+
 
 def _all_users(db):
     return set(u['user'] for u in db.command('usersInfo').get('users', []))
 
 
 class ClientContext(object):
+    MULTI_MONGOS_LB_URI = MULTI_MONGOS_LB_URI
 
     def __init__(self):
         """Create a client and grab essential information from the server."""
@@ -180,6 +228,7 @@ class ClientContext(object):
         self.version = Version(-1)  # Needs to be comparable with Version
         self.auth_enabled = False
         self.test_commands_enabled = False
+        self.server_parameters = {}
         self.is_mongos = False
         self.mongoses = []
         self.is_rs = False
@@ -191,13 +240,20 @@ class ClientContext(object):
         self.sessions_enabled = False
         self.client = None
         self.conn_lock = threading.Lock()
-
+        self.is_data_lake = False
+        self.load_balancer = TEST_LOADBALANCER
+        self.serverless = TEST_SERVERLESS
+        if self.load_balancer or self.serverless:
+            self.default_client_options["loadBalanced"] = True
         if COMPRESSORS:
             self.default_client_options["compressors"] = COMPRESSORS
+        if MONGODB_API_VERSION:
+            server_api = ServerApi(MONGODB_API_VERSION)
+            self.default_client_options["server_api"] = server_api
 
     @property
-    def ismaster(self):
-        return self.client.admin.command('isMaster')
+    def hello(self):
+        return self.client.admin.command(HelloCompat.LEGACY_CMD)
 
     def _connect(self, host, port, **kwargs):
         # Jython takes a long time to connect.
@@ -205,17 +261,16 @@ class ClientContext(object):
             timeout_ms = 10000
         else:
             timeout_ms = 5000
-        if COMPRESSORS:
-            kwargs["compressors"] = COMPRESSORS
+        kwargs.update(self.default_client_options)
         client = pymongo.MongoClient(
             host, port, serverSelectionTimeoutMS=timeout_ms, **kwargs)
         try:
             try:
-                client.admin.command('isMaster')  # Can we connect?
+                client.admin.command('ping')  # Can we connect?
             except pymongo.errors.OperationFailure as exc:
                 # SERVER-32063
                 self.connection_attempts.append(
-                    'connected client %r, but isMaster failed: %s' % (
+                    'connected client %r, but hello failed: %s' % (
                         client, exc))
             else:
                 self.connection_attempts.append(
@@ -231,6 +286,19 @@ class ClientContext(object):
 
     def _init_client(self):
         self.client = self._connect(host, port)
+
+        if self.client is not None:
+            # Return early when connected to dataLake as mongohoused does not
+            # support the getCmdLineOpts command and is tested without TLS.
+            build_info = self.client.admin.command('buildInfo')
+            if 'dataLake' in build_info:
+                self.is_data_lake = True
+                self.auth_enabled = True
+                self.client = self._connect(
+                    host, port, username=db_user, password=db_pwd)
+                self.connected = True
+                return
+
         if HAVE_SSL and not self.client:
             # Is MongoDB configured for SSL?
             self.client = self._connect(host, port, **TLS_OPTIONS)
@@ -242,22 +310,26 @@ class ClientContext(object):
         if self.client:
             self.connected = True
 
-            try:
-                self.cmd_line = self.client.admin.command('getCmdLineOpts')
-            except pymongo.errors.OperationFailure as e:
-                msg = e.details.get('errmsg', '')
-                if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
-                    # Unauthorized.
-                    self.auth_enabled = True
-                else:
-                    raise
+            if self.serverless:
+                self.auth_enabled = True
             else:
-                self.auth_enabled = self._server_started_with_auth()
+                try:
+                    self.cmd_line = self.client.admin.command('getCmdLineOpts')
+                except pymongo.errors.OperationFailure as e:
+                    msg = e.details.get('errmsg', '')
+                    if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
+                        # Unauthorized.
+                        self.auth_enabled = True
+                    else:
+                        raise
+                else:
+                    self.auth_enabled = self._server_started_with_auth()
 
             if self.auth_enabled:
-                # See if db_user already exists.
-                if not self._check_user_provided():
-                    _create_user(self.client.admin, db_user, db_pwd)
+                if not self.serverless:
+                    # See if db_user already exists.
+                    if not self._check_user_provided():
+                        _create_user(self.client.admin, db_user, db_pwd)
 
                 self.client = self._connect(
                     host, port, username=db_user, password=db_pwd,
@@ -267,16 +339,19 @@ class ClientContext(object):
                 # May not have this if OperationFailure was raised earlier.
                 self.cmd_line = self.client.admin.command('getCmdLineOpts')
 
-            self.server_status = self.client.admin.command('serverStatus')
-            if self.storage_engine == "mmapv1":
-                # MMAPv1 does not support retryWrites=True.
-                self.default_client_options['retryWrites'] = False
+            if self.serverless:
+                self.server_status = {}
+            else:
+                self.server_status = self.client.admin.command('serverStatus')
+                if self.storage_engine == "mmapv1":
+                    # MMAPv1 does not support retryWrites=True.
+                    self.default_client_options['retryWrites'] = False
 
-            ismaster = self.ismaster
-            self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
+            hello = self.hello
+            self.sessions_enabled = 'logicalSessionTimeoutMinutes' in hello
 
-            if 'setName' in ismaster:
-                self.replica_set_name = str(ismaster['setName'])
+            if 'setName' in hello:
+                self.replica_set_name = str(hello['setName'])
                 self.is_rs = True
                 if self.auth_enabled:
                     # It doesn't matter which member we use as the seed here.
@@ -294,44 +369,55 @@ class ClientContext(object):
                         replicaSet=self.replica_set_name,
                         **self.default_client_options)
 
-                # Get the authoritative ismaster result from the primary.
-                ismaster = self.ismaster
+                # Get the authoritative hello result from the primary.
+                hello = self.hello
                 nodes = [partition_node(node.lower())
-                         for node in ismaster.get('hosts', [])]
+                         for node in hello.get('hosts', [])]
                 nodes.extend([partition_node(node.lower())
-                              for node in ismaster.get('passives', [])])
+                              for node in hello.get('passives', [])])
                 nodes.extend([partition_node(node.lower())
-                              for node in ismaster.get('arbiters', [])])
+                              for node in hello.get('arbiters', [])])
                 self.nodes = set(nodes)
             else:
                 self.nodes = set([(host, port)])
-            self.w = len(ismaster.get("hosts", [])) or 1
+            self.w = len(hello.get("hosts", [])) or 1
             self.version = Version.from_client(self.client)
 
-            if 'enableTestCommands=1' in self.cmd_line['argv']:
+            if self.serverless:
+                self.server_parameters = {
+                    'requireApiVersion': False,
+                    'enableTestCommands': True,
+                }
                 self.test_commands_enabled = True
-            elif 'parsed' in self.cmd_line:
-                params = self.cmd_line['parsed'].get('setParameter', [])
-                if 'enableTestCommands=1' in params:
+                self.has_ipv6 = False
+            else:
+                self.server_parameters = self.client.admin.command(
+                    'getParameter', '*')
+                if 'enableTestCommands=1' in self.cmd_line['argv']:
                     self.test_commands_enabled = True
-                else:
-                    params = self.cmd_line['parsed'].get('setParameter', {})
-                    if params.get('enableTestCommands') == '1':
+                elif 'parsed' in self.cmd_line:
+                    params = self.cmd_line['parsed'].get('setParameter', [])
+                    if 'enableTestCommands=1' in params:
                         self.test_commands_enabled = True
+                    else:
+                        params = self.cmd_line['parsed'].get('setParameter', {})
+                        if params.get('enableTestCommands') == '1':
+                            self.test_commands_enabled = True
+                    self.has_ipv6 = self._server_started_with_ipv6()
 
-            self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
-            self.has_ipv6 = self._server_started_with_ipv6()
+            self.is_mongos = (self.hello.get('msg') == 'isdbgrid')
             if self.is_mongos:
-                # Check for another mongos on the next port.
                 address = self.client.address
-                next_address = address[0], address[1] + 1
                 self.mongoses.append(address)
-                mongos_client = self._connect(*next_address,
-                                              **self.default_client_options)
-                if mongos_client:
-                    ismaster = mongos_client.admin.command('ismaster')
-                    if ismaster.get('msg') == 'isdbgrid':
-                        self.mongoses.append(next_address)
+                if not self.serverless:
+                    # Check for another mongos on the next port.
+                    next_address = address[0], address[1] + 1
+                    mongos_client = self._connect(
+                        *next_address, **self.default_client_options)
+                    if mongos_client:
+                        hello = mongos_client.admin.command(HelloCompat.LEGACY_CMD)
+                        if hello.get('msg') == 'isdbgrid':
+                            self.mongoses.append(next_address)
 
     def init(self):
         with self.conn_lock:
@@ -466,6 +552,13 @@ class ClientContext(object):
             "Cannot connect to MongoDB on %s" % (self.pair,),
             func=func)
 
+    def require_data_lake(self, func):
+        """Run a test only if we are connected to Atlas Data Lake."""
+        return self._require(
+            lambda: self.is_data_lake,
+            "Not connected to Atlas Data Lake on %s" % (self.pair,),
+            func=func)
+
     def require_no_mmap(self, func):
         """Run a test only if the server is not using the MMAPv1 storage
         engine. Only works for standalone and replica sets; tests are
@@ -520,6 +613,24 @@ class ClientContext(object):
         return self._require(lambda: sec_count() >= count,
                              "Not enough secondaries available")
 
+    @property
+    def supports_secondary_read_pref(self):
+        if self.has_secondaries:
+            return True
+        if self.is_mongos:
+            shard = self.client.config.shards.find_one()['host']
+            num_members = shard.count(',') + 1
+            return num_members > 1
+        return False
+
+    def require_secondary_read_pref(self):
+        """Run a test only if the client is connected to a cluster that
+        supports secondary read preference
+        """
+        return self._require(lambda: self.supports_secondary_read_pref,
+                             "This cluster does not support secondary read "
+                             "preference")
+
     def require_no_replica_set(self, func):
         """Run a test if the client is *not* connected to a replica set."""
         return self._require(
@@ -564,6 +675,19 @@ class ClientContext(object):
                              "Must be connected to a replica set or mongos",
                              func=func)
 
+    def require_load_balancer(self, func):
+        """Run a test only if the client is connected to a load balancer."""
+        return self._require(lambda: self.load_balancer,
+                             "Must be connected to a load balancer",
+                             func=func)
+
+    def require_no_load_balancer(self, func):
+        """Run a test only if the client is not connected to a load balancer.
+        """
+        return self._require(lambda: not self.load_balancer,
+                             "Must not be connected to a load balancer",
+                             func=func)
+
     def check_auth_with_sharding(self, func):
         """Skip a test when connected to mongos < 2.0 and running with auth."""
         condition = lambda: not (self.auth_enabled and
@@ -573,12 +697,30 @@ class ClientContext(object):
                              func=func)
 
     def is_topology_type(self, topologies):
+        unknown = set(topologies) - {'single', 'replicaset', 'sharded',
+                                     'sharded-replicaset', 'load-balanced'}
+        if unknown:
+            raise AssertionError('Unknown topologies: %r' % (unknown,))
+        if self.load_balancer:
+            if 'load-balanced' in topologies:
+                return True
+            return False
         if 'single' in topologies and not (self.is_mongos or self.is_rs):
             return True
         if 'replicaset' in topologies and self.is_rs:
             return True
         if 'sharded' in topologies and self.is_mongos:
             return True
+        if 'sharded-replicaset' in topologies and self.is_mongos:
+            shards = list(client_context.client.config.shards.find())
+            for shard in shards:
+                # For a 3-member RS-backed sharded cluster, shard['host']
+                # will be 'replicaName/ip1:port1,ip2:port2,ip3:port3'
+                # Otherwise it will be 'ip1:port1'
+                host_spec = shard['host']
+                if not len(host_spec.split('/')) > 1:
+                    return False
+            return True
         return False
 
     def require_cluster_type(self, topologies=[]):
@@ -664,6 +806,12 @@ class ClientContext(object):
                              "Transactions are not supported",
                              func=func)
 
+    def require_no_api_version(self, func):
+        """Skip this test when testing with requireApiVersion."""
+        return self._require(lambda: not MONGODB_API_VERSION,
+                             "This test does not work with requireApiVersion",
+                             func=func)
+
     def mongos_seeds(self):
         return ','.join('%s:%s' % address for address in self.mongoses)
 
@@ -707,6 +855,9 @@ def sanitize_cmd(cmd):
     cp.pop('$db', None)
     cp.pop('$readPreference', None)
     cp.pop('lsid', None)
+    if MONGODB_API_VERSION:
+        # Versioned api parameters
+        cp.pop('apiVersion', None)
     # OP_MSG encoding may move the payload type one field to the
     # end of the command. Do the same here.
     name = next(iter(cp))
@@ -751,6 +902,12 @@ class IntegrationTest(PyMongoTestCase):
     @classmethod
     @client_context.require_connection
     def setUpClass(cls):
+        if (client_context.load_balancer and
+                not getattr(cls, 'RUN_ON_LOAD_BALANCER', False)):
+            raise SkipTest('this test does not support load balancers')
+        if (client_context.serverless and
+                not getattr(cls, 'RUN_ON_SERVERLESS', False)):
+            raise SkipTest('this test does not support serverless')
         cls.client = client_context.client
         cls.db = cls.client.pymongo_test
         if client_context.auth_enabled:
@@ -758,6 +915,10 @@ class IntegrationTest(PyMongoTestCase):
         else:
             cls.credentials = {}
 
+    def patch_system_certs(self, ca_certs):
+        patcher = SystemCertsPatcher(ca_certs)
+        self.addCleanup(patcher.disable)
+
 
 # Use assertRaisesRegex if available, otherwise use Python 2.7's
 # deprecated assertRaisesRegexp, with a 'p'.
@@ -774,6 +935,14 @@ class MockClientTest(unittest.TestCase):
     The class temporarily overrides HEARTBEAT_FREQUENCY to speed up tests.
     """
 
+    # MockClients tests that use replicaSet, directConnection=True, pass
+    # multiple seed addresses, or wait for heartbeat events are incompatible
+    # with loadBalanced=True.
+    @classmethod
+    @client_context.require_no_load_balancer
+    def setUpClass(cls):
+        pass
+
     def setUp(self):
         super(MockClientTest, self).setUp()
 
@@ -846,12 +1015,13 @@ def teardown():
         assert False, '\n'.join(garbage)
     c = client_context.client
     if c:
-        c.drop_database("pymongo-pooling-tests")
-        c.drop_database("pymongo_test")
-        c.drop_database("pymongo_test1")
-        c.drop_database("pymongo_test2")
-        c.drop_database("pymongo_test_mike")
-        c.drop_database("pymongo_test_bernie")
+        if not client_context.is_data_lake:
+            c.drop_database("pymongo-pooling-tests")
+            c.drop_database("pymongo_test")
+            c.drop_database("pymongo_test1")
+            c.drop_database("pymongo_test2")
+            c.drop_database("pymongo_test_mike")
+            c.drop_database("pymongo_test_bernie")
         c.close()
 
     # Jython does not support gc.get_objects.
@@ -894,3 +1064,26 @@ def clear_warning_registry():
     for name, module in list(sys.modules.items()):
         if hasattr(module, "__warningregistry__"):
             setattr(module, "__warningregistry__", {})
+
+
+class SystemCertsPatcher(object):
+    def __init__(self, ca_certs):
+        if sys.version_info < (2, 7, 9):
+            raise SkipTest("Can't load system CA certificates.")
+        if (ssl.OPENSSL_VERSION.lower().startswith('libressl') and
+                sys.platform == 'darwin' and not _ssl.IS_PYOPENSSL):
+            raise SkipTest(
+                "LibreSSL on OSX doesn't support setting CA certificates "
+                "using SSL_CERT_FILE environment variable.")
+        if sys.platform == 'win32' and _ssl.IS_PYOPENSSL:
+            raise SkipTest(
+                "SSL_CERT_FILE does not work on Windows with PyOpenSSL")
+        self.original_certs = os.environ.get('SSL_CERT_FILE')
+        # Tell OpenSSL where CA certificates live.
+        os.environ['SSL_CERT_FILE'] = ca_certs
+
+    def disable(self):
+        if self.original_certs is None:
+            os.environ.pop('SSL_CERT_FILE')
+        else:
+            os.environ['SSL_CERT_FILE'] = self.original_certs

test/atlas/test_connection.py

@@ -18,51 +18,111 @@ import os
 import sys
 import unittest
 
+from collections import defaultdict
+
 sys.path[0:0] = [""]
 
 import pymongo
 from pymongo.ssl_support import HAS_SNI
 
-
-_REPL = os.environ.get("ATLAS_REPL")
-_SHRD = os.environ.get("ATLAS_SHRD")
-_FREE = os.environ.get("ATLAS_FREE")
-_TLS11 = os.environ.get("ATLAS_TLS11")
-_TLS12 = os.environ.get("ATLAS_TLS12")
+try:
+    import dns
+    HAS_DNS = True
+except ImportError:
+    HAS_DNS = False
 
 
-def _connect(uri):
+URIS = {
+    "ATLAS_REPL": os.environ.get("ATLAS_REPL"),
+    "ATLAS_SHRD": os.environ.get("ATLAS_SHRD"),
+    "ATLAS_FREE": os.environ.get("ATLAS_FREE"),
+    "ATLAS_TLS11": os.environ.get("ATLAS_TLS11"),
+    "ATLAS_TLS12": os.environ.get("ATLAS_TLS12"),
+    "ATLAS_SERVERLESS": os.environ.get("ATLAS_SERVERLESS"),
+    "ATLAS_SRV_REPL": os.environ.get("ATLAS_SRV_REPL"),
+    "ATLAS_SRV_SHRD": os.environ.get("ATLAS_SRV_SHRD"),
+    "ATLAS_SRV_FREE": os.environ.get("ATLAS_SRV_FREE"),
+    "ATLAS_SRV_TLS11": os.environ.get("ATLAS_SRV_TLS11"),
+    "ATLAS_SRV_TLS12": os.environ.get("ATLAS_SRV_TLS12"),
+    "ATLAS_SRV_SERVERLESS": os.environ.get("ATLAS_SRV_SERVERLESS"),
+}
+
+# Set this variable to true to run the SRV tests even when dnspython is not
+# installed.
+MUST_TEST_SRV = os.environ.get("MUST_TEST_SRV")
+
+
+def connect(uri):
+    if not uri:
+        raise Exception("Must set env variable to test.")
     client = pymongo.MongoClient(uri)
     # No TLS error
-    client.admin.command('ismaster')
+    client.admin.command('ping')
     # No auth error
     client.test.test.count_documents({})
 
 
 class TestAtlasConnect(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        if not all([_REPL, _SHRD, _FREE]):
-            raise Exception(
-                "Must set ATLAS_REPL/SHRD/FREE env variables to test.")
+    @unittest.skipUnless(HAS_SNI, 'Free tier requires SNI support')
+    def test_free_tier(self):
+        connect(URIS['ATLAS_FREE'])
 
     def test_replica_set(self):
-        _connect(_REPL)
+        connect(URIS['ATLAS_REPL'])
 
     def test_sharded_cluster(self):
-        _connect(_SHRD)
-
-    def test_free_tier(self):
-        if not HAS_SNI:
-            raise unittest.SkipTest("Free tier requires SNI support.")
-        _connect(_FREE)
+        connect(URIS['ATLAS_SHRD'])
 
     def test_tls_11(self):
-        _connect(_TLS11)
+        connect(URIS['ATLAS_TLS11'])
 
     def test_tls_12(self):
-        _connect(_TLS12)
+        connect(URIS['ATLAS_TLS12'])
+
+    @unittest.skipIf(sys.platform.startswith('java'),
+                     'Jython does not support serverless TLS')
+    def test_serverless(self):
+        connect(URIS['ATLAS_SERVERLESS'])
+
+    def connect_srv(self, uri):
+        connect(uri)
+        self.assertIn('mongodb+srv://', uri)
+
+    @unittest.skipUnless(HAS_SNI, 'Free tier requires SNI support')
+    @unittest.skipUnless(HAS_DNS or MUST_TEST_SRV, 'SRV requires dnspython')
+    def test_srv_free_tier(self):
+        self.connect_srv(URIS['ATLAS_SRV_FREE'])
+
+    @unittest.skipUnless(HAS_DNS or MUST_TEST_SRV, 'SRV requires dnspython')
+    def test_srv_replica_set(self):
+        self.connect_srv(URIS['ATLAS_SRV_REPL'])
+
+    @unittest.skipUnless(HAS_DNS or MUST_TEST_SRV, 'SRV requires dnspython')
+    def test_srv_sharded_cluster(self):
+        self.connect_srv(URIS['ATLAS_SRV_SHRD'])
+
+    @unittest.skipUnless(HAS_DNS or MUST_TEST_SRV, 'SRV requires dnspython')
+    def test_srv_tls_11(self):
+        self.connect_srv(URIS['ATLAS_SRV_TLS11'])
+
+    @unittest.skipUnless(HAS_DNS or MUST_TEST_SRV, 'SRV requires dnspython')
+    def test_srv_tls_12(self):
+        self.connect_srv(URIS['ATLAS_SRV_TLS12'])
+
+    @unittest.skipUnless(HAS_DNS or MUST_TEST_SRV, 'SRV requires dnspython')
+    def test_srv_serverless(self):
+        self.connect_srv(URIS['ATLAS_SRV_SERVERLESS'])
+
+    def test_uniqueness(self):
+        """Ensure that we don't accidentally duplicate the test URIs."""
+        uri_to_names = defaultdict(list)
+        for name, uri in URIS.items():
+            if uri:
+                uri_to_names[uri].append(name)
+        duplicates = [names for names in uri_to_names.values()
+                      if len(names) > 1]
+        self.assertFalse(duplicates, 'Error: the following env variables have '
+                                     'duplicate values: %s' % (duplicates,))
 
 
 if __name__ == '__main__':

test/bson_corpus/document.json

@@ -17,6 +17,26 @@
             "description": "Single-character key subdoc",
             "canonical_bson": "160000000378000E0000000261000200000062000000",
             "canonical_extjson": "{\"x\" : {\"a\" : \"b\"}}"
+        },
+        {
+            "description": "Dollar-prefixed key in sub-document",
+            "canonical_bson": "170000000378000F000000022461000200000062000000",
+            "canonical_extjson": "{\"x\" : {\"$a\" : \"b\"}}"
+        },
+        {
+            "description": "Dollar as key in sub-document",
+            "canonical_bson": "160000000378000E0000000224000200000061000000",
+            "canonical_extjson": "{\"x\" : {\"$\" : \"a\"}}"
+        },
+        {
+            "description": "Dotted key in sub-document",
+            "canonical_bson": "180000000378001000000002612E62000200000063000000",
+            "canonical_extjson": "{\"x\" : {\"a.b\" : \"c\"}}"
+        },
+        {
+            "description": "Dot as key in sub-document",
+            "canonical_bson": "160000000378000E000000022E000200000061000000",
+            "canonical_extjson": "{\"x\" : {\".\" : \"a\"}}"
         }
     ],
     "decodeErrors": [

test/bson_corpus/top.json

@@ -3,9 +3,24 @@
     "bson_type": "0x00",
     "valid": [
         {
-            "description": "Document with keys that start with $",
+            "description": "Dollar-prefixed key in top-level document",
             "canonical_bson": "0F00000010246B6579002A00000000",
             "canonical_extjson": "{\"$key\": {\"$numberInt\": \"42\"}}"
+        },
+        {
+            "description": "Dollar as key in top-level document",
+            "canonical_bson": "0E00000002240002000000610000",
+            "canonical_extjson": "{\"$\": \"a\"}"
+        },
+        {
+            "description": "Dotted key in top-level document",
+            "canonical_bson": "1000000002612E620002000000630000",
+            "canonical_extjson": "{\"a.b\": \"c\"}"
+        },
+        {
+            "description": "Dot as key in top-level document",
+            "canonical_bson": "0E000000022E0002000000610000",
+            "canonical_extjson": "{\".\": \"a\"}"
         }
     ],
     "decodeErrors": [
@@ -199,14 +214,6 @@
             "description": "Bad $date (extra field)",
             "string": "{\"a\" : {\"$date\" : {\"$numberLong\" : \"1356351330501\"}, \"unrelated\": true}}"
         },
-        {
-            "description": "Bad DBRef (ref is number, not string)",
-            "string": "{\"x\" : {\"$ref\" : 42, \"$id\" : \"abc\"}}"
-        },
-        {
-            "description": "Bad DBRef (db is number, not string)",
-            "string": "{\"x\" : {\"$ref\" : \"a\", \"$id\" : \"abc\", \"$db\" : 42}}"
-        },
         {
             "description": "Bad $minKey (boolean, not integer)",
             "string": "{\"a\" : {\"$minKey\" : true}}"