PYTHON-2816 Generate pip < 20.3 compatible manylinux wheels (#679)

Split old/new manylinux wheel generation into two tasks.

(cherry picked from commit a949142480)

.evergreen/build-manylinux-internal.sh

@@ -19,9 +19,9 @@ for PYTHON in /opt/python/*/bin/python; do
     $PYTHON setup.py bdist_wheel
     rm -rf build
 
-    # Audit wheels and write multilinux tag
+    # Audit wheels and write manylinux tag
     for whl in dist/*.whl; do
-        # Skip already built manylinux1 wheels.
+        # Skip already built manylinux wheels.
         if [[ "$whl" != *"manylinux"* ]]; then
             auditwheel repair $whl -w dist
             rm $whl

.evergreen/build-manylinux.sh

@@ -2,16 +2,32 @@
 
 docker version
 
-# 2020-03-20-2fda31c Was the last release to include Python 3.4.
+# manylinux1 2021-05-05-b64d921 and manylinux2014 2021-05-05-1ac6ef3 were
-images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \
+# the last releases to generate pip < 20.3 compatible wheels. After that
-        quay.io/pypa/manylinux1_i686:2020-03-20-2fda31c \
+# auditwheel was upgraded to v4 which produces PEP 600 manylinux_x_y wheels
-        quay.io/pypa/manylinux1_x86_64 \
+# which requires pip >= 20.3. We use the older docker image to support older
-        quay.io/pypa/manylinux1_i686 \
+# pip versions.
-        quay.io/pypa/manylinux2014_x86_64 \
+BUILD_WITH_TAG="$1"
-        quay.io/pypa/manylinux2014_i686 \
+if [ -n "$BUILD_WITH_TAG" ]; then
-        quay.io/pypa/manylinux2014_aarch64 \
+  # 2020-03-20-2fda31c Was the last release to include Python 3.4.
-        quay.io/pypa/manylinux2014_ppc64le \
+  images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \
-        quay.io/pypa/manylinux2014_s390x)
+          quay.io/pypa/manylinux1_i686:2020-03-20-2fda31c \
+          quay.io/pypa/manylinux1_x86_64:2021-05-05-b64d921 \
+          quay.io/pypa/manylinux1_i686:2021-05-05-b64d921 \
+          quay.io/pypa/manylinux2014_x86_64:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_i686:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_aarch64:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_ppc64le:2021-05-05-1ac6ef3 \
+          quay.io/pypa/manylinux2014_s390x:2021-05-05-1ac6ef3)
+else
+  images=(quay.io/pypa/manylinux1_x86_64 \
+          quay.io/pypa/manylinux1_i686 \
+          quay.io/pypa/manylinux2014_x86_64 \
+          quay.io/pypa/manylinux2014_i686 \
+          quay.io/pypa/manylinux2014_aarch64 \
+          quay.io/pypa/manylinux2014_ppc64le \
+          quay.io/pypa/manylinux2014_s390x)
+fi
 
 for image in "${images[@]}"; do
   docker pull $image

.evergreen/config.yml

@@ -463,6 +463,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           mongo aws_e2e_regular_aws.js
     - command: shell.exec
       type: test
@@ -471,7 +472,7 @@ functions:
         silent: true
         script: |
           cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            alias urlencode='python -c "import sys, urllib as ul; sys.stdout.write(ul.quote_plus(sys.argv[1]))"'
+            alias urlencode='${python3_binary} -c "import sys, urllib.parse as ulp; sys.stdout.write(ulp.quote_plus(sys.argv[1]))"'
             USER=$(urlencode ${iam_auth_ecs_account})
             PASS=$(urlencode ${iam_auth_ecs_secret_access_key})
             MONGODB_URI="mongodb://$USER:$PASS@localhost"
@@ -491,15 +492,8 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          # The aws_e2e_assume_role script requires python3 with boto3.
-          virtualenv -p ${python3_binary} mongovenv
-          if [ "Windows_NT" = "$OS" ]; then
-            . mongovenv/Scripts/activate
-          else
-            . mongovenv/bin/activate
-          fi
-          pip install boto3
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           mongo aws_e2e_assume_role.js
     - command: shell.exec
       type: test
@@ -509,8 +503,8 @@ functions:
         script: |
           # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
           cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            alias urlencode='python -c "import sys, urllib as ul; sys.stdout.write(ul.quote_plus(sys.argv[1]))"'
+            alias urlencode='${python3_binary} -c "import sys, urllib.parse as ulp; sys.stdout.write(ulp.quote_plus(sys.argv[1]))"'
-            alias jsonkey='python -c "import json,sys;sys.stdout.write(json.load(sys.stdin)[sys.argv[1]])" < ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json'
+            alias jsonkey='${python3_binary} -c "import json,sys;sys.stdout.write(json.load(sys.stdin)[sys.argv[1]])" < ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json'
             USER=$(jsonkey AccessKeyId)
             USER=$(urlencode $USER)
             PASS=$(jsonkey SecretAccessKey)
@@ -538,13 +532,8 @@ functions:
              echo "This platform does not support the EC2 auth test, skipping..."
              exit 0
           fi
-          # The mongovenv was created earlier in "run aws auth test with assume role credentials".
-          if [ "Windows_NT" = "$OS" ]; then
-            . mongovenv/Scripts/activate
-          else
-            . mongovenv/bin/activate
-          fi
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           mongo aws_e2e_ec2.js
     - command: shell.exec
       type: test
@@ -583,7 +572,7 @@ functions:
         script: |
           # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
           cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            alias jsonkey='python -c "import json,sys;sys.stdout.write(json.load(sys.stdin)[sys.argv[1]])" < ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json'
+            alias jsonkey='${python3_binary} -c "import json,sys;sys.stdout.write(json.load(sys.stdin)[sys.argv[1]])" < ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json'
             export AWS_ACCESS_KEY_ID=$(jsonkey AccessKeyId)
             export AWS_SECRET_ACCESS_KEY=$(jsonkey SecretAccessKey)
             export AWS_SESSION_TOKEN=$(jsonkey SessionToken)
@@ -608,6 +597,7 @@ functions:
              exit 0
           fi
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           cat <<EOF > setup.js
           const mongo_binaries = "$MONGODB_BINARIES";
           const project_dir = "$PROJECT_DIRECTORY";
@@ -768,6 +758,24 @@ functions:
           # Remove all Docker images
           docker rmi -f $(docker images -a -q) &> /dev/null || true
 
+  "upload release":
+    - command: archive.targz_pack
+      params:
+        target: "release-files.tgz"
+        source_dir: "src/dist"
+        include:
+          - "*"
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: release-files.tgz
+        remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/release/${task_id}-${execution}-release-files.tar.gz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/gzip}
+        display_name: Release files
+
 pre:
   - func: "fetch source"
   - func: "prepare resources"
@@ -819,11 +827,9 @@ tasks:
                genhtml --version || true
                valgrind --version || true
 
-
     - name: "release"
       tags: ["release"]
       exec_timeout_secs: 216000  # 60 minutes (manylinux task is slow).
-      git_tag_only: true
       commands:
         - command: shell.exec
           type: test
@@ -833,22 +839,21 @@ tasks:
               set -o xtrace
               ${PREPARE_SHELL}
               .evergreen/release.sh
-        - command: archive.targz_pack
+        - func: "upload release"
+
+    - name: "release-old-manylinux"
+      tags: ["release"]
+      exec_timeout_secs: 216000  # 60 minutes (manylinux task is slow).
+      commands:
+        - command: shell.exec
+          type: test
           params:
-            target: "release-files.tgz"
+            working_dir: "src"
-            source_dir: "src/dist"
+            script: |
-            include:
+              set -o xtrace
-              - "*"
+              ${PREPARE_SHELL}
-        - command: s3.put
+              .evergreen/build-manylinux.sh BUILD_WITH_TAG
-          params:
+        - func: "upload release"
-            aws_key: ${aws_key}
-            aws_secret: ${aws_secret}
-            local_file: release-files.tgz
-            remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/release/${task_id}-${execution}-release-files.tar.gz
-            bucket: mciuploads
-            permissions: public-read
-            content_type: ${content_type|application/gzip}
-            display_name: Release files
 
 # Standard test tasks {{{
 
@@ -1561,6 +1566,7 @@ axes:
         run_on: debian92-test
         batchtime: 10080  # 7 days
         variables:
+          python3_binary: "/opt/python/3.8/bin/python3"
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian92/master/latest/libmongocrypt.tar.gz
       - id: macos-1014
         display_name: "macOS 10.14"
@@ -1568,7 +1574,7 @@ axes:
         variables:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
-          python3_binary: python3
+          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: rhel62
         display_name: "RHEL 6.2 (x86_64)"
@@ -2160,14 +2166,14 @@ buildvariants:
 - matrix_name: "tests-pyopenssl"
   matrix_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3.5"]
+    python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9"]
     auth: "*"
     ssl: "ssl"
     pyopenssl: "*"
   # Only test "noauth" with Python 3.7.
   exclude_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.4", "3.5", "3.6", "3.8", "3.9", "pypy", "pypy3.5"]
+    python-version: ["2.7", "3.4", "3.5", "3.6", "3.8", "3.9"]
     auth: "noauth"
     ssl: "ssl"
     pyopenssl: "*"
@@ -2177,6 +2183,19 @@ buildvariants:
     # Test standalone and sharded only on 4.4.
     - '.4.4'
 
+- matrix_name: "tests-pyopenssl-pypy"
+  matrix_spec:
+    platform: debian92
+    python-version: ["pypy", "pypy3.5", "pypy3.6"]
+    auth: "auth"
+    ssl: "ssl"
+    pyopenssl: "*"
+  display_name: "PyOpenSSL ${platform} ${python-version} ${auth}"
+  tasks:
+    - '.replica_set !.2.6 !.3.0 !.3.2 !.3.4'
+    # Test standalone and sharded only on 4.4.
+    - '.4.4'
+
 - matrix_name: "test-pyopenssl-old-py27"
   matrix_spec:
     platform:
@@ -2214,7 +2233,7 @@ buildvariants:
   matrix_spec:
     platform: rhel62
     # RHEL 6.2 does not support Python 3.7.x and later.
-    python-version: ["2.7", "3.4", "3.5", "3.6", "pypy", "pypy3.5", "pypy3.6"]
+    python-version: ["2.7", "3.4", "3.5", "3.6"]
     auth-ssl: noauth-nossl
 # TODO: dependency error for 'coverage-report' task:
 # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config
@@ -2223,6 +2242,15 @@ buildvariants:
   display_name: "Encryption ${python-version} ${platform} ${auth-ssl}"
   tasks: *encryption-server-versions
 
+- matrix_name: "tests-pypy-debian-test-encryption"
+  matrix_spec:
+    platform: debian92
+    python-version: ["pypy", "pypy3.5", "pypy3.6"]
+    auth-ssl: noauth-nossl
+    encryption: "*"
+  display_name: "Encryption ${python-version} ${platform} ${auth-ssl}"
+  tasks: *encryption-server-versions
+
 - matrix_name: "tests-python-version-rhel62-without-c-extensions"
   matrix_spec:
     platform: rhel62
@@ -2365,12 +2393,6 @@ buildvariants:
     python-version-windows: "*"
     auth-ssl: "*"
     encryption: "*"
-  exclude_spec:
-    # PYTHON-2366 Skip 3.9 due to cryptography install failures
-    - platform: "*"
-      python-version-windows: ["3.9"]
-      auth-ssl: "*"
-      encryption: "*"
   display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl}"
   tasks: *encryption-server-versions
 
@@ -2533,7 +2555,19 @@ buildvariants:
 - matrix_name: "ocsp-test"
   matrix_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.4", "3.8", "3.9", "pypy", "pypy3.5"]
+    python-version: ["2.7", "3.4", "3.8", "3.9"]
+    mongodb-version: ["4.4", "latest"]
+    auth: "noauth"
+    ssl: "ssl"
+  display_name: "OCSP test ${platform} ${python-version} ${mongodb-version}"
+  batchtime: 20160 # 14 days
+  tasks:
+    - name: ".ocsp"
+
+- matrix_name: "ocsp-test-pypy"
+  matrix_spec:
+    platform: debian92
+    python-version: ["pypy", "pypy3.5", "pypy3.6"]
     mongodb-version: ["4.4", "latest"]
     auth: "noauth"
     ssl: "ssl"
@@ -2545,7 +2579,7 @@ buildvariants:
 - matrix_name: "ocsp-test-windows"
   matrix_spec:
     platform: windows-64-vsMulti-small
-    python-version-windows: ["2.7", "3.4", "3.8"]
+    python-version-windows: ["2.7", "3.4", "3.9"]
     mongodb-version: ["4.4", "latest"]
     auth: "noauth"
     ssl: "ssl"
@@ -2589,8 +2623,15 @@ buildvariants:
   matrix_spec:
     platform: [ubuntu-20.04, windows-64-vsMulti-small, macos-1014]
   display_name: "Release ${platform}"
+  batchtime: 20160 # 14 days
   tasks:
     - name: "release"
+  rules:
+    - if:
+        platform: ubuntu-20.04
+      then:
+        add_tasks:
+          - name: "release-old-manylinux"
 
       # Platform notes
       # i386 builds of OpenSSL or Cyrus SASL are not available

.evergreen/run-mongodb-aws-test.sh

@@ -39,7 +39,12 @@ fi
 # show test output
 set -x
 
-VIRTUALENV=$(command -v virtualenv)
+# Workaround macOS python 3.9 incompatibility with system virtualenv.
+if [ $(uname -s) = "Darwin" ]; then
+    VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.9/bin/python3 -m virtualenv"
+else
+    VIRTUALENV=$(command -v virtualenv)
+fi
 
 authtest () {
     if [ "Windows_NT" = "$OS" ]; then

.evergreen/run-ocsp-tests.sh

@@ -3,6 +3,9 @@
 set -o xtrace
 set -o errexit
 
+# For createvirtualenv.
+. .evergreen/utils.sh
+
 if [ -z "$PYTHON_BINARY" ]; then
     echo "No python binary specified"
     PYTHON=$(command -v python || command -v python3) || true
@@ -14,36 +17,9 @@ else
     PYTHON="$PYTHON_BINARY"
 fi
 
-if $PYTHON -m virtualenv --version; then
+createvirtualenv $PYTHON ocsptest
-    VIRTUALENV="$PYTHON -m virtualenv"
-elif command -v virtualenv; then
-    # We can remove this fallback after:
-    # https://github.com/10gen/mongo-python-toolchain/issues/8
-    VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-else
-    echo "Cannot test without virtualenv"
-    exit 1
-fi
-
-$VIRTUALENV --never-download --no-wheel ocsptest
-if [ "Windows_NT" = "$OS" ]; then
-  . ocsptest/Scripts/activate
-else
-  . ocsptest/bin/activate
-fi
 trap "deactivate; rm -rf ocsptest" EXIT HUP
 
-IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
+python -m pip install --prefer-binary pyopenssl requests service_identity
-if [ $IS_PYTHON_2 = "1" ]; then
-    echo "Using a Python 2"
-    # Upgrade pip to install the cryptography wheel and not the tar.
-    # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-    python -m pip install --upgrade 'pip<20.1'
-    # Upgrade setuptools because cryptography requires 18.5+.
-    # <45 because 45.0 dropped support for 2.7.
-    python -m pip install --upgrade 'setuptools<45'
-fi
-
-python -m pip install pyopenssl requests service_identity
 
 OCSP_TLS_SHOULD_SUCCEED=${OCSP_TLS_SHOULD_SUCCEED} CA_FILE=${CA_FILE} python test/ocsp/test_ocsp.py

.evergreen/run-tests.sh

@@ -94,38 +94,11 @@ fi
 
 # PyOpenSSL test setup.
 if [ -n "$TEST_PYOPENSSL" ]; then
-    if $PYTHON -m virtualenv --version; then
+    createvirtualenv $PYTHON pyopenssltest
-        VIRTUALENV="$PYTHON -m virtualenv"
-    elif command -v virtualenv; then
-        # We can remove this fallback after:
-        # https://github.com/10gen/mongo-python-toolchain/issues/8
-        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-    else
-        echo "Cannot test without virtualenv"
-        exit 1
-    fi
-
-    $VIRTUALENV pyopenssltest
-    if [ "Windows_NT" = "$OS" ]; then
-        . pyopenssltest/Scripts/activate
-    else
-        . pyopenssltest/bin/activate
-    fi
     trap "deactivate; rm -rf pyopenssltest" EXIT HUP
     PYTHON=python
 
-    IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
+    python -m pip install --prefer-binary pyopenssl requests service_identity
-    if [ $IS_PYTHON_2 = "1" ]; then
-        echo "Using a Python 2"
-        # Upgrade pip to install the cryptography wheel and not the tar.
-        # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-        python -m pip install --upgrade 'pip<20.1'
-        # Upgrade setuptools because cryptography requires 18.5+.
-        # <45 because 45.0 dropped support for 2.7.
-        python -m pip install --upgrade 'setuptools<45'
-    fi
-
-    python -m pip install pyopenssl requests service_identity
 fi
 
 if [ -n "$TEST_ENCRYPTION" ]; then
@@ -166,7 +139,8 @@ if [ -n "$TEST_ENCRYPTION" ]; then
 
     # TODO: Test with 'pip install pymongocrypt'
     git clone --branch master https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
-    python -m pip install --upgrade ./libmongocrypt_git/bindings/python
+    python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt
+    python -m pip install ./libmongocrypt_git/bindings/python
     python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
     python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
     # PATH is updated by PREPARE_SHELL for access to mongocryptd.

.evergreen/test-encryption-requirements.txt

@@ -0,0 +1,4 @@
+# cffi==1.14.3 was the last installable release on RHEL 6.2 with Python 3.4
+cffi==1.14.3;python_version=="3.4"
+cffi>=1.12.0,<2;python_version!="3.4"
+cryptography>=2,<4

.evergreen/utils.sh

@@ -8,19 +8,34 @@ createvirtualenv () {
     PYTHON=$1
     VENVPATH=$2
     if $PYTHON -m virtualenv --version; then
-        VIRTUALENV="$PYTHON -m virtualenv"
+        VIRTUALENV="$PYTHON -m virtualenv --never-download"
+    elif $PYTHON -m venv -h>/dev/null; then
+        VIRTUALENV="$PYTHON -m venv"
     elif command -v virtualenv; then
-        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
+        VIRTUALENV="$(command -v virtualenv) -p $PYTHON --never-download"
     else
         echo "Cannot test without virtualenv"
         exit 1
     fi
-    $VIRTUALENV --system-site-packages --never-download $VENVPATH
+    $VIRTUALENV $VENVPATH
     if [ "Windows_NT" = "$OS" ]; then
         . $VENVPATH/Scripts/activate
     else
         . $VENVPATH/bin/activate
     fi
+    # Upgrade to the latest versions of pip setuptools wheel so that
+    # pip can always download the latest cryptography+cffi wheels.
+    PYTHON_VERSION=$(python -c 'import sys;print("%s.%s" % sys.version_info[:2])')
+    if [[ $PYTHON_VERSION == "3.4" ]]; then
+        # pip 19.2 dropped support for Python 3.4.
+        python -m pip install --upgrade 'pip<19.2'
+    elif [[ $PYTHON_VERSION == "2.7" || $PYTHON_VERSION == "3.5" ]]; then
+        # pip 21 will drop support for Python 2.7 and 3.5.
+        python -m pip install --upgrade 'pip<21'
+    else
+        python -m pip install --upgrade pip
+    fi
+    python -m pip install --upgrade setuptools wheel
 }
 
 # Usage:

bson/_cbsonmodule.c

@@ -2621,7 +2621,7 @@ static int _element_to_dict(PyObject* self, const char* string,
     if (name_length > BSON_MAX_SIZE || position + name_length >= max) {
         PyObject* InvalidBSON = _error("InvalidBSON");
         if (InvalidBSON) {
-            PyErr_SetNone(InvalidBSON);
+            PyErr_SetString(InvalidBSON, "field name too large");
             Py_DECREF(InvalidBSON);
         }
         return -1;

bson/codec_options.py

@@ -295,6 +295,17 @@ class CodecOptions(_options_base):
                  self.unicode_decode_error_handler, self.tzinfo,
                  self.type_registry))
 
+    def _options_dict(self):
+        """Dictionary of the arguments used to create this object."""
+        # TODO: PYTHON-2442 use _asdict() instead
+        return {
+            'document_class': self.document_class,
+            'tz_aware': self.tz_aware,
+            'uuid_representation': self.uuid_representation,
+            'unicode_decode_error_handler': self.unicode_decode_error_handler,
+            'tzinfo': self.tzinfo,
+            'type_registry': self.type_registry}
+
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self._arguments_repr())
 
@@ -310,7 +321,7 @@ class CodecOptions(_options_base):
 
         .. versionadded:: 3.5
         """
-        opts = self._asdict()
+        opts = self._options_dict()
         opts.update(kwargs)
         return CodecOptions(**opts)
 

bson/json_util.py

@@ -311,6 +311,16 @@ class JSONOptions(CodecOptions):
                     self.json_mode,
                     super(JSONOptions, self)._arguments_repr()))
 
+    def _options_dict(self):
+        # TODO: PYTHON-2442 use _asdict() instead
+        options_dict = super(JSONOptions, self)._options_dict()
+        options_dict.update({
+            'strict_number_long': self.strict_number_long,
+            'datetime_representation': self.datetime_representation,
+            'strict_uuid': self.strict_uuid,
+            'json_mode': self.json_mode})
+        return options_dict
+
     def with_options(self, **kwargs):
         """
         Make a copy of this JSONOptions, overriding some options::
@@ -324,7 +334,7 @@ class JSONOptions(CodecOptions):
 
         .. versionadded:: 3.12
         """
-        opts = self._asdict()
+        opts = self._options_dict()
         for opt in ('strict_number_long', 'datetime_representation',
                     'strict_uuid', 'json_mode'):
             opts[opt] = kwargs.get(opt, getattr(self, opt))

doc/api/pymongo/collection.rst

@@ -47,8 +47,8 @@
       .. automethod:: aggregate
       .. automethod:: aggregate_raw_batches
       .. automethod:: watch
-      .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None)
+      .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
-      .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+      .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)
       .. automethod:: find_one(filter=None, *args, **kwargs)
       .. automethod:: find_one_and_delete
       .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, hint=None, session=None, **kwargs)

doc/api/pymongo/cursor.rst

@@ -15,7 +15,7 @@
       .. autoattribute:: EXHAUST
          :annotation:
 
-   .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+   .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
       :members:
 
       .. describe:: c[index]
@@ -24,4 +24,4 @@
 
       .. automethod:: __getitem__
 
-   .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+   .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)

doc/changelog.rst

@@ -1,6 +1,64 @@
 Changelog
 =========
 
+Changes in Version 3.11.4
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.11.4 fixes a bug where a MongoClient would mistakenly attempt to
+create minPoolSize connections to arbiter nodes (`PYTHON-2634`_).
+
+See the `PyMongo 3.11.4 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2634: https://jira.mongodb.org/browse/PYTHON-2452
+.. _PyMongo 3.11.4 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=30902
+
+Changes in Version 3.11.3
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.11.3 fixes a bug that prevented PyMongo from retrying writes after
+a ``writeConcernError`` on MongoDB 4.4+ (`PYTHON-2452`_)
+
+See the `PyMongo 3.11.3 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2452: https://jira.mongodb.org/browse/PYTHON-2452
+.. _PyMongo 3.11.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=30355
+
+Changes in Version 3.11.2
+-------------------------
+
+Issues Resolved
+...............
+
+Version 3.11.2 includes a number of bugfixes. Highlights include:
+
+- Fixed a memory leak caused by failing SDAM monitor checks on Python 3 (`PYTHON-2433`_).
+- Fixed a regression that changed the string representation of
+  :exc:`~pymongo.errors.BulkWriteError` (`PYTHON-2438`_).
+- Fixed a bug that made it impossible to use
+  :meth:`bson.codec_options.CodecOptions.with_options` and
+  :meth:`~bson.json_util.JSONOptions.with_options` on some early versions of
+  Python 3.4 and Python 3.5 due to a bug in the standard library implementation
+  of :meth:`collections.namedtuple._asdict` (`PYTHON-2440`_).
+- Fixed a bug that resulted in a :exc:`TypeError` exception when a PyOpenSSL
+  socket was configured with a timeout of ``None`` (`PYTHON-2443`_).
+
+See the `PyMongo 3.11.2 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PYTHON-2433: https://jira.mongodb.org/browse/PYTHON-2433
+.. _PYTHON-2438: https://jira.mongodb.org/browse/PYTHON-2438
+.. _PYTHON-2440: https://jira.mongodb.org/browse/PYTHON-2440
+.. _PYTHON-2443: https://jira.mongodb.org/browse/PYTHON-2443
+.. _PyMongo 3.11.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=30315
+
 Changes in Version 3.11.1
 -------------------------
 

doc/conf.py

@@ -91,13 +91,6 @@ html_theme_options = {
 # Additional static files.
 html_static_path = ['static']
 
-# These paths are either relative to html_static_path
-# or fully qualified paths (eg. https://...)
-# Note: html_js_files was added in Sphinx 1.8.
-html_js_files = [
-    'delighted.js',
-]
-
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None

doc/static/delighted.js

@@ -1,22 +0,0 @@
-/* eslint-disable */
-// Delighted
-!function(e,t,r,n,a){if(!e[a]){for(var i=e[a]=[],s=0;s<r.length;s++){var c=r[s];i[c]=i[c]||function(e){return function(){var t=Array.prototype.slice.call(arguments);i.push([e,t])}}(c)}i.SNIPPET_VERSION="1.0.1";var o=t.createElement("script");o.type="text/javascript",o.async=!0,o.src="https://d2yyd1h5u9mauk.cloudfront.net/integrations/web/v1/library/"+n+"/"+a+".js";var l=t.getElementsByTagName("script")[0];l.parentNode.insertBefore(o,l)}}(window,document,["survey","reset","config","init","set","get","event","identify","track","page","screen","group","alias"],"Dk30CC86ba0nATlK","delighted");
-// Segment
-!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t,e){var n=document.createElement("script");n.type="text/javascript";n.async=!0;n.src="https://cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var a=document.getElementsByTagName("script")[0];a.parentNode.insertBefore(n,a);analytics._loadOptions=e};analytics.SNIPPET_VERSION="4.1.0";
-        analytics.load("aGhVvyxnPWlyP71vVl9ZjGWxAtoVGLXX");
-    }}();
-
-delighted.survey({
-    minTimeOnPage: 180,
-    sampleFactor: 0.1,
-    properties: {
-        project: 'pymongo'
-    }
-});
-
-// Update Segment
-analytics.page({
-    path: location.pathname,
-    url: location.href,
-    project: 'pymongo'
-});

pymongo/__init__.py

@@ -74,7 +74,7 @@ SLOW_ONLY = 1
 ALL = 2
 """Profile all operations."""
 
-version_tuple = (3, 11, 1)
+version_tuple = (3, 11, 4)
 
 def get_version_string():
     if isinstance(version_tuple[-1], str):

pymongo/common.py

@@ -623,12 +623,14 @@ URI_OPTIONS_VALIDATOR_MAP = {
     'tls': validate_boolean_or_string,
     'tlsallowinvalidcertificates': validate_allow_invalid_certs,
     'ssl_cert_reqs': validate_cert_reqs,
+    # Normalized to ssl_match_hostname which is the logical inverse of tlsallowinvalidhostnames
     'tlsallowinvalidhostnames': lambda *x: not validate_boolean_or_string(*x),
     'ssl_match_hostname': validate_boolean_or_string,
     'tlscafile': validate_readable,
     'tlscertificatekeyfile': validate_readable,
     'tlscertificatekeyfilepassword': validate_string_or_none,
-    'tlsdisableocspendpointcheck': validate_boolean_or_string,
+    # Normalized to ssl_check_ocsp_endpoint which is the logical inverse of tlsdisableocspendpointcheck
+    'tlsdisableocspendpointcheck': lambda *x: not validate_boolean_or_string(*x),
     'tlsinsecure': validate_boolean_or_string,
     'w': validate_non_negative_int_or_basestring,
     'wtimeoutms': validate_non_negative_integer,

pymongo/errors.py

@@ -240,8 +240,9 @@ class BulkWriteError(OperationFailure):
     def __init__(self, results):
         super(BulkWriteError, self).__init__(
             "batch op errors occurred", 65, results)
-        # For pickle support
+
-        self.args = (results,)
+    def __reduce__(self):
+        return self.__class__, (self.details,)
 
 
 class InvalidOperation(PyMongoError):

pymongo/helpers.py

@@ -115,7 +115,11 @@ def _check_command_response(response, max_wire_version,
                                max_wire_version)
 
     if parse_write_concern_error and 'writeConcernError' in response:
-        _raise_write_concern_error(response['writeConcernError'])
+        _error = response["writeConcernError"]
+        _labels = response.get("errorLabels")
+        if _labels:
+            _error.update({'errorLabels': _labels})
+        _raise_write_concern_error(_error)
 
     if response["ok"]:
         return
@@ -223,6 +227,9 @@ def _check_write_command_response(result):
 
     error = result.get("writeConcernError")
     if error:
+        error_labels = result.get("errorLabels")
+        if error_labels:
+            error.update({'errorLabels': error_labels})
         _raise_write_concern_error(error)
 
 

pymongo/monitor.py

@@ -18,6 +18,8 @@ import atexit
 import threading
 import weakref
 
+from bson.py3compat import PY3
+
 from pymongo import common, periodic_executor
 from pymongo.errors import (NotMasterError,
                             OperationFailure,
@@ -30,6 +32,14 @@ from pymongo.server_description import ServerDescription
 from pymongo.srv_resolver import _SrvResolver
 
 
+def _sanitize(error):
+    """PYTHON-2433 Clear error traceback info."""
+    if PY3:
+        error.__traceback__ = None
+        error.__context__ = None
+        error.__cause__ = None
+
+
 class MonitorBase(object):
     def __init__(self, topology, name, interval, min_interval):
         """Base class to do periodic work on a background thread.
@@ -169,6 +179,7 @@ class Monitor(MonitorBase):
             try:
                 self._server_description = self._check_server()
             except _OperationCancelled as exc:
+                _sanitize(exc)
                 # Already closed the connection, wait for the next check.
                 self._server_description = ServerDescription(
                     self._server_description.address, error=exc)
@@ -212,6 +223,7 @@ class Monitor(MonitorBase):
         except ReferenceError:
             raise
         except Exception as error:
+            _sanitize(error)
             sd = self._server_description
             address = sd.address
             duration = _time() - start

pymongo/socket_checker.py

@@ -41,7 +41,7 @@ class SocketChecker(object):
             self._poller = None
 
     def select(self, sock, read=False, write=False, timeout=0):
-        """Select for reads or writes with a timeout in seconds.
+        """Select for reads or writes with a timeout in seconds (or None).
 
         Returns True if the socket is readable/writable, False on timeout.
         """
@@ -57,7 +57,8 @@ class SocketChecker(object):
                     try:
                         # poll() timeout is in milliseconds. select()
                         # timeout is in seconds.
-                        res = self._poller.poll(timeout * 1000)
+                        timeout_ = None if timeout is None else timeout * 1000
+                        res = self._poller.poll(timeout_)
                         # poll returns a possibly-empty list containing
                         # (fd, event) 2-tuples for the descriptors that have
                         # events or errors to report. Return True if the list

pymongo/topology.py

@@ -430,15 +430,26 @@ class Topology(object):
                     ServerDescription(address, error=error), True)
                 server.request_check()
 
+    def data_bearing_servers(self):
+        """Return a list of all data-bearing servers.
+
+        This includes any server that might be selected for an operation.
+        """
+        if self._description.topology_type == TOPOLOGY_TYPE.Single:
+            return self._description.known_servers
+        return self._description.readable_servers
+
     def update_pool(self, all_credentials):
         # Remove any stale sockets and add new sockets if pool is too small.
         servers = []
         with self._lock:
-            for server in self._servers.values():
+            # Only update pools for data-bearing servers.
-                servers.append((server, server._pool.generation))
+            for sd in self.data_bearing_servers():
+                server = self._servers[sd.address]
+                servers.append((server, server.pool.generation))
 
         for server, generation in servers:
-            server._pool.remove_stale_sockets(generation, all_credentials)
+            server.pool.remove_stale_sockets(generation, all_credentials)
 
     def close(self):
         """Clear pools and terminate monitors. Topology reopens on demand."""

setup.py

@@ -39,7 +39,7 @@ except ImportError:
     except ImportError:
         _HAVE_SPHINX = False
 
-version = "3.11.1"
+version = "3.11.4"
 
 f = open("README.rst")
 try:

test/pymongo_mocks.py

@@ -85,13 +85,13 @@ class MockMonitor(Monitor):
 class MockClient(MongoClient):
     def __init__(
             self, standalones, members, mongoses, ismaster_hosts=None,
-            *args, **kwargs):
+            arbiters=None, down_hosts=None, *args, **kwargs):
         """A MongoClient connected to the default server, with a mock topology.
 
-        standalones, members, mongoses determine the configuration of the
+        standalones, members, mongoses, arbiters, and down_hosts determine the
-        topology. They are formatted like ['a:1', 'b:2']. ismaster_hosts
+        configuration of the topology. They are formatted like ['a:1', 'b:2'].
-        provides an alternative host list for the server's mocked ismaster
+        ismaster_hosts provides an alternative host list for the server's
-        response; see test_connect_with_internal_ips.
+        mocked ismaster response; see test_connect_with_internal_ips.
         """
         self.mock_standalones = standalones[:]
         self.mock_members = members[:]
@@ -101,6 +101,9 @@ class MockClient(MongoClient):
         else:
             self.mock_primary = None
 
+        # Hosts that should be considered an arbiter.
+        self.mock_arbiters = arbiters[:] if arbiters else []
+
         if ismaster_hosts is not None:
             self.mock_ismaster_hosts = ismaster_hosts
         else:
@@ -109,7 +112,7 @@ class MockClient(MongoClient):
         self.mock_mongoses = mongoses[:]
 
         # Hosts that should raise socket errors.
-        self.mock_down_hosts = []
+        self.mock_down_hosts = down_hosts[:] if down_hosts else []
 
         # Hostname -> (min wire version, max wire version)
         self.mock_wire_versions = {}
@@ -182,6 +185,10 @@ class MockClient(MongoClient):
 
             if self.mock_primary:
                 response['primary'] = self.mock_primary
+
+            if host in self.mock_arbiters:
+                response['arbiterOnly'] = True
+                response['secondary'] = False
         elif host in self.mock_mongoses:
             response = {
                 'ok': 1,

test/test_bson.py

@@ -373,6 +373,13 @@ class TestBSON(unittest.TestCase):
                 with self.assertRaises(InvalidBSON, msg=msg):
                     list(decode_file_iter(scratch))
 
+    def test_invalid_field_name(self):
+        # Decode a truncated field
+        with self.assertRaises(InvalidBSON) as ctx:
+            decode(b'\x0b\x00\x00\x00\x02field\x00')
+        # Assert that the InvalidBSON error message is not empty.
+        self.assertTrue(str(ctx.exception))
+
     def test_data_timestamp(self):
         self.assertEqual({"test": Timestamp(4, 20)},
                          decode(b"\x13\x00\x00\x00\x11\x74\x65\x73\x74\x00\x14"

test/test_client.py

@@ -35,7 +35,7 @@ from bson.py3compat import thread
 from bson.son import SON
 from bson.tz_util import utc
 import pymongo
-from pymongo import auth, message
+from pymongo import auth, message, monitoring
 from pymongo.common import CONNECT_TIMEOUT, _UUID_REPRESENTATIONS
 from pymongo.command_cursor import CommandCursor
 from pymongo.compression_support import _HAVE_SNAPPY, _HAVE_ZSTD
@@ -57,7 +57,8 @@ from pymongo.monotonic import time as monotonic_time
 from pymongo.driver_info import DriverInfo
 from pymongo.pool import SocketInfo, _METADATA
 from pymongo.read_preferences import ReadPreference
-from pymongo.server_selectors import (any_server_selector,
+from pymongo.server_description import ServerDescription
+from pymongo.server_selectors import (readable_server_selector,
                                       writable_server_selector)
 from pymongo.server_type import SERVER_TYPE
 from pymongo.settings import TOPOLOGY_TYPE
@@ -75,6 +76,7 @@ from test import (client_context,
 from test.pymongo_mocks import MockClient
 from test.utils import (assertRaisesExactly,
                         connected,
+                        CMAPListener,
                         delay,
                         FunctionCallRecorder,
                         get_pool,
@@ -279,6 +281,9 @@ class ClientUnitTest(unittest.TestCase):
             readpreference=ReadPreference.NEAREST.mongos_mode)
         self.assertEqual(c.read_preference, ReadPreference.NEAREST)
 
+    @unittest.skipIf(
+        sys.version_info[0] == 3 and sys.version_info[1] == 4,
+        "PYTHON-2442: workaround namedtuple._asdict() bug on Python 3.4")
     def test_metadata(self):
         metadata = copy.deepcopy(_METADATA)
         metadata['application'] = {'name': 'foobar'}
@@ -448,21 +453,25 @@ class ClientUnitTest(unittest.TestCase):
 
 class TestClient(IntegrationTest):
 
-    def test_max_idle_time_reaper(self):
+    def test_max_idle_time_reaper_default(self):
         with client_knobs(kill_cursor_frequency=0.1):
             # Assert reaper doesn't remove sockets when maxIdleTimeMS not set
             client = rs_or_single_client()
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             with server._pool.get_socket({}) as sock_info:
                 pass
             self.assertEqual(1, len(server._pool.sockets))
             self.assertTrue(sock_info in server._pool.sockets)
             client.close()
 
+    def test_max_idle_time_reaper_removes_stale_minPoolSize(self):
+        with client_knobs(kill_cursor_frequency=0.1):
             # Assert reaper removes idle socket and replaces it with a new one
             client = rs_or_single_client(maxIdleTimeMS=500,
                                          minPoolSize=1)
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             with server._pool.get_socket({}) as sock_info:
                 pass
             # When the reaper runs at the same time as the get_socket, two
@@ -474,11 +483,14 @@ class TestClient(IntegrationTest):
                        "replace stale socket")
             client.close()
 
+    def test_max_idle_time_reaper_does_not_exceed_maxPoolSize(self):
+        with client_knobs(kill_cursor_frequency=0.1):
             # Assert reaper respects maxPoolSize when adding new sockets.
             client = rs_or_single_client(maxIdleTimeMS=500,
                                          minPoolSize=1,
                                          maxPoolSize=1)
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             with server._pool.get_socket({}) as sock_info:
                 pass
             # When the reaper runs at the same time as the get_socket,
@@ -490,9 +502,12 @@ class TestClient(IntegrationTest):
                        "replace stale socket")
             client.close()
 
+    def test_max_idle_time_reaper_removes_stale(self):
+        with client_knobs(kill_cursor_frequency=0.1):
             # Assert reaper has removed idle socket and NOT replaced it
             client = rs_or_single_client(maxIdleTimeMS=500)
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             with server._pool.get_socket({}) as sock_info_one:
                 pass
             # Assert that the pool does not close sockets prematurely.
@@ -508,12 +523,14 @@ class TestClient(IntegrationTest):
     def test_min_pool_size(self):
         with client_knobs(kill_cursor_frequency=.1):
             client = rs_or_single_client()
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             self.assertEqual(0, len(server._pool.sockets))
 
             # Assert that pool started up at minPoolSize
             client = rs_or_single_client(minPoolSize=10)
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             wait_until(lambda: 10 == len(server._pool.sockets),
                        "pool initialized with 10 sockets")
 
@@ -528,7 +545,8 @@ class TestClient(IntegrationTest):
         # Use high frequency to test _get_socket_no_auth.
         with client_knobs(kill_cursor_frequency=99999999):
             client = rs_or_single_client(maxIdleTimeMS=500)
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             with server._pool.get_socket({}) as sock_info:
                 pass
             self.assertEqual(1, len(server._pool.sockets))
@@ -542,7 +560,8 @@ class TestClient(IntegrationTest):
 
             # Test that sockets are reused if maxIdleTimeMS is not set.
             client = rs_or_single_client()
-            server = client._get_topology().select_server(any_server_selector)
+            server = client._get_topology().select_server(
+                readable_server_selector)
             with server._pool.get_socket({}) as sock_info:
                 pass
             self.assertEqual(1, len(server._pool.sockets))
@@ -1614,6 +1633,35 @@ class TestClient(IntegrationTest):
         with self.assertRaises(ConfigurationError):
             MongoClient(['host1', 'host2'], directConnection=True)
 
+    @unittest.skipIf(sys.platform.startswith('java'),
+                     'Jython does not support gc.get_objects')
+    def test_continuous_network_errors(self):
+        def server_description_count():
+            i = 0
+            for obj in gc.get_objects():
+                try:
+                    if isinstance(obj, ServerDescription):
+                        i += 1
+                except ReferenceError:
+                    pass
+            return i
+        gc.collect()
+        with client_knobs(min_heartbeat_interval=0.003):
+            client = MongoClient(
+                'invalid:27017',
+                heartbeatFrequencyMS=3,
+                serverSelectionTimeoutMS=100)
+            initial_count = server_description_count()
+            self.addCleanup(client.close)
+            with self.assertRaises(ServerSelectionTimeoutError):
+                client.test.test.find_one()
+            gc.collect()
+            final_count = server_description_count()
+            # If a bug like PYTHON-2433 is reintroduced then too many
+            # ServerDescriptions will be kept alive and this test will fail:
+            # AssertionError: 4 != 22 within 5 delta (18 difference)
+            self.assertAlmostEqual(initial_count, final_count, delta=10)
+
 
 class TestExhaustCursor(IntegrationTest):
     """Test that clients properly handle errors from exhaust cursors."""
@@ -1975,5 +2023,62 @@ class TestMongoClientFailover(MockClientTest):
         self.assertIsNone(ct.get())
 
 
+class TestClientPool(MockClientTest):
+
+    @client_context.require_connection
+    def test_rs_client_does_not_maintain_pool_to_arbiters(self):
+        listener = CMAPListener()
+        c = MockClient(
+            standalones=[],
+            members=['a:1', 'b:2', 'c:3', 'd:4'],
+            mongoses=[],
+            arbiters=['c:3'],  # c:3 is an arbiter.
+            down_hosts=['d:4'],  # d:4 is unreachable.
+            host=['a:1', 'b:2', 'c:3', 'd:4'],
+            replicaSet='rs',
+            minPoolSize=1,  # minPoolSize
+            event_listeners=[listener],
+        )
+        self.addCleanup(c.close)
+
+        wait_until(lambda: len(c.nodes) == 3, 'connect')
+        self.assertEqual(c.address, ('a', 1))
+        self.assertEqual(c.arbiters, set([('c', 3)]))
+        # Assert that we create 2 and only 2 pooled connections.
+        listener.wait_for_event(monitoring.ConnectionReadyEvent, 2)
+        self.assertEqual(
+            listener.event_count(monitoring.ConnectionCreatedEvent), 2)
+        # Assert that we do not create connections to arbiters.
+        arbiter = c._topology.get_server_by_address(('c', 3))
+        self.assertFalse(arbiter.pool.sockets)
+        # Assert that we do not create connections to unknown servers.
+        arbiter = c._topology.get_server_by_address(('d', 4))
+        self.assertFalse(arbiter.pool.sockets)
+
+    @client_context.require_connection
+    def test_direct_client_maintains_pool_to_arbiter(self):
+        listener = CMAPListener()
+        c = MockClient(
+            standalones=[],
+            members=['a:1', 'b:2', 'c:3'],
+            mongoses=[],
+            arbiters=['c:3'],  # c:3 is an arbiter.
+            host='c:3',
+            directConnection=True,
+            minPoolSize=1,  # minPoolSize
+            event_listeners=[listener],
+        )
+        self.addCleanup(c.close)
+
+        wait_until(lambda: len(c.nodes) == 1, 'connect')
+        self.assertEqual(c.address, ('c', 3))
+        # Assert that we create 1 pooled connection.
+        listener.wait_for_event(monitoring.ConnectionReadyEvent, 1)
+        self.assertEqual(
+            listener.event_count(monitoring.ConnectionCreatedEvent), 1)
+        arbiter = c._topology.get_server_by_address(('c', 3))
+        self.assertEqual(len(arbiter.pool.sockets), 1)
+
+
 if __name__ == "__main__":
     unittest.main()

test/test_database.py

@@ -444,6 +444,8 @@ class TestDatabase(IntegrationTest):
         self.assertTrue(isinstance(info[0]['op'], string_type))
         self.assertTrue(isinstance(info[0]["ts"], datetime.datetime))
 
+    # SERVER-47817 removes the resetError command.
+    @client_context.require_version_max(4, 9)
     @client_context.require_no_mongos
     @ignore_deprecations
     def test_errors(self):

test/test_errors.py

@@ -93,6 +93,7 @@ class TestErrors(PyMongoTestCase):
     def test_pickle_BulkWriteError(self):
         exc = BulkWriteError({})
         self.assertOperationFailureEqual(exc, pickle.loads(pickle.dumps(exc)))
+        self.assertIn("batch op errors occurred", str(exc))
 
     def test_pickle_EncryptionError(self):
         cause = OperationFailure('error', code=5, details={},

test/test_heartbeat_monitoring.py

@@ -51,12 +51,12 @@ class TestHeartbeatMonitoring(unittest.TestCase):
             # monitor thread may run multiple times during the execution
             # of this test.
             wait_until(
-                lambda: len(listener.results) >= expected_len,
+                lambda: len(listener.events) >= expected_len,
                 "publish all events")
 
         try:
             # zip gives us len(expected_results) pairs.
-            for expected, actual in zip(expected_results, listener.results):
+            for expected, actual in zip(expected_results, listener.events):
                 self.assertEqual(expected,
                                  actual.__class__.__name__)
                 self.assertEqual(actual.connection_id,

test/test_legacy_api.py

@@ -2112,6 +2112,7 @@ class TestLegacyBulk(BulkTestBase):
                   'op': {'_id': '...', 'b': 6, 'a': 1}}]},
             result)
 
+    @client_context.require_version_max(4, 8)  # PYTHON-2436
     def test_large_inserts_ordered(self):
         big = 'x' * self.coll.database.client.max_bson_size
         batch = self.coll.initialize_ordered_bulk_op()

test/test_pooling.py

@@ -21,7 +21,10 @@ import sys
 import threading
 import time
 
-from pymongo import MongoClient
+from bson.son import SON
+from bson.codec_options import DEFAULT_CODEC_OPTIONS
+
+from pymongo import MongoClient, message
 from pymongo.errors import (AutoReconnect,
                             ConnectionFailure,
                             DuplicateKeyError,
@@ -259,6 +262,37 @@ class TestPooling(_TestPoolingBase):
         s.close()
         self.assertTrue(socket_checker.socket_closed(s))
 
+    def test_socket_checker(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect((client_context.host, client_context.port))
+        socket_checker = SocketChecker()
+        # Socket has nothing to read.
+        self.assertFalse(socket_checker.select(s, read=True))
+        self.assertFalse(socket_checker.select(s, read=True, timeout=0))
+        self.assertFalse(socket_checker.select(s, read=True, timeout=.05))
+        # Socket is writable.
+        self.assertTrue(socket_checker.select(s, write=True, timeout=None))
+        self.assertTrue(socket_checker.select(s, write=True))
+        self.assertTrue(socket_checker.select(s, write=True, timeout=0))
+        self.assertTrue(socket_checker.select(s, write=True, timeout=.05))
+        # Make the socket readable
+        _, msg, _ = message.query(
+            0, 'admin.$cmd', 0, -1, SON([('isMaster', 1)]), None,
+            DEFAULT_CODEC_OPTIONS)
+        s.sendall(msg)
+        # Block until the socket is readable.
+        self.assertTrue(socket_checker.select(s, read=True, timeout=None))
+        self.assertTrue(socket_checker.select(s, read=True))
+        self.assertTrue(socket_checker.select(s, read=True, timeout=0))
+        self.assertTrue(socket_checker.select(s, read=True, timeout=.05))
+        # Socket is still writable.
+        self.assertTrue(socket_checker.select(s, write=True, timeout=None))
+        self.assertTrue(socket_checker.select(s, write=True))
+        self.assertTrue(socket_checker.select(s, write=True, timeout=0))
+        self.assertTrue(socket_checker.select(s, write=True, timeout=.05))
+        s.close()
+        self.assertTrue(socket_checker.socket_closed(s))
+
     def test_return_socket_after_reset(self):
         pool = self.create_pool()
         with pool.get_socket({}) as sock:

test/test_retryable_writes.py

@@ -20,14 +20,17 @@ import sys
 
 sys.path[0:0] = [""]
 
+from bson.codec_options import DEFAULT_CODEC_OPTIONS
 from bson.int64 import Int64
 from bson.objectid import ObjectId
+from bson.raw_bson import RawBSONDocument
 from bson.son import SON
 
 
 from pymongo.errors import (ConnectionFailure,
                             OperationFailure,
-                            ServerSelectionTimeoutError)
+                            ServerSelectionTimeoutError,
+                            WriteConcernError)
 from pymongo.mongo_client import MongoClient
 from pymongo.operations import (InsertOne,
                                 DeleteMany,
@@ -43,6 +46,7 @@ from test.utils import (rs_or_single_client,
                         OvertCommandListener,
                         TestCreator)
 from test.utils_spec_runner import SpecRunner
+from test.version import Version
 
 # Location of JSON test specifications.
 _TEST_PATH = os.path.join(
@@ -454,6 +458,60 @@ class TestRetryableWrites(IgnoreDeprecationsTest):
         self.assertEqual(coll.find_one(projection={'_id': True}), {'_id': 1})
 
 
+class TestWriteConcernError(IntegrationTest):
+    @classmethod
+    @client_context.require_replica_set
+    @client_context.require_no_mmap
+    @client_context.require_failCommand_fail_point
+    def setUpClass(cls):
+        super(TestWriteConcernError, cls).setUpClass()
+        cls.fail_insert = {
+            'configureFailPoint': 'failCommand',
+            'mode': {'times': 2},
+            'data': {
+                'failCommands': ['insert'],
+                'writeConcernError': {
+                    'code': 91,
+                    'errmsg': 'Replication is being shut down'},
+            }}
+
+    @client_context.require_version_min(4, 0)
+    def test_RetryableWriteError_error_label(self):
+        listener = OvertCommandListener()
+        client = rs_or_single_client(
+            retryWrites=True, event_listeners=[listener])
+
+        # Ensure collection exists.
+        client.pymongo_test.testcoll.insert_one({})
+
+        with self.fail_point(self.fail_insert):
+            with self.assertRaises(WriteConcernError) as cm:
+                client.pymongo_test.testcoll.insert_one({})
+            self.assertTrue(cm.exception.has_error_label(
+                'RetryableWriteError'))
+
+        if client_context.version >= Version(4, 4):
+            # In MongoDB 4.4+ we rely on the server returning the error label.
+            self.assertIn(
+                'RetryableWriteError',
+                listener.results['succeeded'][-1].reply['errorLabels'])
+
+    @client_context.require_version_min(4, 4)
+    def test_RetryableWriteError_error_label_RawBSONDocument(self):
+        # using RawBSONDocument should not cause errorLabel parsing to fail
+        with self.fail_point(self.fail_insert):
+            with self.client.start_session() as s:
+                s._start_retryable_write()
+                result = self.client.pymongo_test.command(
+                    'insert', 'testcoll', documents=[{'_id': 1}],
+                    txnNumber=s._server_session.transaction_id, session=s,
+                    codec_options=DEFAULT_CODEC_OPTIONS.with_options(
+                        document_class=RawBSONDocument))
+
+        self.assertIn('writeConcernError', result)
+        self.assertIn('RetryableWriteError', result['errorLabels'])
+
+
 # TODO: Make this a real integration test where we stepdown the primary.
 class TestRetryableWritesTxnNumber(IgnoreDeprecationsTest):
     @client_context.require_version_min(3, 6)

test/test_streaming_protocol.py

@@ -215,7 +215,7 @@ class TestStreamingProtocol(IntegrationTest):
         self.assertTrue(hb_failed_events[0].awaited)
         # Depending on thread scheduling, the failed heartbeat could occur on
         # the second or third check.
-        events = [type(e) for e in hb_listener.results[:4]]
+        events = [type(e) for e in hb_listener.events[:4]]
         if events == [monitoring.ServerHeartbeatStartedEvent,
                       monitoring.ServerHeartbeatSucceededEvent,
                       monitoring.ServerHeartbeatStartedEvent,

test/test_uri_parser.py

@@ -480,6 +480,16 @@ class TestURI(unittest.TestCase):
         with self.assertRaises(InvalidURI):
             parse_uri(uri, validate=False, warn=False, normalize=False)
 
+    def test_tlsDisableOCSPEndpointCheck(self):
+        # check that tlsDisableOCSPEndpointCheck is handled correctly.
+        uri = "mongodb://example.com/?tlsDisableOCSPEndpointCheck=true"
+        res = {'ssl_check_ocsp_endpoint': False}
+        self.assertEqual(res, parse_uri(uri)["options"])
+
+        uri = "mongodb://example.com/?tlsDisableOCSPEndpointCheck=false"
+        res = {'ssl_check_ocsp_endpoint': True}
+        self.assertEqual(res, parse_uri(uri)["options"])
+
     def test_normalize_options(self):
         # check that options are converted to their internal names correctly.
         uri = ("mongodb://example.com/?tls=true&appname=myapp&maxPoolSize=10&"

test/utils.py

@@ -36,7 +36,7 @@ from bson.son import SON
 from pymongo import (MongoClient,
                      monitoring, read_preferences)
 from pymongo.errors import ConfigurationError, OperationFailure
-from pymongo.monitoring import _SENSITIVE_COMMANDS, ConnectionPoolListener
+from pymongo.monitoring import _SENSITIVE_COMMANDS
 from pymongo.pool import (_CancellationContext,
                           PoolOptions)
 from pymongo.read_concern import ReadConcern
@@ -60,7 +60,7 @@ else:
 IMPOSSIBLE_WRITE_CONCERN = WriteConcern(w=50)
 
 
-class CMAPListener(ConnectionPoolListener):
+class BaseListener(object):
     def __init__(self):
         self.events = []
 
@@ -71,8 +71,26 @@ class CMAPListener(ConnectionPoolListener):
         self.events.append(event)
 
     def event_count(self, event_type):
-        return len([event for event in self.events[:]
+        return len(self.events_by_type(event_type))
-                    if isinstance(event, event_type)])
+
+    def events_by_type(self, event_type):
+        """Return the matching events by event class.
+
+        event_type can be a single class or a tuple of classes.
+        """
+        return self.matching(lambda e: isinstance(e, event_type))
+
+    def matching(self, matcher):
+        """Return the matching events."""
+        return [event for event in self.events[:] if matcher(event)]
+
+    def wait_for_event(self, event, count):
+        """Wait for a number of events to be published, or fail."""
+        wait_until(lambda: self.event_count(event) >= count,
+                   'find %s %s event(s)' % (count, event))
+
+
+class CMAPListener(BaseListener, monitoring.ConnectionPoolListener):
 
     def connection_created(self, event):
         self.add_event(event)
@@ -196,25 +214,17 @@ class ServerAndTopologyEventListener(ServerEventListener,
     """Listens to Server and Topology events."""
 
 
-class HeartbeatEventListener(monitoring.ServerHeartbeatListener):
+class HeartbeatEventListener(BaseListener, monitoring.ServerHeartbeatListener):
     """Listens to only server heartbeat events."""
 
-    def __init__(self):
-        self.results = []
-
     def started(self, event):
-        self.results.append(event)
+        self.add_event(event)
 
     def succeeded(self, event):
-        self.results.append(event)
+        self.add_event(event)
 
     def failed(self, event):
-        self.results.append(event)
+        self.add_event(event)
-
-    def matching(self, matcher):
-        """Return the matching events."""
-        results = self.results[:]
-        return [event for event in results if matcher(event)]
 
 
 class MockSocketInfo(object):