diff --git a/.evergreen/config.yml b/.evergreen/config.yml
index 41ebe8fc2..264c028dd 100644
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@@ -339,19 +339,6 @@ functions:
            ${PREPARE_SHELL}
            PYTHON_BINARY=${PYTHON_BINARY} sh ${PROJECT_DIRECTORY}/.evergreen/run-cdecimal-tests.sh
 
-  "run pyopenssl tests":
-     - command: shell.exec
-       type: test
-       params:
-         working_dir: "src"
-         script: |
-           set -o xtrace
-           ${PREPARE_SHELL}
-           if [ -n "${MONGODB_STARTED}" ]; then
-             export PYMONGO_MUST_CONNECT=1
-           fi
-           PYTHON_BINARY=${PYTHON_BINARY} sh ${PROJECT_DIRECTORY}/.evergreen/run-pyopenssl-tests.sh
-
   "run doctests":
      - command: shell.exec
        type: test
@@ -399,6 +386,9 @@ functions:
             export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
             export TEST_ENCRYPTION=1
           fi
+          if [ -n "${test_pyopenssl}" ]; then
+            export TEST_PYOPENSSL=1
+          fi
           if [ -n "${SETDEFAULTENCODING}" ]; then
             export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
           fi
@@ -1076,17 +1066,6 @@ tasks:
             TOPOLOGY: "server"
         - func: "run cdecimal tests"
 
-    # Use latest for this, since we're
-    # adding this to support OCSP stapling
-    - name: "pyopenssl"
-      tags: ["pyopenssl"]
-      commands:
-        - func: "bootstrap mongo-orchestration"
-          vars:
-            VERSION: "latest"
-            TOPOLOGY: "server"
-        - func: "run pyopenssl tests"
-
     - name: "no-server"
       tags: ["no-server"]
       commands:
@@ -1632,6 +1611,17 @@ axes:
         tags: ["encryption_tag"]
         variables:
           test_encryption: true
+        batchtime: 10080  # 7 days
+
+  # Run pyopenssl tests?
+  - id: pyopenssl
+    display_name: "PyOpenSSL"
+    values:
+      - id: "enabled"
+        display_name: "PyOpenSSL"
+        variables:
+          test_pyopenssl: true
+        batchtime: 10080  # 7 days
 
   # Run setdefaultencoding before running the test suite?
   - id: setdefaultencoding
@@ -1834,39 +1824,52 @@ buildvariants:
     python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "pypy", "pypy3.5"]
     auth: "*"
     ssl: "ssl"
+    pyopenssl: "*"
+  # Only test "noauth" with Python 3.7.
+  exclude_spec:
+    platform: ubuntu-16.04
+    python-version: ["2.7", "3.4", "3.5", "3.6", "pypy", "pypy3.5"]
+    auth: "noauth"
+    ssl: "ssl"
+    pyopenssl: "*"
   display_name: "PyOpenSSL ${platform} ${python-version} ${auth}"
   tasks:
-    - "pyopenssl"
+    - '.replica_set !.2.6 !.3.0'
+    # Test standalone and sharded only on 4.4.
+    - '.4.4'
 
 - matrix_name: "test-pyopenssl-old-py27"
   matrix_spec:
     platform:
       # Supported OSes with pre-2.7.9 CPython versions.
       - rhel70 # CPython 2.7.5
-    auth: "*"
+    auth: "auth"
     ssl: "ssl"
+    pyopenssl: "*"
   display_name: "PyOpenSSL ${platform}  Pre-2.7.9 Python ${auth}"
   tasks:
-    - "pyopenssl"
+    - '.replica_set'
 
 - matrix_name: "tests-pyopenssl-macOS"
   matrix_spec:
     platform: macos-1014
-    auth: "*"
+    auth: "auth"
     ssl: "ssl"
+    pyopenssl: "*"
   display_name: "PyOpenSSL ${platform} ${auth}"
   tasks:
-    - "pyopenssl"
+    - '.replica_set !.2.6 !.3.0'
 
 - matrix_name: "tests-pyopenssl-windows"
   matrix_spec:
     platform: windows-64-vsMulti-small
     python-version-windows: "*"
-    auth: "*"
+    auth: "auth"
     ssl: "ssl"
+    pyopenssl: "*"
   display_name: "PyOpenSSL ${platform} ${python-version-windows} ${auth}"
   tasks:
-    - "pyopenssl"
+    - '.replica_set'
 
 - matrix_name: "tests-python-version-rhel62-test-encryption"
   matrix_spec:
diff --git a/.evergreen/run-pyopenssl-tests.sh b/.evergreen/run-pyopenssl-tests.sh
deleted file mode 100644
index ea4271572..000000000
--- a/.evergreen/run-pyopenssl-tests.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-set -o xtrace
-set -o errexit
-
-export DB_USER="bob"
-export DB_PASSWORD="pwd123"
-export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
-export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
-
-if [ -z "$PYTHON_BINARY" ]; then
-    echo "No python binary specified"
-    PYTHON=$(command -v python || command -v python3) || true
-    if [ -z "$PYTHON" ]; then
-        echo "Cannot test without python or python3 installed!"
-        exit 1
-    fi
-else
-    PYTHON="$PYTHON_BINARY"
-fi
-
-if $PYTHON -m virtualenv --version; then
-    VIRTUALENV="$PYTHON -m virtualenv"
-elif command -v virtualenv; then
-    # We can remove this fallback after:
-    # https://github.com/10gen/mongo-python-toolchain/issues/8
-    VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-else
-    echo "Cannot test without virtualenv"
-    exit 1
-fi
-
-$VIRTUALENV pyopenssltest
-if [ "Windows_NT" = "$OS" ]; then
-  . pyopenssltest/Scripts/activate
-else
-  . pyopenssltest/bin/activate
-fi
-trap "deactivate; rm -rf pyopenssltest" EXIT HUP
-
-IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
-if [ $IS_PYTHON_2 = "1" ]; then
-    echo "Using a Python 2"
-    # Upgrade pip to install the cryptography wheel and not the tar.
-    # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-    pip install --upgrade 'pip<20.1'
-    # Upgrade setuptools because cryptography requires 18.5+.
-    # <45 because 45.0 dropped support for 2.7.
-    pip install --upgrade 'setuptools<45'
-fi
-
-pip install pyopenssl requests service_identity
-python -c 'import sys; print(sys.version)'
-python setup.py test
diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh
index 8b27e0ec8..50902d020 100755
--- a/.evergreen/run-tests.sh
+++ b/.evergreen/run-tests.sh
@@ -89,6 +89,41 @@ else
     PYTHON="$PYTHON_BINARY"
 fi
 
+# PyOpenSSL test setup.
+if [ -n "$TEST_PYOPENSSL" ]; then
+    if $PYTHON -m virtualenv --version; then
+        VIRTUALENV="$PYTHON -m virtualenv"
+    elif command -v virtualenv; then
+        # We can remove this fallback after:
+        # https://github.com/10gen/mongo-python-toolchain/issues/8
+        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
+    else
+        echo "Cannot test without virtualenv"
+        exit 1
+    fi
+
+    $VIRTUALENV pyopenssltest
+    if [ "Windows_NT" = "$OS" ]; then
+        . pyopenssltest/Scripts/activate
+    else
+        . pyopenssltest/bin/activate
+    fi
+    trap "deactivate; rm -rf pyopenssltest" EXIT HUP
+
+    IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
+    if [ $IS_PYTHON_2 = "1" ]; then
+        echo "Using a Python 2"
+        # Upgrade pip to install the cryptography wheel and not the tar.
+        # <20.1 because 20.0.2 says a future release may drop support for 2.7.
+        python -m pip install --upgrade 'pip<20.1'
+        # Upgrade setuptools because cryptography requires 18.5+.
+        # <45 because 45.0 dropped support for 2.7.
+        python -m pip install --upgrade 'setuptools<45'
+    fi
+
+    python -m pip install pyopenssl requests service_identity
+fi
+
 if [ -n "$TEST_ENCRYPTION" ]; then
     if [ -z "$LIBMONGOCRYPT_URL" ]; then
         echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"