PYTHON-5260 Fix OCSP test setup (#2254)

.evergreen/config.yml

@@ -207,7 +207,7 @@ functions:
         binary: bash
         working_dir: "src"
         include_expansions_in_env: [VERSION, TOPOLOGY, AUTH, SSL, ORCHESTRATION_FILE, PYTHON_BINARY, PYTHON_VERSION,
-          STORAGE_ENGINE, REQUIRE_API_VERSION, DRIVERS_TOOLS, TEST_CRYPT_SHARED, AUTH_AWS, LOAD_BALANCER]
+          STORAGE_ENGINE, REQUIRE_API_VERSION, DRIVERS_TOOLS, TEST_CRYPT_SHARED, AUTH_AWS, LOAD_BALANCER, LOCAL_ATLAS]
         args: [.evergreen/just.sh, run-server, "${TEST_NAME}"]
     - command: expansions.update
       params:
@@ -229,7 +229,7 @@ functions:
         include_expansions_in_env: [AUTH, SSL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
           AWS_SESSION_TOKEN, COVERAGE, PYTHON_BINARY, LIBMONGOCRYPT_URL, MONGODB_URI, PYTHON_VERSION,
           DISABLE_TEST_COMMANDS, GREEN_FRAMEWORK, NO_EXT, COMPRESSORS, MONGODB_API_VERSION, DEBUG_LOG,
-          ORCHESTRATION_FILE, OCSP_SERVER_TYPE]
+          ORCHESTRATION_FILE, OCSP_SERVER_TYPE, VERSION]
         binary: bash
         working_dir: "src"
         args: [.evergreen/just.sh, setup-tests, "${TEST_NAME}", "${SUB_TEST_NAME}"]

.evergreen/generated_configs/variants.yml

@@ -729,149 +729,29 @@ buildvariants:
       - rhel87-small
 
   # Ocsp tests
-  - name: ocsp-rhel8-v4.4-python3.9
+  - name: ocsp-rhel8
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 v4.4 Python3.9
+    display_name: OCSP RHEL8
     run_on:
       - rhel87-small
     batchtime: 10080
-    expansions:
+  - name: ocsp-win64
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "4.4"
-      PYTHON_BINARY: /opt/python/3.9/bin/python3
-  - name: ocsp-rhel8-v5.0-python3.10
     tasks:
-      - name: .ocsp
+      - name: .ocsp-rsa !.ocsp-staple .latest
-    display_name: OCSP RHEL8 v5.0 Python3.10
+      - name: .ocsp-rsa !.ocsp-staple .4.4
-    run_on:
+    display_name: OCSP Win64
-      - rhel87-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "5.0"
-      PYTHON_BINARY: /opt/python/3.10/bin/python3
-  - name: ocsp-rhel8-v6.0-python3.11
-    tasks:
-      - name: .ocsp
-    display_name: OCSP RHEL8 v6.0 Python3.11
-    run_on:
-      - rhel87-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "6.0"
-      PYTHON_BINARY: /opt/python/3.11/bin/python3
-  - name: ocsp-rhel8-v7.0-python3.12
-    tasks:
-      - name: .ocsp
-    display_name: OCSP RHEL8 v7.0 Python3.12
-    run_on:
-      - rhel87-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "7.0"
-      PYTHON_BINARY: /opt/python/3.12/bin/python3
-  - name: ocsp-rhel8-v8.0-python3.13
-    tasks:
-      - name: .ocsp
-    display_name: OCSP RHEL8 v8.0 Python3.13
-    run_on:
-      - rhel87-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "8.0"
-      PYTHON_BINARY: /opt/python/3.13/bin/python3
-  - name: ocsp-rhel8-rapid-pypy3.10
-    tasks:
-      - name: .ocsp
-    display_name: OCSP RHEL8 rapid PyPy3.10
-    run_on:
-      - rhel87-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: rapid
-      PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
-  - name: ocsp-rhel8-latest-python3.9
-    tasks:
-      - name: .ocsp
-    display_name: OCSP RHEL8 latest Python3.9
-    run_on:
-      - rhel87-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: latest
-      PYTHON_BINARY: /opt/python/3.9/bin/python3
-  - name: ocsp-win64-v4.4-python3.9
-    tasks:
-      - name: .ocsp-rsa !.ocsp-staple
-    display_name: OCSP Win64 v4.4 Python3.9
     run_on:
       - windows-64-vsMulti-small
     batchtime: 10080
-    expansions:
+  - name: ocsp-macos
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "4.4"
-      PYTHON_BINARY: C:/python/Python39/python.exe
-  - name: ocsp-win64-v8.0-python3.13
     tasks:
-      - name: .ocsp-rsa !.ocsp-staple
+      - name: .ocsp-rsa !.ocsp-staple .latest
-    display_name: OCSP Win64 v8.0 Python3.13
+      - name: .ocsp-rsa !.ocsp-staple .4.4
-    run_on:
+    display_name: OCSP macOS
-      - windows-64-vsMulti-small
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "8.0"
-      PYTHON_BINARY: C:/python/Python313/python.exe
-  - name: ocsp-macos-v4.4-python3.9
-    tasks:
-      - name: .ocsp-rsa !.ocsp-staple
-    display_name: OCSP macOS v4.4 Python3.9
     run_on:
       - macos-14
     batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "4.4"
-      PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
-  - name: ocsp-macos-v8.0-python3.13
-    tasks:
-      - name: .ocsp-rsa !.ocsp-staple
-    display_name: OCSP macOS v8.0 Python3.13
-    run_on:
-      - macos-14
-    batchtime: 10080
-    expansions:
-      AUTH: noauth
-      SSL: ssl
-      TOPOLOGY: server
-      VERSION: "8.0"
-      PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
 
   # Oidc auth tests
   - name: auth-oidc-ubuntu-22

.evergreen/scripts/generate_config.py

@@ -249,41 +249,22 @@ def generate_yaml(tasks=None, variants=None):
 
 def create_ocsp_variants() -> list[BuildVariant]:
     variants = []
-    batchtime = BATCHTIME_WEEK
+    # OCSP tests on default host with all servers v4.4+.
-    expansions = dict(AUTH="noauth", SSL="ssl", TOPOLOGY="server")
+    # MongoDB servers on Windows and MacOS do not staple OCSP responses and only support RSA.
-    base_display = "OCSP"
+    # Only test with MongoDB 4.4 and latest.
-
+    for host_name in ["rhel8", "win64", "macos"]:
-    # OCSP tests on default host with all servers v4.4+ and all python versions.
-    versions = get_versions_from("4.4")
-    for version, python in zip_cycle(versions, ALL_PYTHONS):
-        host = DEFAULT_HOST
-        variant = create_variant(
-            [".ocsp"],
-            get_variant_name(base_display, host, version=version, python=python),
-            python=python,
-            version=version,
-            host=host,
-            expansions=expansions,
-            batchtime=batchtime,
-        )
-        variants.append(variant)
-
-    # OCSP tests on Windows and MacOS.
-    # MongoDB servers on these hosts do not staple OCSP responses and only support RSA.
-    for host_name, version in product(["win64", "macos"], ["4.4", "8.0"]):
         host = HOSTS[host_name]
-        python = CPYTHONS[0] if version == "4.4" else CPYTHONS[-1]
+        if host == DEFAULT_HOST:
+            tasks = [".ocsp"]
+        else:
+            tasks = [".ocsp-rsa !.ocsp-staple .latest", ".ocsp-rsa !.ocsp-staple .4.4"]
         variant = create_variant(
-            [".ocsp-rsa !.ocsp-staple"],
+            tasks,
-            get_variant_name(base_display, host, version=version, python=python),
+            get_variant_name("OCSP", host),
-            python=python,
-            version=version,
             host=host,
-            expansions=expansions,
+            batchtime=BATCHTIME_WEEK,
-            batchtime=batchtime,
         )
         variants.append(variant)
-
     return variants
 
 
@@ -965,22 +946,34 @@ def create_mod_wsgi_tasks():
     return tasks
 
 
-def _create_ocsp_task(algo, variant, server_type, base_task_name):
+def _create_ocsp_tasks(algo, variant, server_type, base_task_name):
+    tasks = []
     file_name = f"{algo}-basic-tls-ocsp-{variant}.json"
 
-    vars = dict(TEST_NAME="ocsp", ORCHESTRATION_FILE=file_name)
+    for version in get_versions_from("4.4"):
-    server_func = FunctionCall(func="run server", vars=vars)
+        if version == "latest":
+            python = MIN_MAX_PYTHON[-1]
+        else:
+            python = MIN_MAX_PYTHON[0]
 
-    vars = dict(ORCHESTRATION_FILE=file_name, OCSP_SERVER_TYPE=server_type, TEST_NAME="ocsp")
+        vars = dict(
-    test_func = FunctionCall(func="run tests", vars=vars)
+            ORCHESTRATION_FILE=file_name,
+            OCSP_SERVER_TYPE=server_type,
+            TEST_NAME="ocsp",
+            PYTHON_VERSION=python,
+            VERSION=version,
+        )
+        test_func = FunctionCall(func="run tests", vars=vars)
 
-    tags = ["ocsp", f"ocsp-{algo}"]
+        tags = ["ocsp", f"ocsp-{algo}", version]
-    if "disableStapling" not in variant:
+        if "disableStapling" not in variant:
-        tags.append("ocsp-staple")
+            tags.append("ocsp-staple")
 
-    task_name = f"test-ocsp-{algo}-{base_task_name}"
+        task_name = get_task_name(
-    commands = [server_func, test_func]
+            f"test-ocsp-{algo}-{base_task_name}", python=python, version=version
-    return EvgTask(name=task_name, tags=tags, commands=commands)
+        )
+        tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func]))
+    return tasks
 
 
 def create_aws_lambda_tasks():
@@ -1092,8 +1085,8 @@ def create_ocsp_tasks():
     ]
     for algo in ["ecdsa", "rsa"]:
         for variant, server_type, base_task_name in tests:
-            task = _create_ocsp_task(algo, variant, server_type, base_task_name)
+            new_tasks = _create_ocsp_tasks(algo, variant, server_type, base_task_name)
-            tasks.append(task)
+            tasks.extend(new_tasks)
 
     return tasks
 
@@ -1182,7 +1175,7 @@ def write_tasks_to_file():
         fid.write("tasks:\n")
 
     for name, func in sorted(getmembers(mod, isfunction)):
-        if not name.endswith("_tasks"):
+        if name.startswith("_") or not name.endswith("_tasks"):
             continue
         if not name.startswith("create_"):
             raise ValueError("Task creators must start with create_")

.evergreen/scripts/run_server.py

@@ -28,16 +28,6 @@ def start_server():
     elif test_name == "load_balancer":
         set_env("LOAD_BALANCER")
 
-    elif test_name == "ocsp":
-        opts.ssl = True
-        if "ORCHESTRATION_FILE" not in os.environ:
-            found = False
-            for opt in extra_opts:
-                if opt.startswith("--orchestration-file"):
-                    found = True
-            if not found:
-                raise ValueError("Please provide an orchestration file")
-
     elif test_name == "search_index":
         os.environ["TOPOLOGY"] = "replica_set"
         os.environ["MONGODB_VERSION"] = "7.0"

.evergreen/scripts/setup-dev-env.sh

@@ -19,28 +19,20 @@ fi
 # Ensure dependencies are installed.
 bash $HERE/install-dependencies.sh
 
-# Set the location of the python bin dir.
+# Get the appropriate UV_PYTHON.
-if [ "Windows_NT" = "${OS:-}" ]; then
+. $ROOT/.evergreen/utils.sh
-  BIN_DIR=.venv/Scripts
+set -x
-else
-  BIN_DIR=.venv/bin
-fi
 
-# Ensure there is a python venv.
+if [ -z "${PYTHON_BINARY:-}" ]; then
-if [ ! -d $BIN_DIR ]; then
+    if [ -n "${PYTHON_VERSION:-}" ]; then
-  . $ROOT/.evergreen/utils.sh
+      PYTHON_BINARY=$(get_python_binary $PYTHON_VERSION)
-
+    else
-  if [ -z "${PYTHON_BINARY:-}" ]; then
+      PYTHON_BINARY=$(find_python3)
-      if [ -n "${PYTHON_VERSION:-}" ]; then
+    fi
-        PYTHON_BINARY=$(get_python_binary $PYTHON_VERSION)
-      else
-        PYTHON_BINARY=$(find_python3)
-      fi
-  fi
-  export UV_PYTHON=${PYTHON_BINARY}
-  echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
-  echo "Using python $UV_PYTHON"
 fi
+export UV_PYTHON=${PYTHON_BINARY}
+echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
+echo "Using python $UV_PYTHON"
 
 # Add the default install path to the path if needed.
 if [ -z "${PYMONGO_BIN_DIR:-}" ]; then

.evergreen/scripts/setup_tests.py

@@ -142,7 +142,6 @@ def handle_test_env() -> None:
     test_title = test_name
     if sub_test_name:
         test_title += f" {sub_test_name}"
-    LOGGER.info(f"Setting up '{test_title}' with {AUTH=} and {SSL=}...")
 
     # Create the test env file with the initial set of values.
     with ENV_FILE.open("w", newline="\n") as fid:
@@ -150,8 +149,6 @@ def handle_test_env() -> None:
         fid.write("set +x\n")
     ENV_FILE.chmod(ENV_FILE.stat().st_mode | stat.S_IEXEC)
 
-    write_env("AUTH", AUTH)
-    write_env("SSL", SSL)
     write_env("PIP_QUIET")  # Quiet by default.
     write_env("PIP_PREFER_BINARY")  # Prefer binary dists by default.
     write_env("UV_FROZEN")  # Do not modify lock files.
@@ -197,6 +194,13 @@ def handle_test_env() -> None:
     if test_name == "search_index":
         AUTH = "auth"
 
+    if test_name == "ocsp":
+        SSL = "ssl"
+
+    write_env("AUTH", AUTH)
+    write_env("SSL", SSL)
+    LOGGER.info(f"Setting up '{test_title}' with {AUTH=} and {SSL=}...")
+
     if test_name == "aws_lambda":
         UV_ARGS.append("--group pip")
         # Store AWS creds if they were given.
@@ -318,6 +322,22 @@ def handle_test_env() -> None:
             env["OCSP_ALGORITHM"] = ocsp_algo
             run_command(f"bash {DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh", env=env)
 
+        # The mock OCSP responder MUST BE started before the mongod as the mongod expects that
+        # a responder will be available upon startup.
+        version = os.environ.get("VERSION", "latest")
+        cmd = [
+            "bash",
+            f"{DRIVERS_TOOLS}/.evergreen/run-orchestration.sh",
+            "--ssl",
+            "--version",
+            version,
+        ]
+        if opts.verbose:
+            cmd.append("-v")
+        elif opts.quiet:
+            cmd.append("-q")
+        run_command(cmd, cwd=DRIVERS_TOOLS)
+
     if SSL != "nossl":
         if not DRIVERS_TOOLS:
             raise RuntimeError("Missing DRIVERS_TOOLS")

.evergreen/scripts/utils.py

@@ -54,8 +54,8 @@ SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
 
 EXTRA_TESTS = ["mod_wsgi", "aws_lambda"]
 
-# Tests that do not use run-orchestration.
+# Tests that do not use run-orchestration directly.
-NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"]
+NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless", "ocsp"]
 
 
 def get_test_options(

CONTRIBUTING.md

@@ -335,7 +335,9 @@ You must have `docker` or `podman` installed locally.
 - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`.
 This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`.
 MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA.
-- Run `just run-server ocsp`.
+NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server
+as part of `setup-tests`.
+
 - Run `just setup-tests ocsp <sub test>` (options are "valid", "revoked", "valid-delegate", "revoked-delegate").
 - Run `just run-tests`
 

test/ocsp/test_ocsp.py

@@ -19,6 +19,7 @@ import logging
 import os
 import sys
 import unittest
+from pathlib import Path
 
 import pytest
 
@@ -38,15 +39,10 @@ OCSP_TLS_SHOULD_SUCCEED = os.environ.get("OCSP_TLS_SHOULD_SUCCEED") == "true"
 FORMAT = "%(asctime)s %(levelname)s %(module)s %(message)s"
 logging.basicConfig(format=FORMAT, level=logging.DEBUG)
 
-if sys.platform == "win32":
-    # The non-stapled OCSP endpoint check is slow on Windows.
-    TIMEOUT_MS = 5000
-else:
-    TIMEOUT_MS = 500
-
 
 def _connect(options):
-    uri = f"mongodb://localhost:27017/?serverSelectionTimeoutMS={TIMEOUT_MS}&tlsCAFile={CA_FILE}&{options}"
+    assert CA_FILE is not None
+    uri = f"mongodb://localhost:27017/?serverSelectionTimeoutMS=10000&tlsCAFile={Path(CA_FILE).as_posix()}&{options}"
     print(uri)
     try:
         client = pymongo.MongoClient(uri)