Merge branch 'master' of github.com:mongodb/mongo-python-driver

2024-10-17 09:37:16 -05:00 · 2024-10-17 09:37:16 -05:00 · fd4049bfab
commit fd4049bfab
parent be9cbd1397 8ce21bc121
2 changed files with 371 additions and 104 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -2322,32 +2322,6 @@ axes:
        variables:
           COVERAGE: "coverage"

-  # Run encryption tests?
-  - id: encryption
-    display_name: "Encryption"
-    values:
-      - id: "encryption"
-        display_name: "Encryption"
-        tags: ["encryption_tag"]
-        variables:
-          test_encryption: true
-        batchtime: 10080  # 7 days
-      - id: "encryption_pyopenssl"
-        display_name: "Encryption PyOpenSSL"
-        tags: ["encryption_tag"]
-        variables:
-          test_encryption: true
-          test_encryption_pyopenssl: true
-        batchtime: 10080  # 7 days
-      # The path to crypt_shared is stored in the $CRYPT_SHARED_LIB_PATH expansion.
-      - id: "encryption_crypt_shared"
-        display_name: "Encryption shared lib"
-        tags: ["encryption_tag"]
-        variables:
-          test_encryption: true
-          test_crypt_shared: true
-        batchtime: 10080  # 7 days
-
  # Run pyopenssl tests?
  - id: pyopenssl
    display_name: "PyOpenSSL"
@ -2864,6 +2838,303 @@ buildvariants:
    PYTHON_BINARY: C:/python/32/Python39/python.exe
    SKIP_CSOT_TESTS: "true"

+# Encryption tests.
+- name: encryption-rhel8-py3.9-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption RHEL8 py3.9 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+  tags: [encryption_tag]
+- name: encryption-rhel8-py3.13-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption RHEL8 py3.13 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+  tags: [encryption_tag]
+- name: encryption-rhel8-pypy3.10-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption RHEL8 pypy3.10 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-rhel8-py3.9-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption crypt_shared RHEL8 py3.9 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-rhel8-py3.13-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption crypt_shared RHEL8 py3.13 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-rhel8-pypy3.10-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption crypt_shared RHEL8 pypy3.10 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
+  tags: [encryption_tag]
+- name: encryption-pyopenssl-rhel8-py3.9-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption PyOpenSSL RHEL8 py3.9 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_encryption_pyopenssl: "true"
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+  tags: [encryption_tag]
+- name: encryption-pyopenssl-rhel8-py3.13-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption PyOpenSSL RHEL8 py3.13 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_encryption_pyopenssl: "true"
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+  tags: [encryption_tag]
+- name: encryption-pyopenssl-rhel8-pypy3.10-auth-ssl
+  tasks:
+    - name: .standalone
+    - name: .replica_set
+    - name: .sharded_cluster
+  display_name: Encryption PyOpenSSL RHEL8 pypy3.10 Auth SSL
+  run_on:
+    - rhel87-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_encryption_pyopenssl: "true"
+    PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
+  tags: [encryption_tag]
+- name: encryption-rhel8-py3.10-auth-ssl
+  tasks:
+    - name: .replica_set
+  display_name: Encryption RHEL8 py3.10 Auth SSL
+  run_on:
+    - rhel87-small
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    PYTHON_BINARY: /opt/python/3.10/bin/python3
+- name: encryption-crypt_shared-rhel8-py3.11-auth-nossl
+  tasks:
+    - name: .replica_set
+  display_name: Encryption crypt_shared RHEL8 py3.11 Auth NoSSL
+  run_on:
+    - rhel87-small
+  expansions:
+    AUTH: auth
+    SSL: nossl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: /opt/python/3.11/bin/python3
+- name: encryption-pyopenssl-rhel8-py3.12-auth-ssl
+  tasks:
+    - name: .replica_set
+  display_name: Encryption PyOpenSSL RHEL8 py3.12 Auth SSL
+  run_on:
+    - rhel87-small
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    TEST_ENCRYPTION_PYOPENSSL: "true"
+    PYTHON_BINARY: /opt/python/3.12/bin/python3
+- name: encryption-rhel8-pypy3.9-auth-nossl
+  tasks:
+    - name: .replica_set
+  display_name: Encryption RHEL8 pypy3.9 Auth NoSSL
+  run_on:
+    - rhel87-small
+  expansions:
+    AUTH: auth
+    SSL: nossl
+    test_encryption: "true"
+    PYTHON_BINARY: /opt/python/pypy3.9/bin/python3
+- name: encryption-macos-py3.9-auth-ssl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption macOS py3.9 Auth SSL
+  run_on:
+    - macos-14
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
+  tags: [encryption_tag]
+- name: encryption-macos-py3.13-auth-nossl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption macOS py3.13 Auth NoSSL
+  run_on:
+    - macos-14
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: nossl
+    test_encryption: "true"
+    PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-macos-py3.9-auth-ssl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption crypt_shared macOS py3.9 Auth SSL
+  run_on:
+    - macos-14
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-macos-py3.13-auth-nossl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption crypt_shared macOS py3.13 Auth NoSSL
+  run_on:
+    - macos-14
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: nossl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
+  tags: [encryption_tag]
+- name: encryption-win64-py3.9-auth-ssl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption Win64 py3.9 Auth SSL
+  run_on:
+    - windows-64-vsMulti-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    PYTHON_BINARY: C:/python/Python39/python.exe
+  tags: [encryption_tag]
+- name: encryption-win64-py3.13-auth-nossl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption Win64 py3.13 Auth NoSSL
+  run_on:
+    - windows-64-vsMulti-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: nossl
+    test_encryption: "true"
+    PYTHON_BINARY: C:/python/Python313/python.exe
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-win64-py3.9-auth-ssl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption crypt_shared Win64 py3.9 Auth SSL
+  run_on:
+    - windows-64-vsMulti-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: ssl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: C:/python/Python39/python.exe
+  tags: [encryption_tag]
+- name: encryption-crypt_shared-win64-py3.13-auth-nossl
+  tasks:
+    - name: .latest .replica_set
+  display_name: Encryption crypt_shared Win64 py3.13 Auth NoSSL
+  run_on:
+    - windows-64-vsMulti-small
+  batchtime: 10080
+  expansions:
+    AUTH: auth
+    SSL: nossl
+    test_encryption: "true"
+    test_crypt_shared: "true"
+    PYTHON_BINARY: C:/python/Python313/python.exe
+  tags: [encryption_tag]
+
 - matrix_name: "tests-fips"
  matrix_spec:
    platform:
@ -2874,33 +3145,6 @@ buildvariants:
  tasks:
    - "test-fips-standalone"

- matrix_name: "test-macos-encryption"
-  matrix_spec:
-    platform:
-      - macos
-    auth: "auth"
-    ssl: "nossl"
-    encryption: "*"
-  display_name: "${encryption} ${platform} ${auth} ${ssl}"
-  tasks: "test-latest-replica_set"
-  rules:
-    - if:
-        encryption: ["encryption", "encryption_crypt_shared"]
-        platform: macos
-        auth: "auth"
-        ssl: "nossl"
-      then:
-        add_tasks: &encryption-server-versions
-          - ".rapid"
-          - ".latest"
-          - ".8.0"
-          - ".7.0"
-          - ".6.0"
-          - ".5.0"
-          - ".4.4"
-          - ".4.2"
-          - ".4.0"
-
 # Test one server version with zSeries, POWER8, and ARM.
 - matrix_name: "test-different-cpu-architectures"
  matrix_spec:
@ -2954,26 +3198,6 @@ buildvariants:
  tasks:
    - '.replica_set'

- matrix_name: "tests-python-version-rhel8-test-encryption"
-  matrix_spec:
-    platform: rhel8
-    python-version: "*"
-    auth-ssl: noauth-nossl
-# TODO: dependency error for 'coverage-report' task:
-# dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config
-#    coverage: "*"
-    encryption: "*"
-  display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}"
-  tasks: "test-latest-replica_set"
-  rules:
-    - if:
-        encryption: ["encryption", "encryption_crypt_shared"]
-        platform: rhel8
-        auth-ssl: noauth-nossl
-        python-version: "*"
-      then:
-        add_tasks: *encryption-server-versions
-
 - matrix_name: "tests-python-version-rhel8-without-c-extensions"
  matrix_spec:
    platform: rhel8
@ -3057,23 +3281,6 @@ buildvariants:
  tasks:
     - ".5.0"

- matrix_name: "tests-windows-encryption"
-  matrix_spec:
-    platform: windows
-    python-version-windows: "*"
-    auth-ssl: "*"
-    encryption: "*"
-  display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}"
-  tasks: "test-latest-replica_set"
-  rules:
-    - if:
-        encryption: ["encryption", "encryption_crypt_shared"]
-        platform: windows
-        python-version-windows: "*"
-        auth-ssl: "*"
-      then:
-        add_tasks: *encryption-server-versions
-
 # Storage engine tests on RHEL 8.4 (x86_64) with Python 3.9.
 - matrix_name: "tests-storage-engines"
  matrix_spec:
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@ -45,18 +45,13 @@ class Host:
    name: str
    run_on: str
    display_name: str
-    expansions: dict[str, str]


-_macos_expansions = dict(  # CSOT tests are unreliable on slow hosts.
-    SKIP_CSOT_TESTS="true"
-)
-
-HOSTS["rhel8"] = Host("rhel8", "rhel87-small", "RHEL8", dict())
-HOSTS["win64"] = Host("win64", "windows-64-vsMulti-small", "Win64", _macos_expansions)
-HOSTS["win32"] = Host("win32", "windows-64-vsMulti-small", "Win32", _macos_expansions)
-HOSTS["macos"] = Host("macos", "macos-14", "macOS", _macos_expansions)
-HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64", _macos_expansions)
+HOSTS["rhel8"] = Host("rhel8", "rhel87-small", "RHEL8")
+HOSTS["win64"] = Host("win64", "windows-64-vsMulti-small", "Win64")
+HOSTS["win32"] = Host("win32", "windows-64-vsMulti-small", "Win32")
+HOSTS["macos"] = Host("macos", "macos-14", "macOS")
+HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64")


 ##############
@ -84,7 +79,6 @@ def create_variant(
        expansions["PYTHON_BINARY"] = get_python_binary(python, host)
    if version:
        expansions["VERSION"] = version
-    expansions.update(HOSTS[host].expansions)
    expansions = expansions or None
    return BuildVariant(
        name=name,
@ -129,7 +123,7 @@ def get_display_name(base: str, host: str, **kwargs) -> str:
        elif key.lower() in DISPLAY_LOOKUP:
            name = DISPLAY_LOOKUP[key.lower()][value]
        else:
-            raise ValueError(f"Missing display handling for {key}")
+            continue
        display_name = f"{display_name} {name}"
    return display_name

@ -235,7 +229,7 @@ def create_server_variants() -> list[BuildVariant]:
            zip_cycle(MIN_MAX_PYTHON, AUTH_SSLS, TOPOLOGIES), SYNCS
        ):
            test_suite = "default" if sync == "sync" else "default_async"
-            expansions = dict(AUTH=auth, SSL=ssl, TEST_SUITES=test_suite)
+            expansions = dict(AUTH=auth, SSL=ssl, TEST_SUITES=test_suite, SKIP_CSOT_TESTS="true")
            display_name = get_display_name("Test", host, python=python, **expansions)
            variant = create_variant(
                [f".{topology}"],
@ -249,8 +243,74 @@ def create_server_variants() -> list[BuildVariant]:
    return variants


+def create_encryption_variants() -> list[BuildVariant]:
+    variants = []
+    tags = ["encryption_tag"]
+    batchtime = BATCHTIME_WEEK
+
+    def get_encryption_expansions(encryption, ssl="ssl"):
+        expansions = dict(AUTH="auth", SSL=ssl, test_encryption="true")
+        if "crypt_shared" in encryption:
+            expansions["test_crypt_shared"] = "true"
+        if "PyOpenSSL" in encryption:
+            expansions["test_encryption_pyopenssl"] = "true"
+        return expansions
+
+    host = "rhel8"
+
+    # Test against all server versions and topolgies for the three main python versions.
+    encryptions = ["Encryption", "Encryption crypt_shared", "Encryption PyOpenSSL"]
+    for encryption, python in product(encryptions, [*MIN_MAX_PYTHON, PYPYS[-1]]):
+        expansions = get_encryption_expansions(encryption)
+        display_name = get_display_name(encryption, host, python=python, **expansions)
+        variant = create_variant(
+            [f".{t}" for t in TOPOLOGIES],
+            display_name,
+            python=python,
+            host=host,
+            expansions=expansions,
+            batchtime=batchtime,
+            tags=tags,
+        )
+        variants.append(variant)
+
+    # Test the rest of the pythons on linux for all server versions.
+    for encryption, python, ssl in zip_cycle(
+        encryptions, CPYTHONS[1:-1] + PYPYS[:-1], ["ssl", "nossl"]
+    ):
+        expansions = get_encryption_expansions(encryption, ssl)
+        display_name = get_display_name(encryption, host, python=python, **expansions)
+        variant = create_variant(
+            [".replica_set"],
+            display_name,
+            python=python,
+            host=host,
+            expansions=expansions,
+        )
+        variants.append(variant)
+
+    # Test on macos and linux on one server version and topology for min and max python.
+    encryptions = ["Encryption", "Encryption crypt_shared"]
+    task_names = [".latest .replica_set"]
+    for host, encryption, python in product(["macos", "win64"], encryptions, MIN_MAX_PYTHON):
+        ssl = "ssl" if python == CPYTHONS[0] else "nossl"
+        expansions = get_encryption_expansions(encryption, ssl)
+        display_name = get_display_name(encryption, host, python=python, **expansions)
+        variant = create_variant(
+            task_names,
+            display_name,
+            python=python,
+            host=host,
+            expansions=expansions,
+            batchtime=batchtime,
+            tags=tags,
+        )
+        variants.append(variant)
+    return variants
+
+
 ##################
 # Generate Config
 ##################

-generate_yaml(variants=create_server_variants())
+generate_yaml(variants=create_encryption_variants())