From 6a7e83dc95319c445b95ab1f54f4aa8435986cc4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 18 Oct 2024 10:36:05 -0500 Subject: [PATCH 1/4] PYTHON-4887 Do not test macos arm64 on server versions < 6.0 (#1947) --- .evergreen/config.yml | 48 ++++++++++++++++++++------- .evergreen/scripts/generate_config.py | 9 +++-- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9e2ab7708..705880be6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2592,7 +2592,11 @@ buildvariants: # Server tests for macOS Arm64. - name: test-macos-arm64-py3.9-auth-ssl-sync tasks: - - name: .standalone + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest display_name: Test macOS Arm64 py3.9 Auth SSL Sync run_on: - macos-14-arm64 @@ -2600,11 +2604,15 @@ buildvariants: AUTH: auth SSL: ssl TEST_SUITES: default - PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 - name: test-macos-arm64-py3.9-auth-ssl-async tasks: - - name: .standalone + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest display_name: Test macOS Arm64 py3.9 Auth SSL Async run_on: - macos-14-arm64 @@ -2612,11 +2620,15 @@ buildvariants: AUTH: auth SSL: ssl TEST_SUITES: default_async - PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 - name: test-macos-arm64-py3.13-noauth-ssl-sync tasks: - - name: .replica_set + - name: .replica_set .6.0 + - name: .replica_set .7.0 + - name: .replica_set .8.0 + - name: .replica_set .rapid + - name: .replica_set .latest display_name: Test macOS Arm64 py3.13 NoAuth SSL Sync run_on: - macos-14-arm64 @@ -2624,11 +2636,15 @@ buildvariants: AUTH: noauth SSL: ssl TEST_SUITES: default - PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 - name: test-macos-arm64-py3.13-noauth-ssl-async tasks: - - name: .replica_set + - name: .replica_set .6.0 + - name: .replica_set .7.0 + - name: .replica_set .8.0 + - name: .replica_set .rapid + - name: .replica_set .latest display_name: Test macOS Arm64 py3.13 NoAuth SSL Async run_on: - macos-14-arm64 @@ -2636,11 +2652,15 @@ buildvariants: AUTH: noauth SSL: ssl TEST_SUITES: default_async - PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 - name: test-macos-arm64-py3.9-noauth-nossl-sync tasks: - - name: .sharded_cluster + - name: .sharded_cluster .6.0 + - name: .sharded_cluster .7.0 + - name: .sharded_cluster .8.0 + - name: .sharded_cluster .rapid + - name: .sharded_cluster .latest display_name: Test macOS Arm64 py3.9 NoAuth NoSSL Sync run_on: - macos-14-arm64 @@ -2648,11 +2668,15 @@ buildvariants: AUTH: noauth SSL: nossl TEST_SUITES: default - PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 - name: test-macos-arm64-py3.9-noauth-nossl-async tasks: - - name: .sharded_cluster + - name: .sharded_cluster .6.0 + - name: .sharded_cluster .7.0 + - name: .sharded_cluster .8.0 + - name: .sharded_cluster .rapid + - name: .sharded_cluster .latest display_name: Test macOS Arm64 py3.9 NoAuth NoSSL Async run_on: - macos-14-arm64 @@ -2660,8 +2684,8 @@ buildvariants: AUTH: noauth SSL: nossl TEST_SUITES: default_async - PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 # Server tests for Windows. - name: test-win64-py3.9-auth-ssl-sync diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index c3bfeef7a..a3ec798c3 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -23,6 +23,7 @@ from shrub.v3.shrub_service import ShrubService ############## ALL_VERSIONS = ["4.0", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"] +VERSIONS_6_0_PLUS = ["6.0", "7.0", "8.0", "rapid", "latest"] CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] PYPYS = ["pypy3.9", "pypy3.10"] ALL_PYTHONS = CPYTHONS + PYPYS @@ -239,10 +240,14 @@ def create_server_variants() -> list[BuildVariant]: zip_cycle(MIN_MAX_PYTHON, AUTH_SSLS, TOPOLOGIES), SYNCS ): test_suite = "default" if sync == "sync" else "default_async" + tasks = [f".{topology}"] + # MacOS arm64 only works on server versions 6.0+ + if host == "macos-arm64": + tasks = [f".{topology} .{version}" for version in VERSIONS_6_0_PLUS] expansions = dict(AUTH=auth, SSL=ssl, TEST_SUITES=test_suite, SKIP_CSOT_TESTS="true") display_name = get_display_name("Test", host, python=python, **expansions) variant = create_variant( - [f".{topology}"], + tasks, display_name, python=python, host=host, @@ -409,6 +414,6 @@ def create_enterprise_auth_variants(): # Generate Config ################## -variants = create_enterprise_auth_variants() +variants = create_server_variants() # print(len(variants)) generate_yaml(variants=variants) From 1ae0c3904c9a71bed1f7d8f81183b59070845a81 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 18 Oct 2024 10:58:28 -0500 Subject: [PATCH 2/4] PYTHON-4886 Use shrub.py for PyOpenSSL tests (#1946) --- .evergreen/config.yml | 144 +++++++++++++++++--------- .evergreen/scripts/generate_config.py | 34 +++++- 2 files changed, 126 insertions(+), 52 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 705880be6..9083da145 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2305,16 +2305,6 @@ axes: variables: COVERAGE: "coverage" - # Run pyopenssl tests? - - id: pyopenssl - display_name: "PyOpenSSL" - values: - - id: "enabled" - display_name: "PyOpenSSL" - variables: - test_pyopenssl: true - batchtime: 10080 # 7 days - - id: versionedApi display_name: "versionedApi" values: @@ -3283,6 +3273,99 @@ buildvariants: AUTH: auth PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 +# PyOpenSSL tests. +- name: pyopenssl-macos-py3.9 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL macOS py3.9 + run_on: + - macos-14 + batchtime: 10080 + expansions: + AUTH: noauth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 +- name: pyopenssl-rhel8-py3.10 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 py3.10 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: pyopenssl-rhel8-py3.11 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 py3.11 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: pyopenssl-rhel8-py3.12 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 py3.12 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: pyopenssl-win64-py3.13 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL Win64 py3.13 + run_on: + - windows-64-vsMulti-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: C:/python/Python313/python.exe +- name: pyopenssl-rhel8-pypy3.9 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 pypy3.9 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: pyopenssl-rhel8-pypy3.10 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 pypy3.10 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + - matrix_name: "tests-fips" matrix_spec: platform: @@ -3305,47 +3388,6 @@ buildvariants: tasks: - ".6.0" -- matrix_name: "tests-pyopenssl" - matrix_spec: - platform: rhel8 - python-version: "*" - auth: "*" - ssl: "ssl" - pyopenssl: "*" - # Only test "noauth" with Python 3.9. - exclude_spec: - platform: rhel8 - python-version: ["3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"] - auth: "noauth" - ssl: "ssl" - pyopenssl: "*" - display_name: "PyOpenSSL ${platform} ${python-version} ${auth}" - tasks: - - '.replica_set' - # Test standalone and sharded only on 7.0. - - '.7.0' - -- matrix_name: "tests-pyopenssl-macOS" - matrix_spec: - platform: macos - auth: "auth" - ssl: "ssl" - pyopenssl: "*" - display_name: "PyOpenSSL ${platform} ${auth}" - tasks: - - '.replica_set' - -- matrix_name: "tests-pyopenssl-windows" - matrix_spec: - platform: windows - python-version-windows: "*" - auth: "auth" - ssl: "ssl" - pyopenssl: "*" - display_name: "PyOpenSSL ${platform} ${python-version-windows} ${auth}" - tasks: - - '.replica_set' - - matrix_name: "tests-python-version-rhel8-without-c-extensions" matrix_spec: platform: rhel8 diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index a3ec798c3..6d614a9af 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -410,10 +410,42 @@ def create_enterprise_auth_variants(): return variants +def create_pyopenssl_variants(): + base_name = "PyOpenSSL" + batchtime = BATCHTIME_WEEK + base_expansions = dict(test_pyopenssl="true", SSL="ssl") + variants = [] + + for python in ALL_PYTHONS: + # Only test "noauth" with min python. + auth = "noauth" if python == CPYTHONS[0] else "auth" + if python == CPYTHONS[0]: + host = "macos" + elif python == CPYTHONS[-1]: + host = "win64" + else: + host = "rhel8" + expansions = dict(AUTH=auth) + expansions.update(base_expansions) + + display_name = get_display_name(base_name, host, python=python) + variant = create_variant( + [".replica_set", ".7.0"], + display_name, + python=python, + host=host, + expansions=expansions, + batchtime=batchtime, + ) + variants.append(variant) + + return variants + + ################## # Generate Config ################## -variants = create_server_variants() +variants = create_pyopenssl_variants() # print(len(variants)) generate_yaml(variants=variants) From a1ade45dd3b67a6e4baa50404b9807f68062f043 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 18 Oct 2024 13:32:09 -0400 Subject: [PATCH 3/4] PYTHON-4881 - Use OvertCommandListener wherever sensitive events are not needed (#1943) Co-authored-by: Steven Silvester --- test/asynchronous/test_change_stream.py | 5 +++-- test/asynchronous/test_collation.py | 4 ++-- test/asynchronous/test_collection.py | 3 ++- test/asynchronous/test_cursor.py | 4 ++-- test/asynchronous/test_grid_file.py | 4 ++-- test/asynchronous/test_monitoring.py | 21 ++++++++++++--------- test/asynchronous/test_session.py | 3 ++- test/auth_oidc/test_auth_oidc.py | 6 +++--- test/test_change_stream.py | 5 +++-- test/test_collation.py | 4 ++-- test/test_collection.py | 3 ++- test/test_cursor.py | 4 ++-- test/test_grid_file.py | 4 ++-- test/test_index_management.py | 4 ++-- test/test_monitoring.py | 21 ++++++++++++--------- test/test_read_write_concern_spec.py | 6 +++--- test/test_server_selection.py | 3 ++- test/test_session.py | 3 ++- test/test_ssl.py | 1 + 19 files changed, 61 insertions(+), 47 deletions(-) diff --git a/test/asynchronous/test_change_stream.py b/test/asynchronous/test_change_stream.py index 883ed72c4..98641f46e 100644 --- a/test/asynchronous/test_change_stream.py +++ b/test/asynchronous/test_change_stream.py @@ -39,6 +39,7 @@ from test.unified_format import generate_test_classes from test.utils import ( AllowListEventListener, EventListener, + OvertCommandListener, async_wait_until, ) @@ -179,7 +180,7 @@ class APITestsMixin: @no_type_check async def test_try_next_runs_one_getmore(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. await client.admin.command("ping") @@ -237,7 +238,7 @@ class APITestsMixin: @no_type_check async def test_batch_size_is_honored(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. await client.admin.command("ping") diff --git a/test/asynchronous/test_collation.py b/test/asynchronous/test_collation.py index be3ea22e4..d95f4c991 100644 --- a/test/asynchronous/test_collation.py +++ b/test/asynchronous/test_collation.py @@ -18,7 +18,7 @@ from __future__ import annotations import functools import warnings from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest -from test.utils import EventListener +from test.utils import EventListener, OvertCommandListener from typing import Any from pymongo.asynchronous.helpers import anext @@ -101,7 +101,7 @@ class TestCollation(AsyncIntegrationTest): @async_client_context.require_connection async def _setup_class(cls): await super()._setup_class() - cls.listener = EventListener() + cls.listener = OvertCommandListener() cls.client = await cls.unmanaged_async_rs_or_single_client(event_listeners=[cls.listener]) cls.db = cls.client.pymongo_test cls.collation = Collation("en_US") diff --git a/test/asynchronous/test_collection.py b/test/asynchronous/test_collection.py index 612090b69..db52bad4a 100644 --- a/test/asynchronous/test_collection.py +++ b/test/asynchronous/test_collection.py @@ -36,6 +36,7 @@ from test.asynchronous import ( # TODO: fix sync imports in PYTHON-4528 from test.utils import ( IMPOSSIBLE_WRITE_CONCERN, EventListener, + OvertCommandListener, async_get_pool, async_is_mongos, async_wait_until, @@ -2116,7 +2117,7 @@ class AsyncTestCollection(AsyncIntegrationTest): self.assertEqual(4, (await c.find_one_and_update({}, {"$inc": {"i": 1}}, sort=sort))["j"]) async def test_find_one_and_write_concern(self): - listener = EventListener() + listener = OvertCommandListener() db = (await self.async_single_client(event_listeners=[listener]))[self.db.name] # non-default WriteConcern. c_w0 = db.get_collection("test", write_concern=WriteConcern(w=0)) diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index ee0a757ed..787da3d95 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -1601,7 +1601,7 @@ class TestRawBatchCursor(AsyncIntegrationTest): await anext(c.find_raw_batches()) async def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test await c.drop() @@ -1768,7 +1768,7 @@ class TestRawBatchCommandCursor(AsyncIntegrationTest): await anext(await self.db.test.aggregate_raw_batches([], collation=Collation("en_US"))) async def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test await c.drop() diff --git a/test/asynchronous/test_grid_file.py b/test/asynchronous/test_grid_file.py index 9c57c15c5..54fcd3abf 100644 --- a/test/asynchronous/test_grid_file.py +++ b/test/asynchronous/test_grid_file.py @@ -33,7 +33,7 @@ from pymongo.asynchronous.database import AsyncDatabase sys.path[0:0] = [""] -from test.utils import EventListener +from test.utils import OvertCommandListener from bson.objectid import ObjectId from gridfs.asynchronous.grid_file import ( @@ -810,7 +810,7 @@ Bye""" # Use 102 batches to cause a single getMore. chunk_size = 1024 data = b"d" * (102 * chunk_size) - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test async with AsyncGridIn(db.fs, chunk_size=chunk_size) as infile: diff --git a/test/asynchronous/test_monitoring.py b/test/asynchronous/test_monitoring.py index b5d8708dc..b0c86ab54 100644 --- a/test/asynchronous/test_monitoring.py +++ b/test/asynchronous/test_monitoring.py @@ -31,6 +31,7 @@ from test.asynchronous import ( ) from test.utils import ( EventListener, + OvertCommandListener, async_wait_until, ) @@ -54,7 +55,7 @@ class AsyncTestCommandMonitoring(AsyncIntegrationTest): @async_client_context.require_connection async def _setup_class(cls): await super()._setup_class() - cls.listener = EventListener() + cls.listener = OvertCommandListener() cls.client = await cls.unmanaged_async_rs_or_single_client( event_listeners=[cls.listener], retryWrites=False ) @@ -1100,11 +1101,13 @@ class AsyncTestCommandMonitoring(AsyncIntegrationTest): @async_client_context.require_version_max(6, 1, 99) async def test_sensitive_commands(self): - listeners = self.client._event_listeners + listener = EventListener() + client = await self.async_rs_or_single_client(event_listeners=[listener]) + listeners = client._event_listeners - self.listener.reset() + listener.reset() cmd = SON([("getnonce", 1)]) - listeners.publish_command_start(cmd, "pymongo_test", 12345, await self.client.address, None) # type: ignore[arg-type] + listeners.publish_command_start(cmd, "pymongo_test", 12345, await client.address, None) # type: ignore[arg-type] delta = datetime.timedelta(milliseconds=100) listeners.publish_command_success( delta, @@ -1115,15 +1118,15 @@ class AsyncTestCommandMonitoring(AsyncIntegrationTest): None, database_name="pymongo_test", ) - started = self.listener.started_events[0] - succeeded = self.listener.succeeded_events[0] - self.assertEqual(0, len(self.listener.failed_events)) + started = listener.started_events[0] + succeeded = listener.succeeded_events[0] + self.assertEqual(0, len(listener.failed_events)) self.assertIsInstance(started, monitoring.CommandStartedEvent) self.assertEqual({}, started.command) self.assertEqual("pymongo_test", started.database_name) self.assertEqual("getnonce", started.command_name) self.assertIsInstance(started.request_id, int) - self.assertEqual(await self.client.address, started.connection_id) + self.assertEqual(await client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertEqual(succeeded.duration_micros, 100000) self.assertEqual(started.command_name, succeeded.command_name) @@ -1140,7 +1143,7 @@ class AsyncTestGlobalListener(AsyncIntegrationTest): @async_client_context.require_connection async def _setup_class(cls): await super()._setup_class() - cls.listener = EventListener() + cls.listener = OvertCommandListener() # We plan to call register(), which internally modifies _LISTENERS. cls.saved_listeners = copy.deepcopy(monitoring._LISTENERS) monitoring.register(cls.listener) diff --git a/test/asynchronous/test_session.py b/test/asynchronous/test_session.py index d264b5ecb..b43262179 100644 --- a/test/asynchronous/test_session.py +++ b/test/asynchronous/test_session.py @@ -36,6 +36,7 @@ from test.asynchronous import ( from test.utils import ( EventListener, ExceptionCatchingThread, + OvertCommandListener, async_wait_until, wait_until, ) @@ -199,7 +200,7 @@ class TestSession(AsyncIntegrationTest): lsid_set = set() failures = 0 for _ in range(5): - listener = EventListener() + listener = OvertCommandListener() client = self.async_rs_or_single_client(event_listeners=[listener], maxPoolSize=1) cursor = client.db.test.find({}) ops: List[Tuple[Callable, List[Any]]] = [ diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index 6d31f3db4..6526391da 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -31,7 +31,7 @@ import pytest sys.path[0:0] = [""] from test.unified_format import generate_test_classes -from test.utils import EventListener +from test.utils import EventListener, OvertCommandListener from bson import SON from pymongo import MongoClient @@ -348,7 +348,7 @@ class TestAuthOIDCHuman(OIDCTestBase): # Create a default OIDC client and add an event listener. # The following assumes that the driver does not emit saslStart or saslContinue events. # If the driver does emit those events, ignore/filter them for the purposes of this test. - listener = EventListener() + listener = OvertCommandListener() client = self.create_client(event_listeners=[listener]) # Perform a find operation that succeeds. @@ -1021,7 +1021,7 @@ class TestAuthOIDCMachine(OIDCTestBase): def test_4_4_speculative_authentication_should_be_ignored_on_reauthentication(self): # Create an OIDC configured client that can listen for `SaslStart` commands. - listener = EventListener() + listener = OvertCommandListener() client = self.create_client(event_listeners=[listener]) # Preload the *Client Cache* with a valid access token to enforce Speculative Authentication. diff --git a/test/test_change_stream.py b/test/test_change_stream.py index dae224c5e..3a107122b 100644 --- a/test/test_change_stream.py +++ b/test/test_change_stream.py @@ -39,6 +39,7 @@ from test.unified_format import generate_test_classes from test.utils import ( AllowListEventListener, EventListener, + OvertCommandListener, wait_until, ) @@ -177,7 +178,7 @@ class APITestsMixin: @no_type_check def test_try_next_runs_one_getmore(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. client.admin.command("ping") @@ -235,7 +236,7 @@ class APITestsMixin: @no_type_check def test_batch_size_is_honored(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. client.admin.command("ping") diff --git a/test/test_collation.py b/test/test_collation.py index e5c1c7eb1..b878df2fb 100644 --- a/test/test_collation.py +++ b/test/test_collation.py @@ -18,7 +18,7 @@ from __future__ import annotations import functools import warnings from test import IntegrationTest, client_context, unittest -from test.utils import EventListener +from test.utils import EventListener, OvertCommandListener from typing import Any from pymongo.collation import ( @@ -101,7 +101,7 @@ class TestCollation(IntegrationTest): @client_context.require_connection def _setup_class(cls): super()._setup_class() - cls.listener = EventListener() + cls.listener = OvertCommandListener() cls.client = cls.unmanaged_rs_or_single_client(event_listeners=[cls.listener]) cls.db = cls.client.pymongo_test cls.collation = Collation("en_US") diff --git a/test/test_collection.py b/test/test_collection.py index a2c3b0b0b..84a900d45 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -36,6 +36,7 @@ from test import ( # TODO: fix sync imports in PYTHON-4528 from test.utils import ( IMPOSSIBLE_WRITE_CONCERN, EventListener, + OvertCommandListener, get_pool, is_mongos, wait_until, @@ -2093,7 +2094,7 @@ class TestCollection(IntegrationTest): self.assertEqual(4, (c.find_one_and_update({}, {"$inc": {"i": 1}}, sort=sort))["j"]) def test_find_one_and_write_concern(self): - listener = EventListener() + listener = OvertCommandListener() db = (self.single_client(event_listeners=[listener]))[self.db.name] # non-default WriteConcern. c_w0 = db.get_collection("test", write_concern=WriteConcern(w=0)) diff --git a/test/test_cursor.py b/test/test_cursor.py index 7a6dfc942..9eac0f1c4 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -1590,7 +1590,7 @@ class TestRawBatchCursor(IntegrationTest): next(c.find_raw_batches()) def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test c.drop() @@ -1757,7 +1757,7 @@ class TestRawBatchCommandCursor(IntegrationTest): next(self.db.test.aggregate_raw_batches([], collation=Collation("en_US"))) def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test c.drop() diff --git a/test/test_grid_file.py b/test/test_grid_file.py index fe88aec5f..c35efccef 100644 --- a/test/test_grid_file.py +++ b/test/test_grid_file.py @@ -33,7 +33,7 @@ from pymongo.synchronous.database import Database sys.path[0:0] = [""] -from test.utils import EventListener +from test.utils import OvertCommandListener from bson.objectid import ObjectId from gridfs.errors import NoFile @@ -808,7 +808,7 @@ Bye""" # Use 102 batches to cause a single getMore. chunk_size = 1024 data = b"d" * (102 * chunk_size) - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test with GridIn(db.fs, chunk_size=chunk_size) as infile: diff --git a/test/test_index_management.py b/test/test_index_management.py index ec1e36373..6ca726e2e 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -27,7 +27,7 @@ sys.path[0:0] = [""] from test import IntegrationTest, PyMongoTestCase, unittest from test.unified_format import generate_test_classes -from test.utils import AllowListEventListener, EventListener +from test.utils import AllowListEventListener, EventListener, OvertCommandListener from pymongo.errors import OperationFailure from pymongo.operations import SearchIndexModel @@ -88,7 +88,7 @@ class SearchIndexIntegrationBase(PyMongoTestCase): url = os.environ.get("MONGODB_URI") username = os.environ["DB_USER"] password = os.environ["DB_PASSWORD"] - cls.listener = listener = EventListener() + cls.listener = listener = OvertCommandListener() cls.client = cls.unmanaged_simple_client( url, username=username, password=password, event_listeners=[listener] ) diff --git a/test/test_monitoring.py b/test/test_monitoring.py index a0c520ed2..75fe5c987 100644 --- a/test/test_monitoring.py +++ b/test/test_monitoring.py @@ -31,6 +31,7 @@ from test import ( ) from test.utils import ( EventListener, + OvertCommandListener, wait_until, ) @@ -54,7 +55,7 @@ class TestCommandMonitoring(IntegrationTest): @client_context.require_connection def _setup_class(cls): super()._setup_class() - cls.listener = EventListener() + cls.listener = OvertCommandListener() cls.client = cls.unmanaged_rs_or_single_client( event_listeners=[cls.listener], retryWrites=False ) @@ -1100,11 +1101,13 @@ class TestCommandMonitoring(IntegrationTest): @client_context.require_version_max(6, 1, 99) def test_sensitive_commands(self): - listeners = self.client._event_listeners + listener = EventListener() + client = self.rs_or_single_client(event_listeners=[listener]) + listeners = client._event_listeners - self.listener.reset() + listener.reset() cmd = SON([("getnonce", 1)]) - listeners.publish_command_start(cmd, "pymongo_test", 12345, self.client.address, None) # type: ignore[arg-type] + listeners.publish_command_start(cmd, "pymongo_test", 12345, client.address, None) # type: ignore[arg-type] delta = datetime.timedelta(milliseconds=100) listeners.publish_command_success( delta, @@ -1115,15 +1118,15 @@ class TestCommandMonitoring(IntegrationTest): None, database_name="pymongo_test", ) - started = self.listener.started_events[0] - succeeded = self.listener.succeeded_events[0] - self.assertEqual(0, len(self.listener.failed_events)) + started = listener.started_events[0] + succeeded = listener.succeeded_events[0] + self.assertEqual(0, len(listener.failed_events)) self.assertIsInstance(started, monitoring.CommandStartedEvent) self.assertEqual({}, started.command) self.assertEqual("pymongo_test", started.database_name) self.assertEqual("getnonce", started.command_name) self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) + self.assertEqual(client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertEqual(succeeded.duration_micros, 100000) self.assertEqual(started.command_name, succeeded.command_name) @@ -1140,7 +1143,7 @@ class TestGlobalListener(IntegrationTest): @client_context.require_connection def _setup_class(cls): super()._setup_class() - cls.listener = EventListener() + cls.listener = OvertCommandListener() # We plan to call register(), which internally modifies _LISTENERS. cls.saved_listeners = copy.deepcopy(monitoring._LISTENERS) monitoring.register(cls.listener) diff --git a/test/test_read_write_concern_spec.py b/test/test_read_write_concern_spec.py index 67943d495..db53b67ae 100644 --- a/test/test_read_write_concern_spec.py +++ b/test/test_read_write_concern_spec.py @@ -24,7 +24,7 @@ sys.path[0:0] = [""] from test import IntegrationTest, client_context, unittest from test.unified_format import generate_test_classes -from test.utils import EventListener +from test.utils import OvertCommandListener from pymongo import DESCENDING from pymongo.errors import ( @@ -44,7 +44,7 @@ _TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "read_wri class TestReadWriteConcernSpec(IntegrationTest): def test_omit_default_read_write_concern(self): - listener = EventListener() + listener = OvertCommandListener() # Client with default readConcern and writeConcern client = self.rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) @@ -205,7 +205,7 @@ class TestReadWriteConcernSpec(IntegrationTest): @client_context.require_version_min(4, 9) def test_write_error_details_exposes_errinfo(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) db = client.errinfotest diff --git a/test/test_server_selection.py b/test/test_server_selection.py index 67e9716bf..984b967f5 100644 --- a/test/test_server_selection.py +++ b/test/test_server_selection.py @@ -33,6 +33,7 @@ from test import IntegrationTest, client_context, unittest from test.utils import ( EventListener, FunctionCallRecorder, + OvertCommandListener, wait_until, ) from test.utils_selection_tests import ( @@ -74,7 +75,7 @@ class TestCustomServerSelectorFunction(IntegrationTest): return [servers[idx]] # Initialize client with appropriate listeners. - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client( server_selector=custom_selector, event_listeners=[listener] ) diff --git a/test/test_session.py b/test/test_session.py index 9f94ded92..d0bbb075a 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -36,6 +36,7 @@ from test import ( from test.utils import ( EventListener, ExceptionCatchingThread, + OvertCommandListener, wait_until, ) @@ -198,7 +199,7 @@ class TestSession(IntegrationTest): lsid_set = set() failures = 0 for _ in range(5): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener], maxPoolSize=1) cursor = client.db.test.find({}) ops: List[Tuple[Callable, List[Any]]] = [ diff --git a/test/test_ssl.py b/test/test_ssl.py index 36d7ba12b..04db9b61a 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -33,6 +33,7 @@ from test import ( ) from test.utils import ( EventListener, + OvertCommandListener, cat_files, ignore_deprecations, ) From 849ed7970f45a104d35f77dec8c7ec684e596087 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 18 Oct 2024 15:35:44 -0500 Subject: [PATCH 4/4] PYTHON-4888 Use shrub.py for versioned api tests (#1949) --- .evergreen/config.yml | 106 +++++++++++++++++--------- .evergreen/scripts/generate_config.py | 47 +++++++++++- 2 files changed, 111 insertions(+), 42 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9083da145..cf43d7c24 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2305,28 +2305,6 @@ axes: variables: COVERAGE: "coverage" - - id: versionedApi - display_name: "versionedApi" - values: - # Test against a cluster with requireApiVersion=1. - - id: "requireApiVersion1" - display_name: "requireApiVersion1" - tags: [ "versionedApi_tag" ] - variables: - # REQUIRE_API_VERSION is set to make drivers-evergreen-tools - # start a cluster with the requireApiVersion parameter. - REQUIRE_API_VERSION: "1" - # MONGODB_API_VERSION is the apiVersion to use in the test suite. - MONGODB_API_VERSION: "1" - # Test against a cluster with acceptApiVersion2 but without - # requireApiVersion, and don't automatically add apiVersion to - # clients created in the test suite. - - id: "acceptApiVersion2" - display_name: "acceptApiVersion2" - tags: [ "versionedApi_tag" ] - variables: - ORCHESTRATION_FILE: "versioned-api-testing.json" - - id: serverless display_name: "Serverless" values: @@ -3366,6 +3344,74 @@ buildvariants: SSL: ssl PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 +# Versioned API tests. +- name: versioned-api-require-v1-rhel8-py3.9-auth + tasks: + - name: .standalone .5.0 + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest + display_name: Versioned API require v1 RHEL8 py3.9 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + REQUIRE_API_VERSION: "1" + MONGODB_API_VERSION: "1" + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [versionedApi_tag] +- name: versioned-api-accept-v2-rhel8-py3.9-auth + tasks: + - name: .standalone .5.0 + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest + display_name: Versioned API accept v2 RHEL8 py3.9 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + ORCHESTRATION_FILE: versioned-api-testing.json + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [versionedApi_tag] +- name: versioned-api-require-v1-rhel8-py3.13-auth + tasks: + - name: .standalone .5.0 + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest + display_name: Versioned API require v1 RHEL8 py3.13 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + REQUIRE_API_VERSION: "1" + MONGODB_API_VERSION: "1" + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [versionedApi_tag] +- name: versioned-api-accept-v2-rhel8-py3.13-auth + tasks: + - name: .standalone .5.0 + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest + display_name: Versioned API accept v2 RHEL8 py3.13 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + ORCHESTRATION_FILE: versioned-api-testing.json + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [versionedApi_tag] + - matrix_name: "tests-fips" matrix_spec: platform: @@ -3559,22 +3605,6 @@ buildvariants: tasks: - name: atlas-data-lake-tests -- matrix_name: "stable-api-tests" - matrix_spec: - platform: rhel8 - python-version: ["3.9", "3.10"] - auth: "auth" - versionedApi: "*" - display_name: "Versioned API ${versionedApi} ${python-version}" - batchtime: 10080 # 7 days - tasks: - # Versioned API was introduced in MongoDB 4.7 - - "test-latest-standalone" - - "test-8.0-standalone" - - "test-7.0-standalone" - - "test-6.0-standalone" - - "test-5.0-standalone" - # OCSP test matrix. - name: ocsp-test-rhel8-v4.4-py3.9 tasks: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 6d614a9af..dafcd4ff4 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -23,7 +23,6 @@ from shrub.v3.shrub_service import ShrubService ############## ALL_VERSIONS = ["4.0", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"] -VERSIONS_6_0_PLUS = ["6.0", "7.0", "8.0", "rapid", "latest"] CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] PYPYS = ["pypy3.9", "pypy3.10"] ALL_PYTHONS = CPYTHONS + PYPYS @@ -112,6 +111,14 @@ def get_python_binary(python: str, host: str) -> str: raise ValueError(f"no match found for python {python} on {host}") +def get_pythons_from(min_version: str) -> list[str]: + """Get all pythons starting from a minimum version.""" + min_version_float = float(min_version) + rapid_latest = ["rapid", "latest"] + versions = [v for v in ALL_VERSIONS if v not in rapid_latest] + return [v for v in versions if float(v) >= min_version_float] + rapid_latest + + def get_display_name(base: str, host: str, **kwargs) -> str: """Get the display name of a variant.""" display_name = f"{base} {HOSTS[host].display_name}" @@ -243,7 +250,7 @@ def create_server_variants() -> list[BuildVariant]: tasks = [f".{topology}"] # MacOS arm64 only works on server versions 6.0+ if host == "macos-arm64": - tasks = [f".{topology} .{version}" for version in VERSIONS_6_0_PLUS] + tasks = [f".{topology} .{version}" for version in get_pythons_from("6.0")] expansions = dict(AUTH=auth, SSL=ssl, TEST_SUITES=test_suite, SKIP_CSOT_TESTS="true") display_name = get_display_name("Test", host, python=python, **expansions) variant = create_variant( @@ -330,7 +337,7 @@ def create_load_balancer_variants(): task_names = ["load-balancer-test"] batchtime = BATCHTIME_WEEK expansions_base = dict(test_loadbalancer="true") - versions = ["6.0", "7.0", "8.0", "latest", "rapid"] + versions = get_pythons_from("6.0") variants = [] pythons = CPYTHONS + PYPYS for ind, (version, (auth, ssl)) in enumerate(product(versions, AUTH_SSLS)): @@ -442,10 +449,42 @@ def create_pyopenssl_variants(): return variants +def create_versioned_api_tests(): + host = "rhel8" + tags = ["versionedApi_tag"] + tasks = [f".standalone .{v}" for v in get_pythons_from("5.0")] + variants = [] + types = ["require v1", "accept v2"] + + # All python versions across platforms. + for python, test_type in product(MIN_MAX_PYTHON, types): + expansions = dict(AUTH="auth") + # Test against a cluster with requireApiVersion=1. + if test_type == types[0]: + # REQUIRE_API_VERSION is set to make drivers-evergreen-tools + # start a cluster with the requireApiVersion parameter. + expansions["REQUIRE_API_VERSION"] = "1" + # MONGODB_API_VERSION is the apiVersion to use in the test suite. + expansions["MONGODB_API_VERSION"] = "1" + else: + # Test against a cluster with acceptApiVersion2 but without + # requireApiVersion, and don't automatically add apiVersion to + # clients created in the test suite. + expansions["ORCHESTRATION_FILE"] = "versioned-api-testing.json" + base_display_name = f"Versioned API {test_type}" + display_name = get_display_name(base_display_name, host, python=python, **expansions) + variant = create_variant( + tasks, display_name, host=host, python=python, tags=tags, expansions=expansions + ) + variants.append(variant) + + return variants + + ################## # Generate Config ################## -variants = create_pyopenssl_variants() +variants = create_versioned_api_tests() # print(len(variants)) generate_yaml(variants=variants)