From bf329add7cf59ba7f2faf616ec358cd630047f61 Mon Sep 17 00:00:00 2001 From: Iris <58442094+sleepyStick@users.noreply.github.com> Date: Fri, 6 Sep 2024 08:57:32 -0700 Subject: [PATCH 1/7] PYTHON-4732 Migrate test_auth_spec.py to async (#1836) --- test/asynchronous/test_auth_spec.py | 108 ++++++++++++++++++++++++++++ test/test_auth_spec.py | 4 +- tools/synchro.py | 1 + 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test/asynchronous/test_auth_spec.py diff --git a/test/asynchronous/test_auth_spec.py b/test/asynchronous/test_auth_spec.py new file mode 100644 index 000000000..329b3eec6 --- /dev/null +++ b/test/asynchronous/test_auth_spec.py @@ -0,0 +1,108 @@ +# Copyright 2018-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run the auth spec tests.""" +from __future__ import annotations + +import glob +import json +import os +import sys +import warnings + +sys.path[0:0] = [""] + +from test import unittest +from test.unified_format import generate_test_classes + +from pymongo import AsyncMongoClient +from pymongo.asynchronous.auth_oidc import OIDCCallback + +_IS_SYNC = False + +_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "auth") + + +class TestAuthSpec(unittest.IsolatedAsyncioTestCase): + pass + + +class SampleHumanCallback(OIDCCallback): + def fetch(self, context): + pass + + +def create_test(test_case): + def run_test(self): + uri = test_case["uri"] + valid = test_case["valid"] + credential = test_case.get("credential") + + if not valid: + with warnings.catch_warnings(): + warnings.simplefilter("default") + self.assertRaises(Exception, AsyncMongoClient, uri, connect=False) + else: + client = AsyncMongoClient(uri, connect=False) + credentials = client.options.pool_options._credentials + if credential is None: + self.assertIsNone(credentials) + else: + self.assertIsNotNone(credentials) + self.assertEqual(credentials.username, credential["username"]) + self.assertEqual(credentials.password, credential["password"]) + self.assertEqual(credentials.source, credential["source"]) + if credential["mechanism"] is not None: + self.assertEqual(credentials.mechanism, credential["mechanism"]) + else: + self.assertEqual(credentials.mechanism, "DEFAULT") + expected = credential["mechanism_properties"] + if expected is not None: + actual = credentials.mechanism_properties + for key, value in expected.items(): + self.assertEqual(getattr(actual, key.lower()), value) + else: + if credential["mechanism"] == "MONGODB-AWS": + self.assertIsNone(credentials.mechanism_properties.aws_session_token) + else: + self.assertIsNone(credentials.mechanism_properties) + + return run_test + + +def create_tests(): + for filename in glob.glob(os.path.join(_TEST_PATH, "legacy", "*.json")): + test_suffix, _ = os.path.splitext(os.path.basename(filename)) + with open(filename) as auth_tests: + test_cases = json.load(auth_tests)["tests"] + for test_case in test_cases: + if test_case.get("optional", False): + continue + test_method = create_test(test_case) + name = str(test_case["description"].lower().replace(" ", "_")) + setattr(TestAuthSpec, f"test_{test_suffix}_{name}", test_method) + + +create_tests() + + +globals().update( + generate_test_classes( + os.path.join(_TEST_PATH, "unified"), + module=__name__, + ) +) + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_auth_spec.py b/test/test_auth_spec.py index 9ec7e07f3..38e5f19bf 100644 --- a/test/test_auth_spec.py +++ b/test/test_auth_spec.py @@ -27,7 +27,9 @@ from test import unittest from test.unified_format import generate_test_classes from pymongo import MongoClient -from pymongo.asynchronous.auth_oidc import OIDCCallback +from pymongo.synchronous.auth_oidc import OIDCCallback + +_IS_SYNC = True _TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "auth") diff --git a/tools/synchro.py b/tools/synchro.py index f38a83f12..e49405ccb 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -160,6 +160,7 @@ converted_tests = [ "pymongo_mocks.py", "utils_spec_runner.py", "qcheck.py", + "test_auth_spec.py", "test_bulk.py", "test_client.py", "test_client_bulk_write.py", From 4e102235added02a4c3cf5e94acc9842eb301df1 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 6 Sep 2024 10:16:38 -0700 Subject: [PATCH 2/7] PYTHON-4560 Disable rsSyncApplyStop tests on 8.0+ (#1840) --- test/asynchronous/test_bulk.py | 2 ++ test/test_bulk.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/asynchronous/test_bulk.py b/test/asynchronous/test_bulk.py index b5f2eefde..79d8e1a0f 100644 --- a/test/asynchronous/test_bulk.py +++ b/test/asynchronous/test_bulk.py @@ -976,6 +976,7 @@ class AsyncTestBulkWriteConcern(AsyncBulkTestBase): finally: await self.secondary.admin.command("configureFailPoint", "rsSyncApplyStop", mode="off") + @async_client_context.require_version_max(7, 1) # PYTHON-4560 @async_client_context.require_replica_set @async_client_context.require_secondaries_count(1) async def test_write_concern_failure_ordered(self): @@ -1055,6 +1056,7 @@ class AsyncTestBulkWriteConcern(AsyncBulkTestBase): failed = details["writeErrors"][0] self.assertTrue("duplicate" in failed["errmsg"]) + @async_client_context.require_version_max(7, 1) # PYTHON-4560 @async_client_context.require_replica_set @async_client_context.require_secondaries_count(1) async def test_write_concern_failure_unordered(self): diff --git a/test/test_bulk.py b/test/test_bulk.py index 9069109cf..63b8c7790 100644 --- a/test/test_bulk.py +++ b/test/test_bulk.py @@ -974,6 +974,7 @@ class TestBulkWriteConcern(BulkTestBase): finally: self.secondary.admin.command("configureFailPoint", "rsSyncApplyStop", mode="off") + @client_context.require_version_max(7, 1) # PYTHON-4560 @client_context.require_replica_set @client_context.require_secondaries_count(1) def test_write_concern_failure_ordered(self): @@ -1053,6 +1054,7 @@ class TestBulkWriteConcern(BulkTestBase): failed = details["writeErrors"][0] self.assertTrue("duplicate" in failed["errmsg"]) + @client_context.require_version_max(7, 1) # PYTHON-4560 @client_context.require_replica_set @client_context.require_secondaries_count(1) def test_write_concern_failure_unordered(self): From 22b66b2ed698c3f161dcaf6f00c1d999a4a9fb87 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 6 Sep 2024 12:17:47 -0500 Subject: [PATCH 3/7] PYTHON-4695 Fix test event loop policy and improve error traceback for ClientBulkWriteException (#1828) --- pymongo/_client_bulk_shared.py | 2 ++ pyproject.toml | 1 + test/asynchronous/conftest.py | 2 ++ test/conftest.py | 2 ++ 4 files changed, 7 insertions(+) diff --git a/pymongo/_client_bulk_shared.py b/pymongo/_client_bulk_shared.py index 4dd1af210..649f1c6aa 100644 --- a/pymongo/_client_bulk_shared.py +++ b/pymongo/_client_bulk_shared.py @@ -74,4 +74,6 @@ def _throw_client_bulk_write_exception( "to your connection string." ) raise OperationFailure(errmsg, code, full_result) + if isinstance(full_result["error"], BaseException): + raise ClientBulkWriteException(full_result, verbose_results) from full_result["error"] raise ClientBulkWriteException(full_result, verbose_results) diff --git a/pyproject.toml b/pyproject.toml index 8452bfe95..225be8e1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ addopts = ["-ra", "--strict-config", "--strict-markers", "--junitxml=xunit-resul testpaths = ["test"] log_cli_level = "INFO" faulthandler_timeout = 1500 +asyncio_default_fixture_loop_scope = "session" xfail_strict = true filterwarnings = [ "error", diff --git a/test/asynchronous/conftest.py b/test/asynchronous/conftest.py index e443dff6c..c08f224ab 100644 --- a/test/asynchronous/conftest.py +++ b/test/asynchronous/conftest.py @@ -17,6 +17,8 @@ def event_loop_policy(): # has issues with sharing sockets across loops (https://github.com/python/cpython/issues/122240) # We explicitly use a different loop implementation here to prevent that issue if sys.platform == "win32": + # Needed for Python 3.8. + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) return asyncio.WindowsSelectorEventLoopPolicy() # type: ignore[attr-defined] return asyncio.get_event_loop_policy() diff --git a/test/conftest.py b/test/conftest.py index a3d954c7c..ca817a5a6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -15,6 +15,8 @@ def event_loop_policy(): # has issues with sharing sockets across loops (https://github.com/python/cpython/issues/122240) # We explicitly use a different loop implementation here to prevent that issue if sys.platform == "win32": + # Needed for Python 3.8. + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) return asyncio.WindowsSelectorEventLoopPolicy() # type: ignore[attr-defined] return asyncio.get_event_loop_policy() From 1eb3b8550e5ec41d57012cbb0acb086836187d9e Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 6 Sep 2024 10:20:29 -0700 Subject: [PATCH 4/7] PYTHON-4735 Resync SDAM tests to fix TestUnifiedLoggingLoadbalanced (#1839) --- test/discovery_and_monitoring/rs/compatible.json | 2 +- .../rs/compatible_unknown.json | 2 +- .../sharded/compatible.json | 2 +- .../single/compatible.json | 2 +- .../single/too_old_then_upgraded.json | 4 ++-- .../unified/logging-loadbalanced.json | 16 ++++++++++++++++ 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/test/discovery_and_monitoring/rs/compatible.json b/test/discovery_and_monitoring/rs/compatible.json index 444b13e9d..dfd5d57df 100644 --- a/test/discovery_and_monitoring/rs/compatible.json +++ b/test/discovery_and_monitoring/rs/compatible.json @@ -16,7 +16,7 @@ "b:27017" ], "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ], [ diff --git a/test/discovery_and_monitoring/rs/compatible_unknown.json b/test/discovery_and_monitoring/rs/compatible_unknown.json index cf92dd1ed..95e03ea95 100644 --- a/test/discovery_and_monitoring/rs/compatible_unknown.json +++ b/test/discovery_and_monitoring/rs/compatible_unknown.json @@ -16,7 +16,7 @@ "b:27017" ], "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/sharded/compatible.json b/test/discovery_and_monitoring/sharded/compatible.json index e531db97f..ceb0ec24c 100644 --- a/test/discovery_and_monitoring/sharded/compatible.json +++ b/test/discovery_and_monitoring/sharded/compatible.json @@ -23,7 +23,7 @@ "isWritablePrimary": true, "msg": "isdbgrid", "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/single/compatible.json b/test/discovery_and_monitoring/single/compatible.json index 302927598..493d9b748 100644 --- a/test/discovery_and_monitoring/single/compatible.json +++ b/test/discovery_and_monitoring/single/compatible.json @@ -11,7 +11,7 @@ "helloOk": true, "isWritablePrimary": true, "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/single/too_old_then_upgraded.json b/test/discovery_and_monitoring/single/too_old_then_upgraded.json index 58ae7d9de..c3dd98cf6 100644 --- a/test/discovery_and_monitoring/single/too_old_then_upgraded.json +++ b/test/discovery_and_monitoring/single/too_old_then_upgraded.json @@ -1,5 +1,5 @@ { - "description": "Standalone with default maxWireVersion of 0 is upgraded to one with maxWireVersion 6", + "description": "Standalone with default maxWireVersion of 0 is upgraded to one with maxWireVersion 21", "uri": "mongodb://a", "phases": [ { @@ -35,7 +35,7 @@ "helloOk": true, "isWritablePrimary": true, "minWireVersion": 0, - "maxWireVersion": 6 + "maxWireVersion": 21 } ] ], diff --git a/test/discovery_and_monitoring/unified/logging-loadbalanced.json b/test/discovery_and_monitoring/unified/logging-loadbalanced.json index 45440d255..0ad3b0cea 100644 --- a/test/discovery_and_monitoring/unified/logging-loadbalanced.json +++ b/test/discovery_and_monitoring/unified/logging-loadbalanced.json @@ -132,6 +132,22 @@ } } }, + { + "level": "debug", + "component": "topology", + "data": { + "message": "Topology description changed", + "topologyId": { + "$$exists": true + }, + "previousDescription": { + "$$exists": true + }, + "newDescription": { + "$$exists": true + } + } + }, { "level": "debug", "component": "topology", From 6bdaf19c78b6062dbf2e51513f24b941352f76c3 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 6 Sep 2024 10:46:10 -0700 Subject: [PATCH 5/7] PYTHON-4617 Skip unified retryable writes tests on MMAPv1 (#1841) --- test/asynchronous/test_transactions.py | 58 +++++++++--------- test/crud_v2_format.py | 55 ----------------- test/test_retryable_reads.py | 81 -------------------------- test/test_retryable_writes.py | 44 -------------- test/test_transactions.py | 58 +++++++++--------- test/unified_format.py | 14 +++-- 6 files changed, 62 insertions(+), 248 deletions(-) delete mode 100644 test/crud_v2_format.py diff --git a/test/asynchronous/test_transactions.py b/test/asynchronous/test_transactions.py index 8fa1e70d0..4034c8e2c 100644 --- a/test/asynchronous/test_transactions.py +++ b/test/asynchronous/test_transactions.py @@ -15,7 +15,6 @@ """Execute Transactions Spec tests.""" from __future__ import annotations -import os import sys from io import BytesIO @@ -23,8 +22,7 @@ from gridfs.asynchronous.grid_file import AsyncGridFS, AsyncGridFSBucket sys.path[0:0] = [""] -from test.asynchronous import async_client_context, unittest -from test.asynchronous.utils_spec_runner import AsyncSpecRunner +from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest from test.utils import ( OvertCommandListener, async_rs_client, @@ -54,8 +52,6 @@ from pymongo.read_preferences import ReadPreference _IS_SYNC = False -_TXN_TESTS_DEBUG = os.environ.get("TRANSACTION_TESTS_DEBUG") - # Max number of operations to perform after a transaction to prove unpinning # occurs. Chosen so that there's a low false positive rate. With 2 mongoses, # 50 attempts yields a one in a quadrillion chance of a false positive @@ -63,31 +59,7 @@ _TXN_TESTS_DEBUG = os.environ.get("TRANSACTION_TESTS_DEBUG") UNPIN_TEST_MAX_ATTEMPTS = 50 -class AsyncTransactionsBase(AsyncSpecRunner): - @classmethod - async def _setup_class(cls): - await super()._setup_class() - if async_client_context.supports_transactions(): - for address in async_client_context.mongoses: - cls.mongos_clients.append(await async_single_client("{}:{}".format(*address))) - - @classmethod - async def _tearDown_class(cls): - for client in cls.mongos_clients: - await client.close() - await super()._tearDown_class() - - def maybe_skip_scenario(self, test): - super().maybe_skip_scenario(test) - if ( - "secondary" in self.id() - and not async_client_context.is_mongos - and not async_client_context.has_secondaries - ): - raise unittest.SkipTest("No secondaries") - - -class TestTransactions(AsyncTransactionsBase): +class TestTransactions(AsyncIntegrationTest): RUN_ON_SERVERLESS = True @async_client_context.require_transactions @@ -421,7 +393,31 @@ class PatchSessionTimeout: client_session._WITH_TRANSACTION_RETRY_TIME_LIMIT = self.real_timeout -class TestTransactionsConvenientAPI(AsyncTransactionsBase): +class TestTransactionsConvenientAPI(AsyncIntegrationTest): + @classmethod + async def _setup_class(cls): + await super()._setup_class() + cls.mongos_clients = [] + if async_client_context.supports_transactions(): + for address in async_client_context.mongoses: + cls.mongos_clients.append(await async_single_client("{}:{}".format(*address))) + + @classmethod + async def _tearDown_class(cls): + for client in cls.mongos_clients: + await client.close() + await super()._tearDown_class() + + async def _set_fail_point(self, client, command_args): + cmd = {"configureFailPoint": "failCommand"} + cmd.update(command_args) + await client.admin.command(cmd) + + async def set_fail_point(self, command_args): + clients = self.mongos_clients if self.mongos_clients else [self.client] + for client in clients: + await self._set_fail_point(client, command_args) + @async_client_context.require_transactions async def test_callback_raises_custom_error(self): class _MyException(Exception): diff --git a/test/crud_v2_format.py b/test/crud_v2_format.py deleted file mode 100644 index 8eadad843..000000000 --- a/test/crud_v2_format.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2020-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""v2 format CRUD test runner. - -https://github.com/mongodb/specifications/blob/master/source/crud/tests/README.rst -""" -from __future__ import annotations - -from test.utils_spec_runner import SpecRunner - - -class TestCrudV2(SpecRunner): - # Default test database and collection names. - TEST_DB = None - TEST_COLLECTION = None - - def allowable_errors(self, op): - """Override expected error classes.""" - errors = super().allowable_errors(op) - errors += (ValueError,) - return errors - - def get_scenario_db_name(self, scenario_def): - """Crud spec says database_name is optional.""" - return scenario_def.get("database_name", self.TEST_DB) - - def get_scenario_coll_name(self, scenario_def): - """Crud spec says collection_name is optional.""" - return scenario_def.get("collection_name", self.TEST_COLLECTION) - - def get_object_name(self, op): - """Crud spec says object is optional and defaults to 'collection'.""" - return op.get("object", "collection") - - def get_outcome_coll_name(self, outcome, collection): - """Crud spec says outcome has an optional 'collection.name'.""" - return outcome["collection"].get("name", collection.name) - - def setup_scenario(self, scenario_def): - """Allow specs to override a test's setup.""" - # PYTHON-1935 Only create the collection if there is data to insert. - if scenario_def["data"]: - super().setup_scenario(scenario_def) diff --git a/test/test_retryable_reads.py b/test/test_retryable_reads.py index 9ea546ba9..b0fa42a0c 100644 --- a/test/test_retryable_reads.py +++ b/test/test_retryable_reads.py @@ -20,7 +20,6 @@ import pprint import sys import threading -from bson import SON from pymongo.errors import AutoReconnect sys.path[0:0] = [""] @@ -34,14 +33,10 @@ from test import ( ) from test.utils import ( CMAPListener, - EventListener, OvertCommandListener, - SpecTestCreator, - rs_client, rs_or_single_client, set_fail_point, ) -from test.utils_spec_runner import SpecRunner from pymongo.monitoring import ( ConnectionCheckedOutEvent, @@ -50,7 +45,6 @@ from pymongo.monitoring import ( PoolClearedEvent, ) from pymongo.synchronous.mongo_client import MongoClient -from pymongo.write_concern import WriteConcern # Location of JSON test specifications. _TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryable_reads", "legacy") @@ -74,81 +68,6 @@ class TestClientOptions(PyMongoTestCase): self.assertEqual(client.options.retry_reads, False) -class TestSpec(SpecRunner): - RUN_ON_LOAD_BALANCER = True - RUN_ON_SERVERLESS = True - - @classmethod - @client_context.require_failCommand_fail_point - # TODO: remove this once PYTHON-1948 is done. - @client_context.require_no_mmap - def setUpClass(cls): - super().setUpClass() - - def maybe_skip_scenario(self, test): - super().maybe_skip_scenario(test) - skip_names = ["listCollectionObjects", "listIndexNames", "listDatabaseObjects"] - for name in skip_names: - if name.lower() in test["description"].lower(): - self.skipTest(f"PyMongo does not support {name}") - - # Serverless does not support $out and collation. - if client_context.serverless: - for operation in test["operations"]: - if operation["name"] == "aggregate": - for stage in operation["arguments"]["pipeline"]: - if "$out" in stage: - self.skipTest("MongoDB Serverless does not support $out") - if "collation" in operation["arguments"]: - self.skipTest("MongoDB Serverless does not support collations") - - # Skip changeStream related tests on MMAPv1 and serverless. - test_name = self.id().rsplit(".")[-1] - if "changestream" in test_name.lower(): - if client_context.storage_engine == "mmapv1": - self.skipTest("MMAPv1 does not support change streams.") - if client_context.serverless: - self.skipTest("Serverless does not support change streams.") - - def get_scenario_coll_name(self, scenario_def): - """Override a test's collection name to support GridFS tests.""" - if "bucket_name" in scenario_def: - return scenario_def["bucket_name"] - return super().get_scenario_coll_name(scenario_def) - - def setup_scenario(self, scenario_def): - """Override a test's setup to support GridFS tests.""" - if "bucket_name" in scenario_def: - data = scenario_def["data"] - db_name = self.get_scenario_db_name(scenario_def) - db = client_context.client[db_name] - # Create a bucket for the retryable reads GridFS tests with as few - # majority writes as possible. - wc = WriteConcern(w="majority") - if data: - db["fs.chunks"].drop() - db["fs.files"].drop() - db["fs.chunks"].insert_many(data["fs.chunks"]) - db.get_collection("fs.files", write_concern=wc).insert_many(data["fs.files"]) - else: - db.get_collection("fs.chunks").drop() - db.get_collection("fs.files", write_concern=wc).drop() - else: - super().setup_scenario(scenario_def) - - -def create_test(scenario_def, test, name): - @client_context.require_test_commands - def run_scenario(self): - self.run_scenario(scenario_def, test) - - return run_scenario - - -test_creator = SpecTestCreator(create_test, TestSpec, _TEST_PATH) -test_creator.create_tests() - - class FindThread(threading.Thread): def __init__(self, collection): super().__init__() diff --git a/test/test_retryable_writes.py b/test/test_retryable_writes.py index 45a740e84..2938b7efa 100644 --- a/test/test_retryable_writes.py +++ b/test/test_retryable_writes.py @@ -16,7 +16,6 @@ from __future__ import annotations import copy -import os import pprint import sys import threading @@ -29,11 +28,9 @@ from test.utils import ( DeprecationFilter, EventListener, OvertCommandListener, - SpecTestCreator, rs_or_single_client, set_fail_point, ) -from test.utils_spec_runner import SpecRunner from test.version import Version from bson.codec_options import DEFAULT_CODEC_OPTIONS @@ -65,9 +62,6 @@ from pymongo.operations import ( from pymongo.synchronous.mongo_client import MongoClient from pymongo.write_concern import WriteConcern -# Location of JSON test specifications. -_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryable_writes", "legacy") - class InsertEventListener(EventListener): def succeeded(self, event: CommandSucceededEvent) -> None: @@ -89,44 +83,6 @@ class InsertEventListener(EventListener): ) -class TestAllScenarios(SpecRunner): - RUN_ON_LOAD_BALANCER = True - RUN_ON_SERVERLESS = True - - def get_object_name(self, op): - return op.get("object", "collection") - - def get_scenario_db_name(self, scenario_def): - return scenario_def.get("database_name", "pymongo_test") - - def get_scenario_coll_name(self, scenario_def): - return scenario_def.get("collection_name", "test") - - def run_test_ops(self, sessions, collection, test): - # Transform retryable writes spec format into transactions. - operation = test["operation"] - outcome = test["outcome"] - if "error" in outcome: - operation["error"] = outcome["error"] - if "result" in outcome: - operation["result"] = outcome["result"] - test["operations"] = [operation] - super().run_test_ops(sessions, collection, test) - - -def create_test(scenario_def, test, name): - @client_context.require_test_commands - @client_context.require_no_mmap - def run_scenario(self): - self.run_scenario(scenario_def, test) - - return run_scenario - - -test_creator = SpecTestCreator(create_test, TestAllScenarios, _TEST_PATH) -test_creator.create_tests() - - def retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}), diff --git a/test/test_transactions.py b/test/test_transactions.py index b1869bec7..c8c3c32d5 100644 --- a/test/test_transactions.py +++ b/test/test_transactions.py @@ -15,7 +15,6 @@ """Execute Transactions Spec tests.""" from __future__ import annotations -import os import sys from io import BytesIO @@ -23,14 +22,13 @@ from gridfs.synchronous.grid_file import GridFS, GridFSBucket sys.path[0:0] = [""] -from test import client_context, unittest +from test import IntegrationTest, client_context, unittest from test.utils import ( OvertCommandListener, rs_client, single_client, wait_until, ) -from test.utils_spec_runner import SpecRunner from typing import List from bson import encode @@ -54,8 +52,6 @@ from pymongo.synchronous.helpers import next _IS_SYNC = True -_TXN_TESTS_DEBUG = os.environ.get("TRANSACTION_TESTS_DEBUG") - # Max number of operations to perform after a transaction to prove unpinning # occurs. Chosen so that there's a low false positive rate. With 2 mongoses, # 50 attempts yields a one in a quadrillion chance of a false positive @@ -63,31 +59,7 @@ _TXN_TESTS_DEBUG = os.environ.get("TRANSACTION_TESTS_DEBUG") UNPIN_TEST_MAX_ATTEMPTS = 50 -class TransactionsBase(SpecRunner): - @classmethod - def _setup_class(cls): - super()._setup_class() - if client_context.supports_transactions(): - for address in client_context.mongoses: - cls.mongos_clients.append(single_client("{}:{}".format(*address))) - - @classmethod - def _tearDown_class(cls): - for client in cls.mongos_clients: - client.close() - super()._tearDown_class() - - def maybe_skip_scenario(self, test): - super().maybe_skip_scenario(test) - if ( - "secondary" in self.id() - and not client_context.is_mongos - and not client_context.has_secondaries - ): - raise unittest.SkipTest("No secondaries") - - -class TestTransactions(TransactionsBase): +class TestTransactions(IntegrationTest): RUN_ON_SERVERLESS = True @client_context.require_transactions @@ -417,7 +389,31 @@ class PatchSessionTimeout: client_session._WITH_TRANSACTION_RETRY_TIME_LIMIT = self.real_timeout -class TestTransactionsConvenientAPI(TransactionsBase): +class TestTransactionsConvenientAPI(IntegrationTest): + @classmethod + def _setup_class(cls): + super()._setup_class() + cls.mongos_clients = [] + if client_context.supports_transactions(): + for address in client_context.mongoses: + cls.mongos_clients.append(single_client("{}:{}".format(*address))) + + @classmethod + def _tearDown_class(cls): + for client in cls.mongos_clients: + client.close() + super()._tearDown_class() + + def _set_fail_point(self, client, command_args): + cmd = {"configureFailPoint": "failCommand"} + cmd.update(command_args) + client.admin.command(cmd) + + def set_fail_point(self, command_args): + clients = self.mongos_clients if self.mongos_clients else [self.client] + for client in clients: + self._set_fail_point(client, command_args) + @client_context.require_transactions def test_callback_raises_custom_error(self): class _MyException(Exception): diff --git a/test/unified_format.py b/test/unified_format.py index d35aed435..168d35ee1 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1100,6 +1100,13 @@ class UnifiedSpecTestMixinV1(IntegrationTest): if not cls.should_run_on(run_on_spec): raise unittest.SkipTest(f"{cls.__name__} runOnRequirements not satisfied") + # add any special-casing for skipping tests here + if client_context.storage_engine == "mmapv1": + if "retryable-writes" in cls.TEST_SPEC["description"] or "retryable_writes" in str( + cls.TEST_PATH + ): + raise unittest.SkipTest("MMAPv1 does not support retryWrites=True") + # Handle mongos_clients for transactions tests. cls.mongos_clients = [] if ( @@ -1110,11 +1117,6 @@ class UnifiedSpecTestMixinV1(IntegrationTest): for address in client_context.mongoses: cls.mongos_clients.append(single_client("{}:{}".format(*address))) - # add any special-casing for skipping tests here - if client_context.storage_engine == "mmapv1": - if "retryable-writes" in cls.TEST_SPEC["description"]: - raise unittest.SkipTest("MMAPv1 does not support retryWrites=True") - # Speed up the tests by decreasing the heartbeat frequency. cls.knobs = client_knobs( heartbeat_frequency=0.1, @@ -2157,7 +2159,7 @@ def generate_test_classes( raise ValueError( f"test file '{fpath}' has unsupported schemaVersion '{schema_version}'" ) - module_dict = {"__module__": module} + module_dict = {"__module__": module, "TEST_PATH": test_path} module_dict.update(kwargs) test_klasses[class_name] = type( class_name, From f2cd655d0481aa93e89af4c99fd8a03a6979636f Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 6 Sep 2024 16:04:39 -0400 Subject: [PATCH 6/7] PYTHON-4746 - Bump minimum pytest and pytest-asyncio versions (#1845) --- requirements/test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 1facbf03b..135114fef 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,2 +1,2 @@ -pytest>=7 -pytest-asyncio +pytest>=8.2 +pytest-asyncio>=0.24.0 From c883012b562128091bb4b8d184032be763d495f0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 6 Sep 2024 15:38:58 -0500 Subject: [PATCH 7/7] PYTHON-4703 MongoClient should default to connect=False on FaaS environments (#1844) --- doc/changelog.rst | 7 +++++++ pymongo/asynchronous/mongo_client.py | 9 ++++++++- pymongo/synchronous/mongo_client.py | 12 ++++++++++-- test/lambda/mongodb/app.py | 4 ++++ tools/synchro.py | 3 ++- 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index c5a4f47d7..6fffcdf69 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -42,6 +42,12 @@ PyMongo 4.9 brings a number of improvements including: - Fixed a bug where PyMongo would raise ``InvalidBSON: date value out of range`` when using :attr:`~bson.codec_options.DatetimeConversion.DATETIME_CLAMP` or :attr:`~bson.codec_options.DatetimeConversion.DATETIME_AUTO` with a non-UTC timezone. +- The default value for ``connect`` in ``MongoClient`` is changed to ``False`` when running on + unction-as-a-service (FaaS) like AWS Lambda, Google Cloud Functions, and Microsoft Azure Functions. + On some FaaS systems, there is a ``fork()`` operation at function + startup. By delaying the connection to the first operation, we avoid a deadlock. See + `Is PyMongo Fork-Safe`_ for more information. + Issues Resolved ............... @@ -49,6 +55,7 @@ Issues Resolved See the `PyMongo 4.9 release notes in JIRA`_ for the list of resolved issues in this release. +.. _Is PyMongo Fork-Safe : https://www.mongodb.com/docs/languages/python/pymongo-driver/current/faq/#is-pymongo-fork-safe- .. _PyMongo 4.9 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=39940 diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 2af773c44..b5e73e8de 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -720,6 +720,10 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): .. versionchanged:: 4.7 Deprecated parameter ``wTimeoutMS``, use :meth:`~pymongo.timeout`. + + .. versionchanged:: 4.9 + The default value of ``connect`` is changed to ``False`` when running in a + Function-as-a-service environment. """ doc_class = document_class or dict self._init_kwargs: dict[str, Any] = { @@ -803,7 +807,10 @@ class AsyncMongoClient(common.BaseObject, Generic[_DocumentType]): if tz_aware is None: tz_aware = opts.get("tz_aware", False) if connect is None: - connect = opts.get("connect", True) + # Default to connect=True unless on a FaaS system, which might use fork. + from pymongo.pool_options import _is_faas + + connect = opts.get("connect", not _is_faas()) keyword_opts["tz_aware"] = tz_aware keyword_opts["connect"] = connect diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 6c5f68b7e..26af488ac 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -263,7 +263,8 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): aware (otherwise they will be naive) :param connect: If ``True`` (the default), immediately begin connecting to MongoDB in the background. Otherwise connect - on the first operation. + on the first operation. The default value is ``False`` when + running in a Function-as-a-service environment. :param type_registry: instance of :class:`~bson.codec_options.TypeRegistry` to enable encoding and decoding of custom types. @@ -719,6 +720,10 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): .. versionchanged:: 4.7 Deprecated parameter ``wTimeoutMS``, use :meth:`~pymongo.timeout`. + + .. versionchanged:: 4.9 + The default value of ``connect`` is changed to ``False`` when running in a + Function-as-a-service environment. """ doc_class = document_class or dict self._init_kwargs: dict[str, Any] = { @@ -802,7 +807,10 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): if tz_aware is None: tz_aware = opts.get("tz_aware", False) if connect is None: - connect = opts.get("connect", True) + # Default to connect=True unless on a FaaS system, which might use fork. + from pymongo.pool_options import _is_faas + + connect = opts.get("connect", not _is_faas()) keyword_opts["tz_aware"] = tz_aware keyword_opts["connect"] = connect diff --git a/test/lambda/mongodb/app.py b/test/lambda/mongodb/app.py index 5840347d9..274990d3b 100644 --- a/test/lambda/mongodb/app.py +++ b/test/lambda/mongodb/app.py @@ -8,6 +8,7 @@ from __future__ import annotations import json import os +import warnings from bson import has_c as has_bson_c from pymongo import MongoClient @@ -18,6 +19,9 @@ from pymongo.monitoring import ( ServerHeartbeatListener, ) +# Ensure there are no warnings raised in normal operation. +warnings.simplefilter("error") + open_connections = 0 heartbeat_count = 0 streaming_heartbeat_count = 0 diff --git a/tools/synchro.py b/tools/synchro.py index e49405ccb..e79cfce40 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -108,7 +108,8 @@ replacements = { docstring_replacements: dict[tuple[str, str], str] = { ("MongoClient", "connect"): """If ``True`` (the default), immediately begin connecting to MongoDB in the background. Otherwise connect - on the first operation.""", + on the first operation. The default value is ``False`` when + running in a Function-as-a-service environment.""", ("Collection", "create"): """If ``True``, force collection creation even without options being set.""", ("Collection", "session"): """A