From d2b95d1bf027c17ee1f049c3077354a0ecdcf947 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 8 Jul 2022 19:40:25 -0500 Subject: [PATCH] PYTHON-3336 Test Failure - test_load_balancer failing (#1000) --- .pre-commit-config.yaml | 2 +- CONTRIBUTING.rst | 17 +++++++++++++++++ pymongo/errors.py | 9 +++++++++ pymongo/mongo_client.py | 8 ++++++-- pymongo/monitoring.py | 2 +- pymongo/pool.py | 5 +++-- test/test_cmap.py | 9 +++++++-- test/utils.py | 26 +++++++++++++++++++++++++- 8 files changed, 69 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fd86e092..d72d51971 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: rev: 0.11.1 hooks: - id: doc8 - args: [--max-line-length=200] + args: ["--ignore=D001"] # ignore line length stages: [manual] - repo: https://github.com/sirosen/check-jsonschema diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b8bbad93f..f44e74688 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -70,6 +70,23 @@ branch and submit a `pull request `_ button. +Running Tests Locally +--------------------- +- Ensure you have started the appropriate Mongo Server(s). +- Run ``python setup.py test`` to run all of the tests. +- Run ``python setup.py test -s test...`` to + run specific tests. You can omit the ```` to test a full class + and the ```` to test a full module. For example: + ``python setup.py test -s test.test_change_stream.TestUnifiedChangeStreamsErrors.test_change_stream_errors_on_ElectionInProgress``. + +Running Load Balancer Tests Locally +----------------------------------- +- Install ``haproxy`` (available as ``brew install haproxy`` on macOS). +- Clone ``drivers-evergreen-tools``: ``git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git``. +- Start the servers using ``LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=./drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=./drivers-evergreen-tools/.evergreen/orchestration ./drivers-evergreen-tools/.evergreen/run-orchestration.sh``. +- Start the load balancer using: ``MONGODB_URI='mongodb://localhost:27017,localhost:27018/' .evergreen/run-load-balancer.sh start``. +- Run the tests using: ``LOADBALANCER=1 TEST_LOADBALANCER=1 SINGLE_MONGOS_LB_URI='mongodb://127.0.0.1:8000/?loadBalanced=true' MULTI_MONGOS_LB_URI='mongodb://127.0.0.1:8001/?loadBalanced=true' MONGODB_URI='mongodb://localhost:27017,localhost:27018/' python setup.py test -s test.test_load_balancer``. + Re-sync Spec Tests ------------------ diff --git a/pymongo/errors.py b/pymongo/errors.py index 4a167383c..a01911c7e 100644 --- a/pymongo/errors.py +++ b/pymongo/errors.py @@ -61,6 +61,15 @@ class ConnectionFailure(PyMongoError): """Raised when a connection to the database cannot be made or is lost.""" +class WaitQueueTimeoutError(ConnectionFailure): + """Raised when an operation times out waiting to checkout a connection from the pool. + + Subclass of :exc:`~pymongo.errors.ConnectionFailure`. + + .. versionadded:: 4.2 + """ + + class AutoReconnect(ConnectionFailure): """Raised when a connection to the database is lost and an attempt to auto-reconnect will be made. diff --git a/pymongo/mongo_client.py b/pymongo/mongo_client.py index 6d139a238..1defe3253 100644 --- a/pymongo/mongo_client.py +++ b/pymongo/mongo_client.py @@ -80,6 +80,7 @@ from pymongo.errors import ( OperationFailure, PyMongoError, ServerSelectionTimeoutError, + WaitQueueTimeoutError, ) from pymongo.pool import ConnectionClosedReason from pymongo.read_preferences import ReadPreference, _ServerMode @@ -1182,6 +1183,7 @@ class MongoClient(common.BaseObject, Generic[_DocumentType]): with _MongoClientErrorHandler(self, server, session) as err_handler: # Reuse the pinned connection, if it exists. if in_txn and session._pinned_connection: + err_handler.contribute_socket(session._pinned_connection) yield session._pinned_connection return with server.get_socket(handler=err_handler) as sock_info: @@ -2064,9 +2066,11 @@ def _add_retryable_write_error(exc, max_wire_version): if code in helpers._RETRYABLE_ERROR_CODES: exc._add_error_label("RetryableWriteError") - # Connection errors are always retryable except NotPrimaryError which is + # Connection errors are always retryable except NotPrimaryError and WaitQueueTimeoutError which is # handled above. - if isinstance(exc, ConnectionFailure) and not isinstance(exc, NotPrimaryError): + if isinstance(exc, ConnectionFailure) and not isinstance( + exc, (NotPrimaryError, WaitQueueTimeoutError) + ): exc._add_error_label("RetryableWriteError") diff --git a/pymongo/monitoring.py b/pymongo/monitoring.py index ad604f3f1..f3f773fbb 100644 --- a/pymongo/monitoring.py +++ b/pymongo/monitoring.py @@ -1774,7 +1774,7 @@ class _EventListeners(object): event = ConnectionCheckOutFailedEvent(address, reason) for subscriber in self.__cmap_listeners: try: - subscriber.connection_check_out_started(event) + subscriber.connection_check_out_failed(event) except Exception: _handle_exception() diff --git a/pymongo/pool.py b/pymongo/pool.py index f8cc60329..493a544d0 100644 --- a/pymongo/pool.py +++ b/pymongo/pool.py @@ -52,6 +52,7 @@ from pymongo.errors import ( NotPrimaryError, OperationFailure, PyMongoError, + WaitQueueTimeoutError, _CertificateError, ) from pymongo.hello import Hello, HelloCompat @@ -1637,7 +1638,7 @@ class Pool: timeout = _csot.get_timeout() or self.opts.wait_queue_timeout if self.opts.load_balanced: other_ops = self.active_sockets - self.ncursors - self.ntxns - raise ConnectionFailure( + raise WaitQueueTimeoutError( "Timeout waiting for connection from the connection pool. " "maxPoolSize: %s, connections in use by cursors: %s, " "connections in use by transactions: %s, connections in use " @@ -1650,7 +1651,7 @@ class Pool: timeout, ) ) - raise ConnectionFailure( + raise WaitQueueTimeoutError( "Timed out while checking out a connection from connection pool. " "maxPoolSize: %s, timeout: %s" % (self.opts.max_pool_size, timeout) ) diff --git a/test/test_cmap.py b/test/test_cmap.py index a2a1d8d21..360edef0e 100644 --- a/test/test_cmap.py +++ b/test/test_cmap.py @@ -38,7 +38,12 @@ from test.utils_spec_runner import SpecRunnerThread from bson.objectid import ObjectId from bson.son import SON -from pymongo.errors import ConnectionFailure, OperationFailure, PyMongoError +from pymongo.errors import ( + ConnectionFailure, + OperationFailure, + PyMongoError, + WaitQueueTimeoutError, +) from pymongo.monitoring import ( ConnectionCheckedInEvent, ConnectionCheckedOutEvent, @@ -73,7 +78,7 @@ OBJECT_TYPES = { "ConnectionPoolClosed": PoolClosedEvent, # Error types. "PoolClosedError": _PoolClosedError, - "WaitQueueTimeoutError": ConnectionFailure, + "WaitQueueTimeoutError": WaitQueueTimeoutError, } diff --git a/test/utils.py b/test/utils.py index 7071764b1..d80bf551d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -38,7 +38,20 @@ from pymongo.collection import ReturnDocument from pymongo.cursor import CursorType from pymongo.errors import ConfigurationError, OperationFailure from pymongo.hello import HelloCompat -from pymongo.monitoring import _SENSITIVE_COMMANDS +from pymongo.monitoring import ( + _SENSITIVE_COMMANDS, + ConnectionCheckedInEvent, + ConnectionCheckedOutEvent, + ConnectionCheckOutFailedEvent, + ConnectionCheckOutStartedEvent, + ConnectionClosedEvent, + ConnectionCreatedEvent, + ConnectionReadyEvent, + PoolClearedEvent, + PoolClosedEvent, + PoolCreatedEvent, + PoolReadyEvent, +) from pymongo.pool import _CancellationContext, _PoolGeneration from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference @@ -81,36 +94,47 @@ class BaseListener(object): class CMAPListener(BaseListener, monitoring.ConnectionPoolListener): def connection_created(self, event): + assert isinstance(event, ConnectionCreatedEvent) self.add_event(event) def connection_ready(self, event): + assert isinstance(event, ConnectionReadyEvent) self.add_event(event) def connection_closed(self, event): + assert isinstance(event, ConnectionClosedEvent) self.add_event(event) def connection_check_out_started(self, event): + assert isinstance(event, ConnectionCheckOutStartedEvent) self.add_event(event) def connection_check_out_failed(self, event): + assert isinstance(event, ConnectionCheckOutFailedEvent) self.add_event(event) def connection_checked_out(self, event): + assert isinstance(event, ConnectionCheckedOutEvent) self.add_event(event) def connection_checked_in(self, event): + assert isinstance(event, ConnectionCheckedInEvent) self.add_event(event) def pool_created(self, event): + assert isinstance(event, PoolCreatedEvent) self.add_event(event) def pool_ready(self, event): + assert isinstance(event, PoolReadyEvent) self.add_event(event) def pool_cleared(self, event): + assert isinstance(event, PoolClearedEvent) self.add_event(event) def pool_closed(self, event): + assert isinstance(event, PoolClosedEvent) self.add_event(event)