From 3c9fd60fedfa7364b04a39d81c3df1c76aebf818 Mon Sep 17 00:00:00 2001 From: "A. Jesse Jiryu Davis" Date: Fri, 19 Sep 2014 14:51:22 -0400 Subject: [PATCH] PYTHON-525 Use MongoClient in replica set tests. Connect the MongoClient to the replica set if one is running, otherwise connect it to the standalone. Only continue to use MongoReplicaSetClient when testing the specific behavior of that deprecated class. Also suppress some warnings when running tests, and better determine whether to test IPv6. --- test/__init__.py | 60 +- test/mod_wsgi_test/README.rst | 23 +- test/mod_wsgi_test/mod_wsgi_test.conf | 9 +- ..._single_server.wsgi => mod_wsgi_test.wsgi} | 6 + .../mod_wsgi_test_replica_set.wsgi | 55 -- test/test_auth.py | 41 +- test/test_client.py | 221 ++---- test/test_collection.py | 11 +- test/test_cursor.py | 9 +- test/test_read_preferences.py | 3 +- test/test_replica_set_client.py | 733 ++---------------- test/test_ssl.py | 123 +-- test/test_threads.py | 76 +- test/test_threads_replica_set_client.py | 76 -- test/utils.py | 96 ++- 15 files changed, 335 insertions(+), 1207 deletions(-) rename test/mod_wsgi_test/{mod_wsgi_test_single_server.wsgi => mod_wsgi_test.wsgi} (89%) delete mode 100644 test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi delete mode 100644 test/test_threads_replica_set_client.py diff --git a/test/__init__.py b/test/__init__.py index 216814d51..ab5f6b0a3 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -16,7 +16,10 @@ """ import os +import socket import sys +from pymongo.mongo_client import _partition_node + if sys.version_info[:2] == (2, 6): import unittest2 as unittest from unittest2 import SkipTest @@ -34,9 +37,9 @@ from bson.py3compat import _unicode, reraise from pymongo import common from test.version import Version -# hostnames retrieved by MongoReplicaSetClient from isMaster will be of unicode -# type in Python 2, so ensure these hostnames are unicodes, too. It makes tests -# like `test_repr` predictable. +# hostnames retrieved from isMaster will be of unicode type in Python 2, +# so ensure these hostnames are unicodes, too. It makes tests like +# `test_repr` predictable. host = _unicode(os.environ.get("DB_IP", 'localhost')) port = int(os.environ.get("DB_PORT", 27017)) pair = '%s:%d' % (host, port) @@ -86,6 +89,7 @@ class ClientContext(object): self.connected = False self.ismaster = {} self.w = None + self.nodes = set() self.setname = None self.rs_client = None self.cmd_line = None @@ -93,6 +97,9 @@ class ClientContext(object): self.auth_enabled = False self.test_commands_enabled = False self.is_mongos = False + self.is_rs = False + self.has_ipv6 = False + try: with client_knobs(server_wait_time=0.1): client = pymongo.MongoClient(host, port) @@ -105,12 +112,20 @@ class ClientContext(object): self.connected = True self.ismaster = self.client.admin.command('ismaster') self.w = len(self.ismaster.get("hosts", [])) or 1 + self.nodes = set([(host, port)]) self.setname = self.ismaster.get('setName', '') self.rs_client = None self.version = Version.from_client(self.client) if self.setname: - self.rs_client = pymongo.MongoReplicaSetClient( + self.is_rs = True + self.rs_client = pymongo.MongoClient( pair, replicaSet=self.setname) + + self.nodes = set([_partition_node(node) + for node in self.ismaster.get('hosts', [])]) + + self.rs_or_standalone_client = self.rs_client or self.client + try: self.cmd_line = self.client.admin.command('getCmdLineOpts') except pymongo.errors.OperationFailure as e: @@ -141,6 +156,7 @@ class ClientContext(object): self.test_commands_enabled = ('testCommandsEnabled=1' in self.cmd_line['argv']) self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid') + self.has_ipv6 = self._server_started_with_ipv6() def _check_user_provided(self): try: @@ -172,6 +188,27 @@ class ClientContext(object): argv = self.cmd_line['argv'] return '--auth' in argv or '--keyFile' in argv + def _server_started_with_ipv6(self): + if not socket.has_ipv6: + return False + + if 'parsed' in self.cmd_line: + if not self.cmd_line['parsed'].get('net', {}).get('ipv6'): + return False + else: + if '--ipv6' not in self.cmd_line['argv']: + return False + + # The server was started with --ipv6. Is there an IPv6 route to it? + try: + for info in socket.getaddrinfo(host, port): + if info[0] == socket.AF_INET6: + return True + except socket.error: + pass + + return False + def _require(self, condition, msg, func=None): def make_wrapper(f): @wraps(f) @@ -225,10 +262,23 @@ class ClientContext(object): def require_replica_set(self, func): """Run a test only if the client is connected to a replica set.""" - return self._require(self.rs_client is not None, + return self._require(self.is_rs, "Not connected to a replica set", func=func) + def require_no_replica_set(self, func): + """Run a test if the client is *not* connected to a replica set.""" + return self._require( + not self.is_rs, + "Connected to a replica set, not a standalone mongod", + func=func) + + def require_ipv6(self, func): + """Run a test only if the client can connect to a server via IPv6.""" + return self._require(self.has_ipv6, + "No IPv6", + func=func) + def require_no_mongos(self, func): """Run a test only if the client is not connected to a mongos.""" return self._require(not self.is_mongos, diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index 673864753..2dea6d6d5 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -15,7 +15,8 @@ Test Matrix PyMongo should be tested with several versions of mod_wsgi and a selection of Python versions. Each combination of mod_wsgi and Python version should -be tested with both MongoClient and MongoReplicaSetClient. +be tested with a standalone and a replica set. ``mod_wsgi_test.wsgi`` +detects if the deployment is a replica set and connects to the whole set. Setup ----- @@ -56,25 +57,23 @@ Run the test Run the included ``test_client.py`` script:: python test/mod_wsgi_test/test_client.py -n 2500 -t 100 parallel \ - http://localhost/${mongodb_configuration}/${WORKSPACE} + http://localhost/${WORKSPACE} ...where the "n" argument is the total number of requests to make to Apache, -and "t" specifies the number of threads. ``mongodb_configuration`` should be -"single_server" or "replica_set", depending on how you started mongod. -``WORKSPACE`` is the location of the PyMongo checkout. +and "t" specifies the number of threads. ``WORKSPACE`` is the location of +the PyMongo checkout. Run this script again with different arguments to make serial requests:: - python ${WORKSPACE}/test/mod_wsgi_test/test_client.py -n 25000 serial \ - http://localhost/${mongodb_configuration}/${WORKSPACE} + python test/mod_wsgi_test/test_client.py -n 25000 serial \ + http://localhost/${WORKSPACE} The ``test_client.py`` script merely makes HTTP requests to Apache. Its exit code is non-zero if any of its requests fails, for example with an HTTP 500. -The core of the test is in the WSGI scripts: -``mod_wsgi_test_single_server.wsgi`` and ``mod_wsgi_test_replica_set.wsgi``. -These scripts insert some documents into MongoDB at startup, then query +The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`. +This script inserts some documents into MongoDB at startup, then queries documents for each HTTP request. If PyMongo is leaking connections and "n" is much greater than the ulimit, @@ -85,5 +84,5 @@ Automation At MongoDB, Inc. we use a Jenkins job that tests each combination in the matrix. The job copies the appropriate version of ``mod_wsgi.so`` into -place, sets up Apache, and runs ``test_client.py`` with the proper -arguments. +place, sets up Apache, starts a single server or replica set, +and runs ``test_client.py`` with the proper arguments. diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index 0b8c714af..8815f0b34 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -28,12 +28,9 @@ WSGISocketPrefix /tmp/ WSGIProcessGroup mod_wsgi_test # For the convienience of unittests, rather than hard-code the location of - # mod_wsgi_test_single_server.wsgi and mod_wsgi_test_replica_set.wsgi, - # include it in the URL, so - # http://localhost/single_server/location-of-pymongo-checkout will work: + # mod_wsgi_test.wsgi, include it in the URL, so + # http://localhost/location-of-pymongo-checkout will work: - WSGIScriptAliasMatch ^/single_server(.+) $1/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi - - WSGIScriptAliasMatch ^/replica_set(.+) $1/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi + WSGIScriptAliasMatch ^/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.wsgi diff --git a/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi b/test/mod_wsgi_test/mod_wsgi_test.wsgi similarity index 89% rename from test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi rename to test/mod_wsgi_test/mod_wsgi_test.wsgi index 9285561ca..74aca931b 100644 --- a/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi +++ b/test/mod_wsgi_test/mod_wsgi_test.wsgi @@ -28,6 +28,12 @@ import pymongo from pymongo.mongo_client import MongoClient client = MongoClient() + +# If the deployment is a replica set, connect to the whole set. +replica_set_name = client.admin.command('ismaster').get('setName') +if replica_set_name: + client = MongoClient(replicaSet=replica_set_name) + collection = client.test.test ndocs = 20 diff --git a/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi b/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi deleted file mode 100644 index a43dff146..000000000 --- a/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2012-2014 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Minimal test of PyMongo in a WSGI application with MongoReplicaSetClient, - see bug PYTHON-353. -""" - -import os -import sys - -this_path = os.path.dirname(os.path.join(os.getcwd(), __file__)) - -# Location of PyMongo checkout -repository_path = os.path.normpath(os.path.join(this_path, '..', '..')) -sys.path.insert(0, repository_path) - -import pymongo -from pymongo.mongo_replica_set_client import MongoReplicaSetClient - -client = MongoReplicaSetClient(replicaSet='repl0') -collection = client.test.test - -ndocs = 20 - -collection.drop() -collection.insert([{'i': i} for i in range(ndocs)]) -client.disconnect() # Discard main thread's request socket. - -try: - from mod_wsgi import version as mod_wsgi_version -except: - mod_wsgi_version = None - - -def application(environ, start_response): - # Requests are part of the PYTHON-353 pathology. - client.start_request() - results = list(collection.find().batch_size(10)) - assert len(results) == ndocs - output = 'python %s, mod_wsgi %s, pymongo %s' % ( - sys.version, mod_wsgi_version, pymongo.version) - response_headers = [('Content-Length', str(len(output)))] - start_response('200 OK', response_headers) - return [output] diff --git a/test/test_auth.py b/test/test_auth.py index da7e217f6..d66f926af 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -26,7 +26,7 @@ except ImportError: sys.path[0:0] = [""] -from pymongo import MongoClient, MongoReplicaSetClient +from pymongo import MongoClient from pymongo.auth import HAVE_KERBEROS, _build_credentials_tuple from pymongo.errors import OperationFailure, ConfigurationError from pymongo.read_preferences import ReadPreference @@ -114,9 +114,9 @@ class TestGSSAPI(unittest.TestCase): set_name = client.admin.command('ismaster').get('setName') if set_name: - client = MongoReplicaSetClient(GSSAPI_HOST, - port=GSSAPI_PORT, - replicaSet=set_name) + client = MongoClient(GSSAPI_HOST, + port=GSSAPI_PORT, + replicaSet=set_name) # Without gssapiServiceName self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) @@ -124,7 +124,7 @@ class TestGSSAPI(unittest.TestCase): uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertTrue(client.database_names()) # With gssapiServiceName @@ -137,7 +137,7 @@ class TestGSSAPI(unittest.TestCase): GSSAPI_HOST, GSSAPI_PORT, str(set_name))) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertTrue(client.database_names()) def test_gssapi_threaded(self): @@ -160,9 +160,9 @@ class TestGSSAPI(unittest.TestCase): set_name = client.admin.command('ismaster').get('setName') if set_name: preference = ReadPreference.SECONDARY - client = MongoReplicaSetClient(GSSAPI_HOST, - replicaSet=set_name, - read_preference=preference) + client = MongoClient(GSSAPI_HOST, + replicaSet=set_name, + read_preference=preference) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) self.assertTrue(client.test.command('dbstats')) @@ -201,9 +201,9 @@ class TestSASLPlain(unittest.TestCase): set_name = client.admin.command('ismaster').get('setName') if set_name: - client = MongoReplicaSetClient(SASL_HOST, - port=SASL_PORT, - replicaSet=set_name) + client = MongoClient(SASL_HOST, + port=SASL_PORT, + replicaSet=set_name) self.assertTrue(client.ldap.authenticate(SASL_USER, SASL_PASS, SASL_DB, 'PLAIN')) client.ldap.test.find_one() @@ -213,7 +213,7 @@ class TestSASLPlain(unittest.TestCase): quote_plus(SASL_PASS), SASL_HOST, SASL_PORT, SASL_DB, str(set_name))) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) client.ldap.test.find_one() def test_sasl_plain_bad_credentials(self): @@ -283,8 +283,8 @@ class TestSCRAMSHA1(unittest.TestCase): client.pymongo_test.command('dbstats') if self.set_name: - client = MongoReplicaSetClient(host, port, - replicaSet='%s' % (self.set_name,)) + client = MongoClient(host, port, + replicaSet='%s' % (self.set_name,)) self.assertTrue(client.pymongo_test.authenticate( 'user', 'pass', mechanism='SCRAM-SHA-1')) client.pymongo_test.command('dbstats') @@ -292,7 +292,7 @@ class TestSCRAMSHA1(unittest.TestCase): uri = ('mongodb://user:pass' '@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1' '&replicaSet=%s' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) client.pymongo_test.command('dbstats') client.read_preference = ReadPreference.SECONDARY client.pymongo_test.command('dbstats') @@ -321,8 +321,7 @@ class TestAuthURIOptions(unittest.TestCase): # GLE requires authentication. client.admin.authenticate('admin', 'pass') # Make sure the admin user is replicated after calling add_user - # above. This avoids a race in the MRSC tests below. Adding a - # user is just an insert into system.users. + # above. This avoids a race in the replica set tests below. client.admin.command('getLastError', w=len(response['hosts'])) self.client = client @@ -342,7 +341,7 @@ class TestAuthURIOptions(unittest.TestCase): if self.set_name: uri = ('mongodb://admin:pass' '@%s:%d/?replicaSet=%s' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertTrue(client.admin.command('dbstats')) client.read_preference = ReadPreference.SECONDARY self.assertTrue(client.admin.command('dbstats')) @@ -356,7 +355,7 @@ class TestAuthURIOptions(unittest.TestCase): if self.set_name: uri = ('mongodb://user:pass@%s:%d' '/pymongo_test?replicaSet=%s' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertRaises(OperationFailure, client.admin.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) @@ -374,7 +373,7 @@ class TestAuthURIOptions(unittest.TestCase): if self.set_name: uri = ('mongodb://user:pass@%s:%d/pymongo_test2?replicaSet=' '%s;authSource=pymongo_test' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertRaises(OperationFailure, client.pymongo_test2.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) diff --git a/test/test_client.py b/test/test_client.py index c8e89a7b2..b376fc648 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -59,16 +59,18 @@ from test import (client_context, from test.pymongo_mocks import MockClient from test.utils import (assertRaisesExactly, delay, - get_client, + ignore_deprecations, remove_all_users, server_is_master_with_slave, TestRequestMixin, _TestLazyConnectMixin, - lazy_client_trial, get_pool, one, connected, - wait_until) + wait_until, + rs_or_single_client, + rs_or_single_client_noauth, + lazy_client_trial) class ClientUnitTest(unittest.TestCase, TestRequestMixin): @@ -129,7 +131,7 @@ class TestClient(IntegrationTest, TestRequestMixin): @classmethod def setUpClass(cls): super(TestClient, cls).setUpClass() - cls.client = client_context.client + cls.client = client_context.rs_or_standalone_client def test_constants(self): # Set bad defaults. @@ -156,13 +158,16 @@ class TestClient(IntegrationTest, TestRequestMixin): self.fail(self._formatMessage(msg, standardMsg)) def test_init_disconnected(self): - c = MongoClient(host, port, connect=False) + c = rs_or_single_client(connect=False) self.assertIsInstance(c.is_primary, bool) self.assertIsInstance(c.is_mongos, bool) self.assertIsInstance(c.max_pool_size, int) self.assertIsInstance(c.nodes, frozenset) - self.assertEqual(dict, c.get_document_class()) + + with ignore_deprecations(): + self.assertEqual(dict, c.get_document_class()) + self.assertIsInstance(c.tz_aware, bool) self.assertIsInstance(c.max_bson_size, int) self.assertIsInstance(c.min_wire_version, int) @@ -170,6 +175,8 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertIsInstance(c.max_write_batch_size, int) self.assertEqual(None, c.host) self.assertEqual(None, c.port) + self.assertFalse(c.primary) + self.assertFalse(c.secondaries) c.pymongo_test.command('ismaster') # Auto-connect. self.assertEqual(host, c.host) @@ -193,13 +200,11 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) def test_equality(self): - c = connected(MongoClient(host, port)) - - # ClientContext.client is constructed as MongoClient(host, port) - self.assertEqual(self.client, c) + c = connected(rs_or_single_client()) + self.assertEqual(client_context.rs_or_standalone_client, c) # Explicitly test inequality - self.assertFalse(self.client != c) + self.assertFalse(client_context.rs_or_standalone_client != c) def test_host_w_port(self): with client_knobs(server_wait_time=0.01): @@ -215,7 +220,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_getters(self): self.assertEqual(self.client.host, host) self.assertEqual(self.client.port, port) - self.assertEqual(set([(host, port)]), self.client.nodes) + self.assertEqual(client_context.nodes, self.client.nodes) def test_database_names(self): self.client.pymongo_test.test.save({"dummy": u("object")}) @@ -298,7 +303,7 @@ class TestClient(IntegrationTest, TestRequestMixin): c.drop_database("pymongo_test1") c.admin.add_user("admin", "password") - auth_c = MongoClient(host, port) + auth_c = rs_or_single_client_noauth() auth_c.admin.authenticate("admin", "password") try: auth_c.pymongo_test.add_user("mike", "password") @@ -339,7 +344,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_from_uri(self): self.assertEqual( self.client, - connected(MongoClient("mongodb://%s:%d" % (host, port)))) + connected(rs_or_single_client("mongodb://%s:%d" % (host, port)))) @client_context.require_auth def test_auth_from_uri(self): @@ -349,27 +354,29 @@ class TestClient(IntegrationTest, TestRequestMixin): "user", "pass", roles=['userAdmin', 'readWrite']) with self.assertRaises(OperationFailure): - connected(MongoClient("mongodb://a:b@%s:%d" % (host, port))) + connected(rs_or_single_client_noauth( + "mongodb://a:b@%s:%d" % (host, port))) # No error. - connected(MongoClient("mongodb://admin:pass@%s:%d" % (host, port))) + connected(rs_or_single_client_noauth( + "mongodb://admin:pass@%s:%d" % (host, port))) # Wrong database. uri = "mongodb://admin:pass@%s:%d/pymongo_test" % (host, port) with self.assertRaises(OperationFailure): - connected(MongoClient(uri)) + connected(rs_or_single_client_noauth(uri)) # No error. - connected(MongoClient( + connected(rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port))) # Auth with lazy connection. - MongoClient( + rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port), connect=False).pymongo_test.test.find_one() # Wrong password. - bad_client = MongoClient( + bad_client = rs_or_single_client_noauth( "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port), connect=False) @@ -387,8 +394,8 @@ class TestClient(IntegrationTest, TestRequestMixin): self.client.pymongo_test.add_user('user2', 'pass', roles=['readWrite']) try: - client = MongoClient("mongodb://user1:pass@%s:%d/pymongo_test" % ( - host, port)) + client = rs_or_single_client_noauth( + "mongodb://user1:pass@%s:%d/pymongo_test" % (host, port)) client.pymongo_test.test.find_one() with self.assertRaises(OperationFailure): @@ -413,9 +420,8 @@ class TestClient(IntegrationTest, TestRequestMixin): @client_context.require_auth def test_lazy_auth_raises_operation_failure(self): - lazy_client = MongoClient( - "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port), - connect=False) + lazy_client = rs_or_single_client( + "mongodb://user:wrong@%s/pymongo_test" % host, connect=False) assertRaisesExactly( OperationFailure, lazy_client.test.collection.find_one) @@ -495,7 +501,7 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) - c = get_client(pair, document_class=SON) + c = rs_or_single_client(document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) @@ -511,34 +517,34 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertRaises(DeprecationWarning, c.get_document_class) def test_timeouts(self): - client = MongoClient(host, port, connectTimeoutMS=10500) + client = rs_or_single_client(connectTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.connect_timeout) - client = MongoClient(host, port, socketTimeoutMS=10500) + client = rs_or_single_client(socketTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.socket_timeout) def test_socket_timeout_ms_validation(self): - c = get_client(pair, socketTimeoutMS=10 * 1000) + c = rs_or_single_client(socketTimeoutMS=10 * 1000) self.assertEqual(10, get_pool(c).opts.socket_timeout) - c = connected(get_client(pair, socketTimeoutMS=None)) + c = connected(rs_or_single_client(socketTimeoutMS=None)) self.assertEqual(None, get_pool(c).opts.socket_timeout) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS=0) + rs_or_single_client, socketTimeoutMS=0) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS=-1) + rs_or_single_client, socketTimeoutMS=-1) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS=1e10) + rs_or_single_client, socketTimeoutMS=1e10) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS='foo') + rs_or_single_client, socketTimeoutMS='foo') def test_socket_timeout(self): no_timeout = self.client timeout_sec = 1 - timeout = get_client(pair, socketTimeoutMS=1000 * timeout_sec) + timeout = rs_or_single_client(socketTimeoutMS=1000 * timeout_sec) no_timeout.pymongo_test.drop_collection("test") no_timeout.pymongo_test.test.insert({"x": 1}) @@ -553,11 +559,11 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertRaises(NetworkTimeout, get_x, timeout.pymongo_test) def test_waitQueueTimeoutMS(self): - client = MongoClient(host, port, waitQueueTimeoutMS=2000) + client = rs_or_single_client(waitQueueTimeoutMS=2000) self.assertEqual(get_pool(client).opts.wait_queue_timeout, 2) def test_waitQueueMultiple(self): - client = MongoClient(host, port, max_pool_size=3, waitQueueMultiple=2) + client = rs_or_single_client(max_pool_size=3, waitQueueMultiple=2) pool = get_pool(client) self.assertEqual(pool.opts.wait_queue_multiple, 2) self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6) @@ -565,7 +571,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_tz_aware(self): self.assertRaises(ConfigurationError, MongoClient, tz_aware='foo') - aware = get_client(pair, tz_aware=True) + aware = rs_or_single_client(tz_aware=True) naive = self.client aware.pymongo_test.drop_collection("test") @@ -578,22 +584,18 @@ class TestClient(IntegrationTest, TestRequestMixin): aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), naive.pymongo_test.test.find_one()["x"]) + @client_context.require_ipv6 def test_ipv6(self): - with client_knobs(server_wait_time=0.01): - try: - connected(MongoClient("[::1]")) - except: - # Either mongod was started without --ipv6 - # or the OS doesn't support it (or both). - raise SkipTest("No IPv6") - if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%s[::1]:%d" % (auth_str, port) - client = MongoClient(uri) + if client_context.is_rs: + uri += '/?replicaSet=' + client_context.setname + + client = rs_or_single_client_noauth(uri) client.pymongo_test.test.save({"dummy": u("object")}) client.pymongo_test_bernie.test.save({"dummy": u("object")}) @@ -627,7 +629,7 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertFalse(locked) def test_contextlib(self): - client = get_client(pair) + client = rs_or_single_client(pair) client.pymongo_test.drop_collection("test") client.pymongo_test.test.insert({"foo": "bar"}) @@ -677,33 +679,6 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertNoRequest(pool) self.assertDifferentSock(pool) - def test_nested_request(self): - pool = get_pool(self.client) - self.assertFalse(self.client.in_request()) - - # Start and end request - self.client.start_request() - self.assertInRequestAndSameSock(self.client, pool) - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - - # Double-nesting - self.client.start_request() - self.client.start_request() - self.client.end_request() - self.assertInRequestAndSameSock(self.client, pool) - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - - # Extra end_request calls have no effect - count stays at zero - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - - self.client.start_request() - self.assertInRequestAndSameSock(self.client, pool) - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - def test_request_threads(self): client = self.client pool = get_pool(client) @@ -810,7 +785,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_operation_failure_with_request(self): # Ensure MongoClient doesn't close socket after it gets an error # response to getLastError. PYTHON-395. - c = get_client(pair) + c = rs_or_single_client(pair) c.start_request() pool = get_pool(c) @@ -854,32 +829,21 @@ class TestClient(IntegrationTest, TestRequestMixin): with self.assertRaises(CursorNotFound): list(cursor) - @client_context.require_replica_set - def test_replica_set(self): - name = client_context.setname - connected(MongoClient(host, port, replicaSet=name)) # No error. - - with client_knobs(server_wait_time=0.01): - client = MongoClient(host, port, replicaSet='bad' + name) - - with self.assertRaises(AutoReconnect): - connected(client) - def test_lazy_connect_w0(self): - client = get_client(connection_string(), connect=False) + client = rs_or_single_client(connect=False) client.pymongo_test.test.insert({}, w=0) - client = get_client(connection_string(), connect=False) + client = rs_or_single_client(connect=False) client.pymongo_test.test.update({}, {'$set': {'x': 1}}, w=0) - client = get_client(connection_string(), connect=False) + client = rs_or_single_client(connect=False) client.pymongo_test.test.remove(w=0) @client_context.require_no_mongos def test_exhaust_network_error(self): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid semaphore leaks. - client = get_client(max_pool_size=1) + client = rs_or_single_client(max_pool_size=1) collection = client.pymongo_test.test pool = get_pool(client) pool._check_interval_seconds = None # Never check. @@ -905,7 +869,7 @@ class TestClient(IntegrationTest, TestRequestMixin): # when authenticating a new socket with cached credentials. # Get a client with one socket so we detect if it's leaked. - c = get_client(max_pool_size=1, waitQueueTimeoutMS=1) + c = rs_or_single_client(max_pool_size=1, waitQueueTimeoutMS=1) # Simulate an authenticate() call on a different socket. credentials = auth._build_credentials_tuple( @@ -925,62 +889,17 @@ class TestClient(IntegrationTest, TestRequestMixin): # No semaphore leak, the pool is allowed to make a new socket. c.test.collection.find_one() + @client_context.require_no_replica_set + def test_connect_to_standalone_using_replica_set_name(self): + with client_knobs(server_wait_time=0.1): + client = MongoClient(pair, replicaSet='anything') + + with self.assertRaises(AutoReconnect): + client.test.test.find_one() + class TestClientProperties(MockClientTest): - @client_context.require_connection - def test_wire_version(self): - c = MockClient( - standalones=[], - members=['a:1', 'b:2', 'c:3'], - mongoses=[], - host='b:2', # Pass a secondary. - replicaSet='rs', - connect=False) - - c.set_wire_version_range('a:1', 1, 5) - c._get_topology().select_servers(writable_server_selector) # Connect. - self.assertEqual(c.min_wire_version, 1) - self.assertEqual(c.max_wire_version, 5) - - c.set_wire_version_range('a:1', 10, 11) - c.disconnect() - c._get_topology() - self.assertRaises(ConfigurationError, c.db.collection.find_one) - - def test_max_wire_version(self): - # Disable periodic monitoring. - with client_knobs(heartbeat_frequency=1e6): - c = MockClient( - standalones=[], - members=['a:1', 'b:2', 'c:3'], - mongoses=[], - host='b:2', # Pass a secondary. - replicaSet='rs', - connect=False) - - c.set_max_write_batch_size('a:1', 1) - c.set_max_write_batch_size('b:2', 2) - - # Starts with default max batch size. - self.assertEqual(1000, c.max_write_batch_size) - connected(c) - wait_until(lambda: len(c.nodes) == 3, 'connect') - - # Uses primary's max batch size. - self.assertEqual(c.max_write_batch_size, 1) - - # b becomes primary. - c.mock_primary = 'b:2' - c.disconnect() - - # While disconnected, return to default max batch size. - self.assertEqual(1000, c.max_write_batch_size) - - connected(c) - wait_until(lambda: len(c.nodes) == 3, 'connect') - self.assertEqual(c.max_write_batch_size, 2) - def test_wire_version_mongos_ha(self): # TODO: Reimplement Mongos HA with PyMongo 3's MongoClient. raise SkipTest('Mongos HA must be reimplemented in PyMongo 3') @@ -1015,19 +934,15 @@ class TestClientProperties(MockClientTest): class TestClientLazyConnect(IntegrationTest, _TestLazyConnectMixin): - - def _get_client(self, **kwargs): - return get_client(connection_string(), **kwargs) + def _get_client(self): + return rs_or_single_client(connection_string(), connect=False) class TestClientLazyConnectBadSeeds(IntegrationTest): - - def _get_client(self, **kwargs): - kwargs.setdefault('connectTimeoutMS', 100) - + def _get_client(self): # Assume there are no open mongods listening on a.com, b.com, .... bad_seeds = ['%s.com' % chr(ord('a') + i) for i in range(10)] - return get_client(bad_seeds, **kwargs) + return rs_or_single_client(bad_seeds, connect=False) def test_connect(self): def reset(dummy): diff --git a/test/test_collection.py b/test/test_collection.py index 928222c1d..f15e04d50 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -48,9 +48,10 @@ from pymongo.errors import (DocumentTooLarge, InvalidOperation, OperationFailure, WTimeoutError) -from test.test_client import get_client, IntegrationTest +from test.test_client import IntegrationTest from test.utils import (is_mongos, joinall, enable_text_search, get_pool, - oid_generated_on_client, one) + oid_generated_on_client, one, ignore_deprecations, + get_client) from test import client_context, host, port, pair, qcheck, unittest @@ -232,8 +233,7 @@ class TestCollection(IntegrationTest): self.assertRaises(DeprecationWarning, lambda: db.test.ensure_index("goodbye", ttl=10)) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): self.assertEqual("goodbye_1", db.test.ensure_index("goodbye", ttl=10)) self.assertEqual(None, db.test.ensure_index("goodbye")) @@ -2187,8 +2187,7 @@ class TestCollection(IntegrationTest): for j in range(5): c.insert({'j': j, 'i': 0}) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): sort={'j': DESCENDING} self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, diff --git a/test/test_cursor.py b/test/test_cursor.py index 4611599c0..5492ff192 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -18,7 +18,6 @@ import itertools import random import re import sys -import warnings sys.path[0:0] = [""] @@ -36,7 +35,7 @@ from pymongo.errors import (InvalidOperation, OperationFailure, ExecutionTimeout) from test import client_context, SkipTest, unittest, host, port, IntegrationTest -from test.utils import server_started_with_auth +from test.utils import ignore_deprecations, server_started_with_auth if PY3: long = int @@ -1115,8 +1114,7 @@ class TestCursor(IntegrationTest): client = self.db.connection try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): client.set_cursor_manager(CManager) docs = [] cursor = self.db.test.find().batch_size(10) @@ -1130,8 +1128,7 @@ class TestCursor(IntegrationTest): docs.extend(ccursor) self.assertEqual(len(docs), 200) finally: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): client.set_cursor_manager(CursorManager) if __name__ == "__main__": diff --git a/test/test_read_preferences.py b/test/test_read_preferences.py index 0c6504ca1..a13a7817b 100644 --- a/test/test_read_preferences.py +++ b/test/test_read_preferences.py @@ -33,7 +33,6 @@ from pymongo.server_type import SERVER_TYPE from pymongo.errors import ConfigurationError from test.test_replica_set_client import TestReplicaSetClientBase -from test.test_client import get_client from test import (client_context, host, port, @@ -43,7 +42,7 @@ from test import (client_context, IntegrationTest, db_user, db_pwd) -from test.utils import connected, one, wait_until +from test.utils import connected, get_client, one, wait_until from test.version import Version diff --git a/test/test_replica_set_client.py b/test/test_replica_set_client.py index 3b58db55d..e7aa457c8 100644 --- a/test/test_replica_set_client.py +++ b/test/test_replica_set_client.py @@ -14,30 +14,21 @@ """Test the mongo_replica_set_client module.""" -import datetime -import signal import socket import sys import time -import threading import warnings sys.path[0:0] = [""] -from bson.py3compat import thread, u, _unicode +from bson.py3compat import u from bson.son import SON -from bson.tz_util import utc -from pymongo import auth -from pymongo.database import Database from pymongo.errors import (AutoReconnect, ConfigurationError, - ConnectionFailure, - InvalidName, OperationFailure, - NetworkTimeout) -from pymongo.mongo_client import _partition_node + NetworkTimeout, ConnectionFailure) +from pymongo.mongo_client import MongoClient, _partition_node from pymongo.mongo_replica_set_client import MongoReplicaSetClient -from pymongo.pool import SocketInfo from pymongo.read_preferences import ReadPreference, Secondary, Nearest from test import (client_context, client_knobs, @@ -51,30 +42,12 @@ from test import (client_context, MockClientTest) from test.pymongo_mocks import MockClient from test.utils import ( - delay, assertReadFrom, assertReadFromAll, read_from_which_host, - remove_all_users, assertRaisesExactly, TestRequestMixin, one, - server_started_with_auth, pools_from_rs_client, get_pool, - get_rs_client, _TestLazyConnectMixin, connected, wait_until) + delay, assertReadFrom, assertReadFromAll, ignore_deprecations, + read_from_which_host, assertRaisesExactly, TestRequestMixin, get_pools, + connected, wait_until, get_client, rs_or_single_client) from test.version import Version -class TestReplicaSetClientAgainstStandalone(unittest.TestCase): - """This is a funny beast -- we want to run tests for MongoReplicaSetClient - but only if the database at DB_IP and DB_PORT is a standalone. - """ - @client_context.require_connection - def setUp(self): - if client_context.setname: - raise SkipTest("Connected to a replica set, not a standalone mongod") - - def test_connect(self): - with client_knobs(server_wait_time=0.1): - client = MongoReplicaSetClient(pair, replicaSet='anything') - - with self.assertRaises(AutoReconnect): - client.test.test.find_one() - - class TestReplicaSetClientBase(unittest.TestCase): @classmethod @@ -100,8 +73,8 @@ class TestReplicaSetClientBase(unittest.TestCase): if m['stateStr'] == 'SECONDARY') def _get_client(self, **kwargs): - return get_rs_client(connection_string(), - replicaSet=self.name, **kwargs) + return get_client(connection_string(), + replicaSet=self.name, **kwargs) class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): @@ -115,126 +88,43 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self.fail(msg) - def assertIsInstance(self, obj, cls, msg=None): - """Backport from Python 2.7.""" - if not isinstance(obj, cls): - standardMsg = '%r is not an instance of %r' % (obj, cls) - self.fail(self._formatMessage(msg, standardMsg)) - def test_deprecated(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) with self.assertRaises(DeprecationWarning): MongoReplicaSetClient() - def test_init_disconnected(self): - c = self._get_client(connect=False) - - self.assertIsInstance(c.is_mongos, bool) - self.assertIsInstance(c.max_pool_size, int) - self.assertIsInstance(c.tz_aware, bool) - self.assertIsInstance(c.max_bson_size, int) - self.assertIsInstance(c.min_wire_version, int) - self.assertIsInstance(c.max_wire_version, int) - self.assertIsInstance(c.arbiters, set) - self.assertEqual(dict, c.get_document_class()) - self.assertFalse(c.primary) - self.assertFalse(c.secondaries) - - connected(c) - self.assertTrue(c.primary) - wait_until(lambda: c.secondaries, "found secondaries") - - if Version.from_client(c).at_least(2, 5, 4, -1): - self.assertTrue(c.max_wire_version > 0) - else: - self.assertEqual(c.max_wire_version, 0) - self.assertTrue(c.min_wire_version >= 0) - - c = self._get_client(connect=False) - c.pymongo_test.test.update({}, {}) # Auto-connect for write. - self.assertTrue(c.primary) - - c = self._get_client(connect=False) - c.pymongo_test.test.insert({}) # Auto-connect for write. - self.assertTrue(c.primary) - - c = self._get_client(connect=False) - c.pymongo_test.test.remove({}) # Auto-connect for write. - self.assertTrue(c.primary) - - with client_knobs(server_wait_time=0.1): - c = MongoReplicaSetClient( - "somedomainthatdoesntexist.org", replicaSet="rs", - connectTimeoutMS=1, connect=False) - - self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) - - def test_init_disconnected_with_auth_failure(self): - with client_knobs(server_wait_time=0.1): - c = MongoReplicaSetClient( - "mongodb://user:pass@somedomainthatdoesntexist", - replicaSet="rs", connectTimeoutMS=1, connect=False) - - self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) - - @client_context.require_auth - def test_init_disconnected_with_auth(self): - c = client_context.rs_client - try: - c.pymongo_test.add_user("user", "pass", - roles=['readWrite', 'userAdmin']) - - # Auth with lazy connection. - host = one(self.hosts) - uri = "mongodb://user:pass@%s:%d/pymongo_test?replicaSet=%s" % ( - host[0], host[1], self.name) - - authenticated_client = MongoReplicaSetClient(uri, connect=False) - authenticated_client.pymongo_test.test.find_one() - - # Wrong password. - bad_uri = "mongodb://user:wrong@%s:%d/pymongo_test?replicaSet=%s" % ( - host[0], host[1], self.name) - - bad_client = MongoReplicaSetClient(bad_uri, connect=False) - self.assertRaises( - OperationFailure, bad_client.pymongo_test.test.find_one) - - finally: - # Clean up. - remove_all_users(c.pymongo_test) - def test_connect(self): with client_knobs(server_wait_time=0.1): - client = MongoReplicaSetClient( - "somedomainthatdoesntexist.org:27017", - replicaSet=self.name) - - with self.assertRaises(AutoReconnect): - client.test.test.find_one() - - client = MongoReplicaSetClient(pair, replicaSet='fdlksjfdslkjfd') + client = MongoClient(pair, replicaSet='fdlksjfdslkjfd') with self.assertRaises(ConnectionFailure): client.test.test.find_one() def test_repr(self): - client = client_context.rs_client + with ignore_deprecations(): + client = MongoReplicaSetClient(connection_string(), + replicaSet=self.name) - # Quirk: the RS client makes a frozenset of hosts from a dict's keys, - # so we must do the same to achieve the same order. - host_dict = dict([(host, 1) for host in self.hosts]) - hosts_set = frozenset(host_dict) - hosts_repr = ', '.join([ - repr(_unicode('%s:%s' % host)) for host in hosts_set]) + wait_until(lambda: client.primary == self.primary, "discover primary") + wait_until(lambda: client.secondaries == self.secondaries, + "discover secondaries") - self.assertEqual(repr(client), - "MongoReplicaSetClient([%s])" % hosts_repr) + # repr should be something like + # MongoReplicaSetClient(["localhost:27017", "localhost:27018"]). + self.assertIn("MongoReplicaSetClient([", repr(client)) + for host in self.hosts: + self.assertIn("%s:%d" % host, repr(client)) def test_properties(self): c = client_context.rs_client c.admin.command('ping') + + wait_until(lambda: c.primary == self.primary, "discover primary") + wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") + wait_until(lambda: c.secondaries == self.secondaries, + "discover secondaries") + self.assertEqual(c.primary, self.primary) self.assertEqual(c.secondaries, self.secondaries) self.assertEqual(c.arbiters, self.arbiters) @@ -242,7 +132,8 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self.assertEqual(c.document_class, dict) self.assertEqual(c.tz_aware, False) - # Make sure MRSC's properties are copied to Database and Collection + # Make sure MongoClient's properties are copied to Database and + # Collection. for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, {}) @@ -253,17 +144,12 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets=tag_sets) - c = MongoReplicaSetClient( + c = MongoClient( pair, replicaSet=self.name, max_pool_size=25, document_class=SON, tz_aware=True, read_preference=secondary, secondaryacceptablelatencyms=77) - wait_until(lambda: c.primary == self.primary, "discover primary") - wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") - wait_until(lambda: c.secondaries == self.secondaries, - "discover secondaries") - self.assertEqual(c.max_pool_size, 25) self.assertEqual(c.document_class, SON) self.assertEqual(c.tz_aware, True) @@ -287,26 +173,9 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self.assertEqual(c.max_bson_size, 4194304) c.close() - def test_get_db(self): - client = client_context.rs_client - - def make_db(base, name): - return base[name] - - self.assertRaises(InvalidName, make_db, client, "") - self.assertRaises(InvalidName, make_db, client, "te$t") - self.assertRaises(InvalidName, make_db, client, "te.t") - self.assertRaises(InvalidName, make_db, client, "te\\t") - self.assertRaises(InvalidName, make_db, client, "te/t") - self.assertRaises(InvalidName, make_db, client, "te st") - - self.assertTrue(isinstance(client.test, Database)) - self.assertEqual(client.test, client["test"]) - self.assertEqual(client.test, Database(client, "test")) - def test_auto_reconnect_exception_when_read_preference_is_secondary(self): with client_knobs(server_wait_time=0.1): - c = MongoReplicaSetClient(pair, replicaSet=self.name) + c = MongoClient(pair, replicaSet=self.name) db = c.pymongo_test def raise_socket_error(*args, **kwargs): @@ -321,276 +190,8 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): finally: socket.socket.sendall = old_sendall - @client_context.require_auth - def test_lazy_auth_raises_operation_failure(self): - # Check if we have the prerequisites to run this test. - lazy_client = MongoReplicaSetClient( - "mongodb://user:wrong@%s/pymongo_test" % pair, - replicaSet=self.name, - connect=False) - - assertRaisesExactly( - OperationFailure, lazy_client.test.collection.find_one) - - def test_operations(self): - c = client_context.rs_client - - # Check explicitly for a case we've commonly hit in tests: - # a replica set is started with a tiny oplog, a previous - # test does a big insert that leaves the secondaries - # permanently "RECOVERING", and our insert(w=self.w) hangs - # forever. - rs_status = c.admin.command('replSetGetStatus') - members = rs_status['members'] - self.assertFalse( - [m for m in members if m['stateStr'] == 'RECOVERING'], - "Replica set is recovering, try a larger oplogSize next time" - ) - - db = c.pymongo_test - db.test.remove({}) - self.assertEqual(0, db.test.count()) - db.test.insert({'foo': 'x'}, w=self.w, wtimeout=10000) - self.assertEqual(1, db.test.count()) - - cursor = db.test.find() - doc = next(cursor) - self.assertEqual('x', doc['foo']) - # Ensure we read from the primary - self.assertEqual(c.primary, cursor._Cursor__connection_id) - - cursor = db.test.find(read_preference=ReadPreference.SECONDARY) - doc = next(cursor) - self.assertEqual('x', doc['foo']) - # Ensure we didn't read from the primary - self.assertTrue(cursor._Cursor__connection_id in c.secondaries) - - self.assertEqual(1, db.test.count()) - db.test.remove({}) - self.assertEqual(0, db.test.count()) - db.test.drop() - - def test_database_names(self): - client = client_context.rs_client - - client.pymongo_test.test.save({"dummy": u("object")}) - client.pymongo_test_mike.test.save({"dummy": u("object")}) - - dbs = client.database_names() - self.assertTrue("pymongo_test" in dbs) - self.assertTrue("pymongo_test_mike" in dbs) - - def test_drop_database(self): - client = client_context.rs_client - - self.assertRaises(TypeError, client.drop_database, 5) - self.assertRaises(TypeError, client.drop_database, None) - - client.pymongo_test.test.save({"dummy": u("object")}) - dbs = client.database_names() - self.assertTrue("pymongo_test" in dbs) - client.drop_database("pymongo_test") - dbs = client.database_names() - self.assertTrue("pymongo_test" not in dbs) - - client.pymongo_test.test.save({"dummy": u("object")}) - dbs = client.database_names() - self.assertTrue("pymongo_test" in dbs) - client.drop_database(client.pymongo_test) - dbs = client.database_names() - self.assertTrue("pymongo_test" not in dbs) - - def test_copy_db(self): - c = client_context.rs_client - # We test copy twice; once starting in a request and once not. In - # either case the copy should succeed (because it starts a request - # internally) and should leave us in the same state as before the copy. - with c.start_request(): - self.assertRaises(TypeError, c.copy_database, 4, "foo") - self.assertRaises(TypeError, c.copy_database, "foo", 4) - - self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") - - c.pymongo_test.test.drop() - c.drop_database("pymongo_test1") - c.drop_database("pymongo_test2") - - c.pymongo_test.test.insert({"foo": "bar"}) - - self.assertFalse("pymongo_test1" in c.database_names()) - self.assertFalse("pymongo_test2" in c.database_names()) - - c.copy_database("pymongo_test", "pymongo_test1") - # copy_database() didn't accidentally end the request - self.assertTrue(c.in_request()) - - self.assertTrue("pymongo_test1" in c.database_names()) - self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) - - self.assertFalse(c.in_request()) - - c.copy_database("pymongo_test", "pymongo_test2") - # copy_database() didn't accidentally restart the request - self.assertFalse(c.in_request()) - - time.sleep(1) - - self.assertTrue("pymongo_test2" in c.database_names()) - self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) - - if (Version.from_client(c).at_least(1, 3, 3, 1) and - server_started_with_auth(c)): - c.drop_database("pymongo_test1") - - try: - c.pymongo_test.add_user("mike", "password") - - self.assertRaises(OperationFailure, c.copy_database, - "pymongo_test", "pymongo_test1", - username="foo", password="bar") - self.assertFalse("pymongo_test1" in c.database_names()) - - self.assertRaises(OperationFailure, c.copy_database, - "pymongo_test", "pymongo_test1", - username="mike", password="bar") - self.assertFalse("pymongo_test1" in c.database_names()) - - c.copy_database("pymongo_test", "pymongo_test1", - username="mike", password="password") - self.assertTrue("pymongo_test1" in c.database_names()) - res = c.pymongo_test1.test.find_one( - read_preference=ReadPreference.PRIMARY) - self.assertEqual("bar", res["foo"]) - finally: - # Cleanup - remove_all_users(c.pymongo_test) - - def test_get_default_database(self): - host = one(self.hosts) - uri = "mongodb://%s:%d/foo?replicaSet=%s" % ( - host[0], host[1], self.name) - - c = MongoReplicaSetClient(uri, connect=False) - self.assertEqual(Database(c, 'foo'), c.get_default_database()) - - def test_get_default_database_error(self): - host = one(self.hosts) - # URI with no database. - uri = "mongodb://%s:%d/?replicaSet=%s" % ( - host[0], host[1], self.name) - - c = MongoReplicaSetClient(uri, connect=False) - self.assertRaises(ConfigurationError, c.get_default_database) - - def test_get_default_database_with_authsource(self): - # Ensure we distinguish database name from authSource. - host = one(self.hosts) - uri = "mongodb://%s:%d/foo?replicaSet=%s&authSource=src" % ( - host[0], host[1], self.name) - - c = MongoReplicaSetClient(uri, connect=False) - self.assertEqual(Database(c, 'foo'), c.get_default_database()) - - def test_iteration(self): - client = client_context.rs_client - - def iterate(): - [a for a in client] - - self.assertRaises(TypeError, iterate) - - def test_disconnect(self): - c = client_context.rs_client - coll = c.pymongo_test.bar - - c.disconnect() - c.disconnect() - - coll.count() - - c.disconnect() - c.disconnect() - - coll.count() - - def test_close(self): - # Multiple threads can call close() at once without error. - # Subsequent operations reopen the client. - c = self._get_client() - nthreads = 10 - outcomes = [] - - def close(): - c.close() - outcomes.append(True) - - threads = [threading.Thread(target=close) for _ in range(nthreads)] - for t in threads: - t.start() - - for t in threads: - t.join(10) - - self.assertEqual(nthreads, len(outcomes)) - - # No error. - c.db.collection.insert({}) - c.db.collection.find_one() - - def test_socket_timeout_ms_validation(self): - c = self._get_client(socketTimeoutMS=10 * 1000) - self.assertEqual(10, - c._MongoClient__options.pool_options.socket_timeout) - - c = self._get_client(socketTimeoutMS=None) - self.assertEqual(None, - c._MongoClient__options.pool_options.socket_timeout) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS=0) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS=-1) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS=1e10) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS='foo') - - def test_socket_timeout(self): - no_timeout = client_context.rs_client - timeout_sec = 1 - timeout = self._get_client(socketTimeoutMS=timeout_sec*1000) - - no_timeout.pymongo_test.drop_collection("test") - no_timeout.pymongo_test.test.insert({"x": 1}) - - # A $where clause that takes a second longer than the timeout. - query = {'$where': delay(1 + timeout_sec)} - no_timeout.pymongo_test.test.find_one(query) # No error. - - try: - timeout.pymongo_test.test.find_one(query) - except AutoReconnect as e: - self.assertTrue('%d: timed out' % (port,) in e.args[0]) - else: - self.fail('RS client should have raised timeout error') - - try: - timeout.pymongo_test.test.find_one( - query, - read_preference=ReadPreference.SECONDARY) - except NetworkTimeout as e: - self.assertTrue( - any('%d: timed out' % address[1] in e.args[0] - for address in self.secondaries), - "%r does not mention any secondary's port" % e.args[0]) - else: - self.fail('RS client should have raised timeout error') - def test_timeout_does_not_mark_member_down(self): - # If a query times out, the RS client shouldn't mark the member "down". + # If a query times out, the client shouldn't mark the member "down". # Disable background refresh. with client_knobs(heartbeat_frequency=999999): @@ -619,51 +220,20 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): # No error. collection.find_one(read_preference=ReadPreference.SECONDARY) - def test_waitQueueTimeoutMS(self): - client = self._get_client(waitQueueTimeoutMS=2000) - pool = get_pool(client) - self.assertEqual(pool.opts.wait_queue_timeout, 2) - - def test_waitQueueMultiple(self): - client = self._get_client(max_pool_size=3, waitQueueMultiple=2) - pool = get_pool(client) - self.assertEqual(pool.opts.wait_queue_multiple, 2) - self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6) - - def test_tz_aware(self): - self.assertRaises(ConfigurationError, MongoReplicaSetClient, - tz_aware='foo', replicaSet=self.name) - - aware = self._get_client(tz_aware=True) - naive = client_context.rs_client - aware.pymongo_test.drop_collection("test") - - now = datetime.datetime.utcnow() - aware.pymongo_test.test.insert({"x": now}) - time.sleep(1) - - self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo) - self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo) - self.assertEqual( - aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), - naive.pymongo_test.test.find_one()["x"]) - + @client_context.require_replica_set + @client_context.require_ipv6 def test_ipv6(self): - try: - MongoReplicaSetClient("[::1]:%d" % (port,), replicaSet=self.name) - except: - # Either mongod was started without --ipv6 - # or the OS doesn't support it (or both). - raise SkipTest("No IPv6") + c = MongoClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name) - # Try a few simple things - MongoReplicaSetClient("mongodb://[::1]:%d" % (port,), - replicaSet=self.name) - MongoReplicaSetClient("mongodb://[::1]:%d/?w=0;" - "replicaSet=%s" % (port, self.name)) - MongoReplicaSetClient("[::1]:%d,localhost:" - "%d" % (port, port), - replicaSet=self.name) + # Client switches to IPv4 once it has first ismaster response. + msg = 'discovered primary with IPv4 address "%r"' % (self.primary,) + wait_until(lambda: c.primary == self.primary, msg) + + # Same outcome with both IPv4 and IPv6 seeds. + c = MongoClient("[::1]:%d,localhost:%d" % (port, port), + replicaSet=self.name) + + wait_until(lambda: c.primary == self.primary, msg) if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) @@ -671,7 +241,7 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): auth_str = "" uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port) - client = MongoReplicaSetClient(uri, replicaSet=self.name) + client = MongoClient(uri, replicaSet=self.name) client.pymongo_test.test.save({"dummy": u("object")}) client.pymongo_test_bernie.test.save({"dummy": u("object")}) @@ -727,103 +297,12 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): def test_kill_cursor_explicit_secondary(self): self._test_kill_cursor_explicit(ReadPreference.SECONDARY) - def test_interrupt_signal(self): - if sys.platform.startswith('java'): - raise SkipTest("Can't test interrupts in Jython") - - # Test fix for PYTHON-294 -- make sure client closes its socket if it - # gets an interrupt while waiting to recv() from it. - c = client_context.rs_client - db = c.pymongo_test - - # A $where clause which takes 1.5 sec to execute - where = delay(1.5) - - # Need exactly 1 document so find() will execute its $where clause once - db.drop_collection('foo') - db.foo.insert({'_id': 1}) - - old_signal_handler = None - - try: - def interrupter(): - time.sleep(0.25) - - # Raises KeyboardInterrupt in the main thread - thread.interrupt_main() - - thread.start_new_thread(interrupter, ()) - - raised = False - try: - # Will be interrupted by a KeyboardInterrupt. - next(db.foo.find({'$where': where})) - except KeyboardInterrupt: - raised = True - - # Can't use self.assertRaises() because it doesn't catch system - # exceptions - self.assertTrue(raised, "Didn't raise expected ConnectionFailure") - - # Raises AssertionError due to PYTHON-294 -- Mongo's response to the - # previous find() is still waiting to be read on the socket, so the - # request id's don't match. - self.assertEqual( - {'_id': 1}, - next(db.foo.find()) - ) - finally: - if old_signal_handler: - signal.signal(signal.SIGALRM, old_signal_handler) - - def test_operation_failure_without_request(self): - # Ensure MongoReplicaSetClient doesn't close socket after it gets an - # error response to getLastError. PYTHON-395. - c = self._get_client() - connected(c) - pool = get_pool(c) - self.assertEqual(1, len(pool.sockets)) - old_sock_info = next(iter(pool.sockets)) - c.pymongo_test.test.drop() - c.pymongo_test.test.insert({'_id': 'foo'}) - self.assertRaises( - OperationFailure, - c.pymongo_test.test.insert, {'_id': 'foo'}) - - self.assertEqual(1, len(pool.sockets)) - new_sock_info = next(iter(pool.sockets)) - - self.assertEqual(old_sock_info, new_sock_info) - c.close() - - def test_operation_failure_with_request(self): - # Ensure MongoReplicaSetClient doesn't close socket after it gets an - # error response to getLastError. PYTHON-395. - c = self._get_client() - c.start_request() - c.pymongo_test.test.find_one() - pool = get_pool(c) - - # Client reserved a socket for this thread - self.assertTrue(isinstance(pool._get_request_state(), SocketInfo)) - - old_sock_info = pool._get_request_state() - c.pymongo_test.test.drop() - c.pymongo_test.test.insert({'_id': 'foo'}) - self.assertRaises( - OperationFailure, - c.pymongo_test.test.insert, {'_id': 'foo'}) - - # OperationFailure doesn't affect the request socket - self.assertEqual(old_sock_info, pool._get_request_state()) - c.close() - def test_nested_request(self): - client = self._get_client() + client = rs_or_single_client() connected(client) client.start_request() try: - pools = pools_from_rs_client(client) + pools = get_pools(client) self.assertTrue(client.in_request()) # Start and end request - we're still in "outer" original request @@ -837,8 +316,8 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): client.start_request() for pool in pools: - # MRSC only called start_request() once per pool, although its - # own counter is 3. + # Client only called start_request() once per pool, although + # its own counter is 3. self.assertEqual(1, pool._request_counter.get()) client.end_request() @@ -857,45 +336,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): finally: client.close() - def test_request_threads(self): - client = client_context.rs_client - - pools = pools_from_rs_client(client) - self.assertNotInRequestAndDifferentSock(client, pools) - - started_request, ended_request = threading.Event(), threading.Event() - checked_request = threading.Event() - thread_done = [False] - - # Starting a request in one thread doesn't put the other thread in a - # request - def f(): - self.assertNotInRequestAndDifferentSock(client, pools) - client.start_request() - self.assertInRequestAndSameSock(client, pools) - started_request.set() - checked_request.wait() - checked_request.clear() - self.assertInRequestAndSameSock(client, pools) - client.end_request() - self.assertNotInRequestAndDifferentSock(client, pools) - ended_request.set() - checked_request.wait() - thread_done[0] = True - - t = threading.Thread(target=f) - t.setDaemon(True) - t.start() - started_request.wait() - self.assertNotInRequestAndDifferentSock(client, pools) - checked_request.set() - ended_request.wait() - self.assertNotInRequestAndDifferentSock(client, pools) - checked_request.set() - t.join() - self.assertNotInRequestAndDifferentSock(client, pools) - self.assertTrue(thread_done[0], "Thread didn't complete") - def test_pinned_member(self): raise SkipTest("Secondary pinning not implemented in PyMongo 3") @@ -939,43 +379,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self, client, list(client.secondaries) + [client.primary], ReadPreference.NEAREST, None) - def test_alive(self): - client = client_context.rs_client - self.assertTrue(client.alive()) - - client = MongoReplicaSetClient( - 'doesnt exist', replicaSet='rs', connect=False) - - self.assertFalse(client.alive()) - - @client_context.require_auth - def test_auth_network_error(self): - # Make sure there's no semaphore leak if we get a network error - # when authenticating a new socket with cached credentials. - - # Get a client with one socket so we detect if it's leaked. - c = self._get_client(max_pool_size=1, waitQueueTimeoutMS=1) - - # Simulate an authenticate() call on a different socket. - credentials = auth._build_credentials_tuple( - 'MONGODB-CR', 'admin', db_user, db_pwd, {}) - - c._cache_credentials('test', credentials, connect=False) - - # Cause a network error on the actual socket. - connected(c) - pool = get_pool(c) - socket_info = one(pool.sockets) - socket_info.sock.close() - - # In __check_auth, the client authenticates its socket with the - # new credential, but gets a socket.error. Should be reraised as - # AutoReconnect. - self.assertRaises(AutoReconnect, c.test.collection.find_one) - - # No semaphore leak, the pool is allowed to make a new socket. - c.test.collection.find_one() - class TestReplicaSetWireVersion(MockClientTest): @@ -1017,46 +420,6 @@ class TestReplicaSetWireVersion(MockClientTest): self.assertRaises(ConfigurationError, c.db.collection.insert, {}) -# Test concurrent access to a lazily-connecting RS client. -class TestReplicaSetClientLazyConnect( - TestReplicaSetClientBase, - _TestLazyConnectMixin): - - @classmethod - def setUpClass(cls): - TestReplicaSetClientBase.setUpClass() - - def test_read_mode_secondary(self): - client = MongoReplicaSetClient( - connection_string(), replicaSet=self.name, connect=False, - read_preference=ReadPreference.SECONDARY) - - # No error. - client.pymongo_test.test_collection.find_one() - - -class TestReplicaSetClientLazyConnectBadSeeds( - TestReplicaSetClientBase, - _TestLazyConnectMixin): - - @classmethod - def setUpClass(cls): - TestReplicaSetClientBase.setUpClass() - - def _get_client(self, **kwargs): - kwargs.setdefault('connectTimeoutMS', 500) - - # Assume there are no open mongods listening on a.com, b.com, .... - bad_seeds = ['%s.com' % chr(ord('a') + i) for i in range(5)] - client = MongoReplicaSetClient( - connection_string(seeds=(bad_seeds + [pair])), - replicaSet=self.name, **kwargs) - - # In case of a slow test machine. - client._refresh_timeout_sec = 30 - return client - - class TestReplicaSetClientInternalIPs(MockClientTest): @client_context.require_connection diff --git a/test/test_ssl.py b/test/test_ssl.py index badd9b6b8..0ce8a9aac 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -32,7 +32,7 @@ except ImportError: # Python 2 from urllib import quote_plus -from pymongo import MongoClient, MongoReplicaSetClient +from pymongo import MongoClient from pymongo.errors import (ConfigurationError, ConnectionFailure, OperationFailure) @@ -65,8 +65,8 @@ MONGODB_X509_USERNAME = ( # --sslWeakCertificateValidation # Also, make sure you have 'server' as an alias for localhost in /etc/hosts # -# Note: For all tests to pass with MongoReplicaSetClient the replica -# set configuration must use 'server' for the hostname of all hosts. +# Note: For all replica set tests to pass, the replica set configuration must +# use 'server' for the hostname of all hosts. def is_server_resolvable(): """Returns True if 'server' is resolvable.""" @@ -127,15 +127,10 @@ class TestClientSSL(unittest.TestCase): # Explicit self.assertRaises(ConfigurationError, MongoClient, ssl=True) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, replicaSet='rs', ssl=True) + # Implied self.assertRaises(ConfigurationError, MongoClient, ssl_certfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_certfile=CLIENT_PEM) def test_config_ssl(self): # Tests various ssl configurations @@ -148,20 +143,6 @@ class TestClientSSL(unittest.TestCase): self.assertRaises(TypeError, MongoClient, ssl=5.5) self.assertRaises(TypeError, MongoClient, ssl=[]) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, replicaSet='rs', ssl='foo') - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_certfile=CLIENT_PEM) - self.assertRaises(TypeError, - MongoReplicaSetClient, replicaSet='rs', ssl=0) - self.assertRaises(TypeError, - MongoReplicaSetClient, replicaSet='rs', ssl=5.5) - self.assertRaises(TypeError, - MongoReplicaSetClient, replicaSet='rs', ssl=[]) - self.assertRaises(IOError, MongoClient, ssl_certfile="NoSuchFile") self.assertRaises(TypeError, MongoClient, ssl_certfile=True) self.assertRaises(TypeError, MongoClient, ssl_certfile=[]) @@ -169,19 +150,6 @@ class TestClientSSL(unittest.TestCase): self.assertRaises(TypeError, MongoClient, ssl_keyfile=True) self.assertRaises(TypeError, MongoClient, ssl_keyfile=[]) - self.assertRaises(IOError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_keyfile="NoSuchFile") - self.assertRaises(IOError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_certfile="NoSuchFile") - self.assertRaises(TypeError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_certfile=True) - # Test invalid combinations self.assertRaises(ConfigurationError, MongoClient, @@ -197,23 +165,6 @@ class TestClientSSL(unittest.TestCase): ssl_keyfile=CLIENT_PEM, ssl_certfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_keyfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_certfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_keyfile=CLIENT_PEM, - ssl_certfile=CLIENT_PEM) - class TestSSL(unittest.TestCase): @@ -235,10 +186,10 @@ class TestSSL(unittest.TestCase): client = MongoClient(host, port, ssl=True) response = client.admin.command('ismaster') if 'setName' in response: - client = MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True) + client = MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True) db = client.pymongo_ssl_test db.test.drop() @@ -261,10 +212,10 @@ class TestSSL(unittest.TestCase): client = ssl_client response = ssl_client.admin.command('ismaster') if 'setName' in response: - client = MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, ssl_certfile=CLIENT_PEM) + client = MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, ssl_certfile=CLIENT_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -287,10 +238,10 @@ class TestSSL(unittest.TestCase): client = ssl_client response = ssl_client.admin.command('ismaster') if 'setName' in response: - client = MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl_certfile=CLIENT_PEM) + client = MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl_certfile=CLIENT_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -325,13 +276,13 @@ class TestSSL(unittest.TestCase): raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") - client = MongoReplicaSetClient('server', - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, - ssl_certfile=CLIENT_PEM, - ssl_cert_reqs=ssl.CERT_REQUIRED, - ssl_ca_certs=CA_PEM) + client = MongoClient('server', + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, + ssl_certfile=CLIENT_PEM, + ssl_cert_reqs=ssl.CERT_REQUIRED, + ssl_ca_certs=CA_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -367,13 +318,13 @@ class TestSSL(unittest.TestCase): raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") - client = MongoReplicaSetClient('server', - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, - ssl_certfile=CLIENT_PEM, - ssl_cert_reqs=ssl.CERT_OPTIONAL, - ssl_ca_certs=CA_PEM) + client = MongoClient('server', + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, + ssl_certfile=CLIENT_PEM, + ssl_cert_reqs=ssl.CERT_OPTIONAL, + ssl_ca_certs=CA_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -405,13 +356,13 @@ class TestSSL(unittest.TestCase): if 'setName' in response: try: - MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, - ssl_certfile=CLIENT_PEM, - ssl_cert_reqs=ssl.CERT_REQUIRED, - ssl_ca_certs=CA_PEM) + MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, + ssl_certfile=CLIENT_PEM, + ssl_cert_reqs=ssl.CERT_REQUIRED, + ssl_ca_certs=CA_PEM) self.fail("Invalid hostname should have failed") except CertificateError: pass diff --git a/test/test_threads.py b/test/test_threads.py index afd6374ee..f910a66a4 100644 --- a/test/test_threads.py +++ b/test/test_threads.py @@ -17,9 +17,11 @@ import threading import traceback -from test import unittest, client_context, pair -from test.utils import joinall, remove_all_users, RendezvousThread -from test.test_client import get_client +from test import unittest, client_context, IntegrationTest +from test.utils import (joinall, + remove_all_users, + RendezvousThread, + rs_or_single_client) from test.utils import get_pool from pymongo.pool import SocketInfo, _closed from pymongo.errors import AutoReconnect, OperationFailure @@ -163,30 +165,9 @@ class FindPauseFind(RendezvousThread): assert self.request_sock != pool._get_request_state().sock -class BaseTestThreads(object): - """ - Base test class for TestThreads and TestThreadsReplicaSet. (This - is not itself a unittest.TestCase, otherwise it'd be run twice -- - once when unittest imports this module, and once when unittest - imports test_threads_replica_set_connection.py, which imports this - module.) - """ +class TestThreads(IntegrationTest): def setUp(self): - self.db = self._get_client().pymongo_test - - def tearDown(self): - # Clear client reference so that RSC's monitor thread - # dies. - self.db = None - - def _get_client(self): - """ - Intended for overriding in TestThreadsReplicaSet. This method - returns a MongoClient here, and a MongoReplicaSetClient in - test_threads_replica_set_connection.py. - """ - # Regular test client - return get_client(pair) + self.db = client_context.rs_or_standalone_client.pymongo_test def test_threading(self): self.db.drop_collection("test") @@ -254,7 +235,7 @@ class BaseTestThreads(object): # # If we've fixed PYTHON-345, then only one AutoReconnect is raised, # and all the threads get new request sockets. - cx = get_client(pair) + cx = rs_or_single_client() cx.start_request() collection = cx.db.pymongo_test @@ -305,37 +286,22 @@ class BaseTestThreads(object): self.assertTrue(t.passed, "%s threw exception" % t) -class BaseTestThreadsAuth(object): - """ - Base test class for TestThreadsAuth and TestThreadsAuthReplicaSet. (This is - not itself a unittest.TestCase, otherwise it'd be run twice -- once when - unittest imports this module, and once when unittest imports - test_threads_replica_set_connection.py, which imports this module.) - """ +class TestThreadsAuth(IntegrationTest): @classmethod @client_context.require_auth def setUpClass(cls): - pass - - def _get_client(self): - """ - Intended for overriding in TestThreadsAuthReplicaSet. This method - returns a MongoClient here, and a MongoReplicaSetClient in - test_threads_replica_set_connection.py. - """ - # Regular test client - return get_client(pair) + super(TestThreadsAuth, cls).setUpClass() def setUp(self): - client = self._get_client() - self.client = client + self.client = rs_or_single_client() self.client.admin.add_user('admin-user', 'password', roles=['clusterAdmin', 'dbAdminAnyDatabase', 'readWriteAnyDatabase', 'userAdminAnyDatabase']) - # client returned from self._get_client() may already be authenticated - # to get around restricted localhost exception in MongoDB >= 2.7.1. + + # client is already authenticated to get around restricted + # localhost exception in MongoDB >= 2.7.1. self.client.admin.logout() self.client.admin.authenticate("admin-user", "password") self.client.auth_test.add_user("test-user", "password", @@ -348,18 +314,13 @@ class BaseTestThreadsAuth(object): self.client.drop_database('auth_test') self.client.admin.remove_user('admin-user') self.client.admin.logout() - # Clear client reference so that RSC's monitor thread - # dies. - self.client = None def test_auto_auth_login(self): - client = self._get_client() + client = self.client client.admin.logout() self.assertRaises(OperationFailure, client.auth_test.test.find_one) # Admin auth - client = self._get_client() - client.admin.logout() client.admin.authenticate("admin-user", "password") nthreads = 10 @@ -375,7 +336,6 @@ class BaseTestThreadsAuth(object): self.assertTrue(t.success) # Database-specific auth - client = self._get_client() client.admin.logout() client.auth_test.authenticate("test-user", "password") @@ -390,12 +350,6 @@ class BaseTestThreadsAuth(object): for t in threads: self.assertTrue(t.success) -class TestThreads(BaseTestThreads, unittest.TestCase): - pass - -class TestThreadsAuth(BaseTestThreadsAuth, unittest.TestCase): - pass - if __name__ == "__main__": unittest.main() diff --git a/test/test_threads_replica_set_client.py b/test/test_threads_replica_set_client.py deleted file mode 100644 index f67a5b073..000000000 --- a/test/test_threads_replica_set_client.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2011-2014 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test that pymongo is thread safe.""" - -from test import unittest -from test.test_threads import BaseTestThreads, BaseTestThreadsAuth -from test.test_replica_set_client import TestReplicaSetClientBase, pair -from test.utils import get_rs_client - - -class TestThreadsReplicaSet(TestReplicaSetClientBase, BaseTestThreads): - def setUp(self): - """ - Prepare to test all the same things that TestThreads tests, but do it - with a replica-set client - """ - TestReplicaSetClientBase.setUp(self) - BaseTestThreads.setUp(self) - - def tearDown(self): - TestReplicaSetClientBase.tearDown(self) - BaseTestThreads.tearDown(self) - - def _get_client(self, **kwargs): - return TestReplicaSetClientBase._get_client(self, **kwargs) - - -class TestThreadsAuthReplicaSet(TestReplicaSetClientBase, BaseTestThreadsAuth): - - @classmethod - def setUpClass(cls): - TestReplicaSetClientBase.setUpClass() - BaseTestThreadsAuth.setUpClass() - - @classmethod - def tearDownClass(cls): - TestReplicaSetClientBase.tearDownClass() - - def setUp(self): - """ - Prepare to test all the same things that TestThreads tests, but do it - with a replica-set client - """ - TestReplicaSetClientBase.setUp(self) - BaseTestThreadsAuth.setUp(self) - - def tearDown(self): - TestReplicaSetClientBase.tearDown(self) - BaseTestThreadsAuth.tearDown(self) - - def _get_client(self): - """ - Override TestThreadsAuth, so its tests run on a MongoReplicaSetClient - instead of a regular MongoClient. - """ - return get_rs_client(pair, replicaSet=self.name) - - -if __name__ == "__main__": - suite = unittest.TestSuite([ - unittest.makeSuite(TestThreadsReplicaSet), - unittest.makeSuite(TestThreadsAuthReplicaSet) - ]) - unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/test/utils.py b/test/utils.py index 68bbb40cd..1f8ec9248 100644 --- a/test/utils.py +++ b/test/utils.py @@ -15,20 +15,24 @@ """Utilities for testing pymongo """ +import contextlib import os import struct import sys import threading import time +import warnings -from pymongo import MongoClient, MongoReplicaSetClient +from pymongo import MongoClient from pymongo.errors import AutoReconnect, OperationFailure from pymongo.pool import NO_REQUEST, NO_SOCKET_YET, SocketInfo from pymongo.server_selectors import (any_server_selector, writable_server_selector) from test import (client_context, db_user, - db_pwd) + db_pwd, + host, + port) from test.version import Version @@ -39,11 +43,27 @@ def get_client(*args, **kwargs): return client -def get_rs_client(*args, **kwargs): - client = MongoReplicaSetClient(*args, **kwargs) - if client_context.auth_enabled and kwargs.get("_connect", True): - client.admin.authenticate(db_user, db_pwd) - return client +def rs_or_single_client(h=host, p=port, **kwargs): + """Connect to the replica set if there is one, otherwise the standalone. + + Authenticates if necessary. + """ + if client_context.setname: + return get_client(h, p, replicaSet=client_context.setname, **kwargs) + else: + return get_client(h, p, **kwargs) + + +def rs_or_single_client_noauth(h=host, p=port, **kwargs): + """Connect to the replica set if there is one, otherwise the standalone. + + Like rs_or_single_client, but does not authenticate. + """ + # Just call MongoClient(), not get_client() which does auth. + if client_context.setname: + return MongoClient(h, p, replicaSet=client_context.setname, **kwargs) + else: + return MongoClient(h, p, **kwargs) # No functools in Python 2.4 @@ -160,7 +180,12 @@ def joinall(threads): def connected(client): """Convenience to wait for a newly-constructed client to connect.""" - client.admin.command('ismaster') # Force connection. + with warnings.catch_warnings(): + # Ignore warning that "ismaster" is always routed to primary even + # if client's read preference isn't PRIMARY. + warnings.simplefilter("ignore", UserWarning) + client.admin.command('ismaster') # Force connection. + return client def wait_until(predicate, success_description, timeout=10): @@ -187,12 +212,11 @@ def enable_text_search(client): client.admin.command( 'setParameter', textSearchEnabled=True) - if isinstance(client, MongoReplicaSetClient): - for host, port in client.secondaries: - client = MongoClient(host, port) - if client_context.auth_enabled: - client.admin.authenticate(db_user, db_pwd) - client.admin.command('setParameter', textSearchEnabled=True) + for host, port in client.secondaries: + client = MongoClient(host, port) + if client_context.auth_enabled: + client.admin.authenticate(db_user, db_pwd) + client.admin.command('setParameter', textSearchEnabled=True) def assertRaisesExactly(cls, fn, *args, **kwargs): """ @@ -208,6 +232,11 @@ def assertRaisesExactly(cls, fn, *args, **kwargs): else: raise AssertionError("%s not raised" % cls) +@contextlib.contextmanager +def ignore_deprecations(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + yield class RendezvousThread(threading.Thread): """A thread that starts and pauses at a rendezvous point before resuming. @@ -306,20 +335,22 @@ class RendezvousThread(threading.Thread): self.passed = True def read_from_which_host( - rsc, + client, pref, tag_sets=None, secondary_acceptable_latency_ms=None ): - """Read from a MongoReplicaSetClient with the given Read Preference mode, - tags, and acceptable latency. Return the 'host:port' which was read from. + """Read from a client with the given Read Preference. + + Return the 'host:port' which was read from. :Parameters: - - `rsc`: A MongoReplicaSetClient + - `client`: A MongoClient - `mode`: A ReadPreference - `tag_sets`: List of dicts of tags for data-center-aware reads + - `secondary_acceptable_latency_ms`: Size of latency window """ - db = rsc.pymongo_test + db = client.pymongo_test if isinstance(tag_sets, dict): tag_sets = [tag_sets] @@ -342,27 +373,27 @@ def read_from_which_host( except AutoReconnect: return None -def assertReadFrom(testcase, rsc, member, *args, **kwargs): +def assertReadFrom(testcase, client, member, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from the expected replica-set member. :Parameters: - `testcase`: A unittest.TestCase - - `rsc`: A MongoReplicaSetClient + - `client`: A MongoClient - `member`: A host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads """ for _ in range(10): - testcase.assertEqual(member, read_from_which_host(rsc, *args, **kwargs)) + testcase.assertEqual(member, read_from_which_host(client, *args, **kwargs)) -def assertReadFromAll(testcase, rsc, members, *args, **kwargs): +def assertReadFromAll(testcase, client, members, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from all members in a set, and only members in that set. :Parameters: - `testcase`: A unittest.TestCase - - `rsc`: A MongoReplicaSetClient + - `client`: A MongoClient - `members`: Sequence of host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads @@ -370,18 +401,18 @@ def assertReadFromAll(testcase, rsc, members, *args, **kwargs): members = set(members) used = set() for _ in range(100): - used.add(read_from_which_host(rsc, *args, **kwargs)) + used.add(read_from_which_host(client, *args, **kwargs)) testcase.assertEqual(members, used) def get_pool(client): + """Get the standalone, primary, or mongos pool.""" topology = client._get_topology() server = topology.select_server(writable_server_selector) return server.pool -def pools_from_rs_client(client): - """Get Pool instances from a MongoReplicaSetClient. - """ +def get_pools(client): + """Get all pools.""" return [ server.pool for server in client._get_topology().select_servers(any_server_selector)] @@ -479,7 +510,7 @@ def lazy_client_trial(reset, target, test, get_client): try: for i in range(NTRIALS): reset(collection) - lazy_client = get_client(connect=False) + lazy_client = get_client() lazy_collection = lazy_client.pymongo_test.test run_threads(lazy_collection, target) test(lazy_collection) @@ -495,9 +526,8 @@ def lazy_client_trial(reset, target, test, get_client): class _TestLazyConnectMixin(object): """Test concurrent operations on a lazily-connecting client. - Inherit from this class and from unittest.TestCase, and override - _get_client(self, **kwargs), for testing a lazily-connecting - client, i.e. a client initialized with connect=False. + Inherit from this class and from IntegrationTest, and override + self._get_client() to return a client initialized with connect=False. """ NTRIALS = 5 NTHREADS = 10 @@ -572,7 +602,7 @@ class _TestLazyConnectMixin(object): def test_max_bson_size(self): # Client should have sane defaults before connecting, and should update # its configuration once connected. - c = self._get_client(connect=False) + c = self._get_client() self.assertEqual(16 * (1024 ** 2), c.max_bson_size) self.assertEqual(2 * c.max_bson_size, c.max_message_size)