diff --git a/test/__init__.py b/test/__init__.py index 216814d51..ab5f6b0a3 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -16,7 +16,10 @@ """ import os +import socket import sys +from pymongo.mongo_client import _partition_node + if sys.version_info[:2] == (2, 6): import unittest2 as unittest from unittest2 import SkipTest @@ -34,9 +37,9 @@ from bson.py3compat import _unicode, reraise from pymongo import common from test.version import Version -# hostnames retrieved by MongoReplicaSetClient from isMaster will be of unicode -# type in Python 2, so ensure these hostnames are unicodes, too. It makes tests -# like `test_repr` predictable. +# hostnames retrieved from isMaster will be of unicode type in Python 2, +# so ensure these hostnames are unicodes, too. It makes tests like +# `test_repr` predictable. host = _unicode(os.environ.get("DB_IP", 'localhost')) port = int(os.environ.get("DB_PORT", 27017)) pair = '%s:%d' % (host, port) @@ -86,6 +89,7 @@ class ClientContext(object): self.connected = False self.ismaster = {} self.w = None + self.nodes = set() self.setname = None self.rs_client = None self.cmd_line = None @@ -93,6 +97,9 @@ class ClientContext(object): self.auth_enabled = False self.test_commands_enabled = False self.is_mongos = False + self.is_rs = False + self.has_ipv6 = False + try: with client_knobs(server_wait_time=0.1): client = pymongo.MongoClient(host, port) @@ -105,12 +112,20 @@ class ClientContext(object): self.connected = True self.ismaster = self.client.admin.command('ismaster') self.w = len(self.ismaster.get("hosts", [])) or 1 + self.nodes = set([(host, port)]) self.setname = self.ismaster.get('setName', '') self.rs_client = None self.version = Version.from_client(self.client) if self.setname: - self.rs_client = pymongo.MongoReplicaSetClient( + self.is_rs = True + self.rs_client = pymongo.MongoClient( pair, replicaSet=self.setname) + + self.nodes = set([_partition_node(node) + for node in self.ismaster.get('hosts', [])]) + + self.rs_or_standalone_client = self.rs_client or self.client + try: self.cmd_line = self.client.admin.command('getCmdLineOpts') except pymongo.errors.OperationFailure as e: @@ -141,6 +156,7 @@ class ClientContext(object): self.test_commands_enabled = ('testCommandsEnabled=1' in self.cmd_line['argv']) self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid') + self.has_ipv6 = self._server_started_with_ipv6() def _check_user_provided(self): try: @@ -172,6 +188,27 @@ class ClientContext(object): argv = self.cmd_line['argv'] return '--auth' in argv or '--keyFile' in argv + def _server_started_with_ipv6(self): + if not socket.has_ipv6: + return False + + if 'parsed' in self.cmd_line: + if not self.cmd_line['parsed'].get('net', {}).get('ipv6'): + return False + else: + if '--ipv6' not in self.cmd_line['argv']: + return False + + # The server was started with --ipv6. Is there an IPv6 route to it? + try: + for info in socket.getaddrinfo(host, port): + if info[0] == socket.AF_INET6: + return True + except socket.error: + pass + + return False + def _require(self, condition, msg, func=None): def make_wrapper(f): @wraps(f) @@ -225,10 +262,23 @@ class ClientContext(object): def require_replica_set(self, func): """Run a test only if the client is connected to a replica set.""" - return self._require(self.rs_client is not None, + return self._require(self.is_rs, "Not connected to a replica set", func=func) + def require_no_replica_set(self, func): + """Run a test if the client is *not* connected to a replica set.""" + return self._require( + not self.is_rs, + "Connected to a replica set, not a standalone mongod", + func=func) + + def require_ipv6(self, func): + """Run a test only if the client can connect to a server via IPv6.""" + return self._require(self.has_ipv6, + "No IPv6", + func=func) + def require_no_mongos(self, func): """Run a test only if the client is not connected to a mongos.""" return self._require(not self.is_mongos, diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index 673864753..2dea6d6d5 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -15,7 +15,8 @@ Test Matrix PyMongo should be tested with several versions of mod_wsgi and a selection of Python versions. Each combination of mod_wsgi and Python version should -be tested with both MongoClient and MongoReplicaSetClient. +be tested with a standalone and a replica set. ``mod_wsgi_test.wsgi`` +detects if the deployment is a replica set and connects to the whole set. Setup ----- @@ -56,25 +57,23 @@ Run the test Run the included ``test_client.py`` script:: python test/mod_wsgi_test/test_client.py -n 2500 -t 100 parallel \ - http://localhost/${mongodb_configuration}/${WORKSPACE} + http://localhost/${WORKSPACE} ...where the "n" argument is the total number of requests to make to Apache, -and "t" specifies the number of threads. ``mongodb_configuration`` should be -"single_server" or "replica_set", depending on how you started mongod. -``WORKSPACE`` is the location of the PyMongo checkout. +and "t" specifies the number of threads. ``WORKSPACE`` is the location of +the PyMongo checkout. Run this script again with different arguments to make serial requests:: - python ${WORKSPACE}/test/mod_wsgi_test/test_client.py -n 25000 serial \ - http://localhost/${mongodb_configuration}/${WORKSPACE} + python test/mod_wsgi_test/test_client.py -n 25000 serial \ + http://localhost/${WORKSPACE} The ``test_client.py`` script merely makes HTTP requests to Apache. Its exit code is non-zero if any of its requests fails, for example with an HTTP 500. -The core of the test is in the WSGI scripts: -``mod_wsgi_test_single_server.wsgi`` and ``mod_wsgi_test_replica_set.wsgi``. -These scripts insert some documents into MongoDB at startup, then query +The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`. +This script inserts some documents into MongoDB at startup, then queries documents for each HTTP request. If PyMongo is leaking connections and "n" is much greater than the ulimit, @@ -85,5 +84,5 @@ Automation At MongoDB, Inc. we use a Jenkins job that tests each combination in the matrix. The job copies the appropriate version of ``mod_wsgi.so`` into -place, sets up Apache, and runs ``test_client.py`` with the proper -arguments. +place, sets up Apache, starts a single server or replica set, +and runs ``test_client.py`` with the proper arguments. diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index 0b8c714af..8815f0b34 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -28,12 +28,9 @@ WSGISocketPrefix /tmp/ WSGIProcessGroup mod_wsgi_test # For the convienience of unittests, rather than hard-code the location of - # mod_wsgi_test_single_server.wsgi and mod_wsgi_test_replica_set.wsgi, - # include it in the URL, so - # http://localhost/single_server/location-of-pymongo-checkout will work: + # mod_wsgi_test.wsgi, include it in the URL, so + # http://localhost/location-of-pymongo-checkout will work: - WSGIScriptAliasMatch ^/single_server(.+) $1/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi - - WSGIScriptAliasMatch ^/replica_set(.+) $1/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi + WSGIScriptAliasMatch ^/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.wsgi diff --git a/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi b/test/mod_wsgi_test/mod_wsgi_test.wsgi similarity index 89% rename from test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi rename to test/mod_wsgi_test/mod_wsgi_test.wsgi index 9285561ca..74aca931b 100644 --- a/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi +++ b/test/mod_wsgi_test/mod_wsgi_test.wsgi @@ -28,6 +28,12 @@ import pymongo from pymongo.mongo_client import MongoClient client = MongoClient() + +# If the deployment is a replica set, connect to the whole set. +replica_set_name = client.admin.command('ismaster').get('setName') +if replica_set_name: + client = MongoClient(replicaSet=replica_set_name) + collection = client.test.test ndocs = 20 diff --git a/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi b/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi deleted file mode 100644 index a43dff146..000000000 --- a/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2012-2014 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Minimal test of PyMongo in a WSGI application with MongoReplicaSetClient, - see bug PYTHON-353. -""" - -import os -import sys - -this_path = os.path.dirname(os.path.join(os.getcwd(), __file__)) - -# Location of PyMongo checkout -repository_path = os.path.normpath(os.path.join(this_path, '..', '..')) -sys.path.insert(0, repository_path) - -import pymongo -from pymongo.mongo_replica_set_client import MongoReplicaSetClient - -client = MongoReplicaSetClient(replicaSet='repl0') -collection = client.test.test - -ndocs = 20 - -collection.drop() -collection.insert([{'i': i} for i in range(ndocs)]) -client.disconnect() # Discard main thread's request socket. - -try: - from mod_wsgi import version as mod_wsgi_version -except: - mod_wsgi_version = None - - -def application(environ, start_response): - # Requests are part of the PYTHON-353 pathology. - client.start_request() - results = list(collection.find().batch_size(10)) - assert len(results) == ndocs - output = 'python %s, mod_wsgi %s, pymongo %s' % ( - sys.version, mod_wsgi_version, pymongo.version) - response_headers = [('Content-Length', str(len(output)))] - start_response('200 OK', response_headers) - return [output] diff --git a/test/test_auth.py b/test/test_auth.py index da7e217f6..d66f926af 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -26,7 +26,7 @@ except ImportError: sys.path[0:0] = [""] -from pymongo import MongoClient, MongoReplicaSetClient +from pymongo import MongoClient from pymongo.auth import HAVE_KERBEROS, _build_credentials_tuple from pymongo.errors import OperationFailure, ConfigurationError from pymongo.read_preferences import ReadPreference @@ -114,9 +114,9 @@ class TestGSSAPI(unittest.TestCase): set_name = client.admin.command('ismaster').get('setName') if set_name: - client = MongoReplicaSetClient(GSSAPI_HOST, - port=GSSAPI_PORT, - replicaSet=set_name) + client = MongoClient(GSSAPI_HOST, + port=GSSAPI_PORT, + replicaSet=set_name) # Without gssapiServiceName self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) @@ -124,7 +124,7 @@ class TestGSSAPI(unittest.TestCase): uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertTrue(client.database_names()) # With gssapiServiceName @@ -137,7 +137,7 @@ class TestGSSAPI(unittest.TestCase): GSSAPI_HOST, GSSAPI_PORT, str(set_name))) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertTrue(client.database_names()) def test_gssapi_threaded(self): @@ -160,9 +160,9 @@ class TestGSSAPI(unittest.TestCase): set_name = client.admin.command('ismaster').get('setName') if set_name: preference = ReadPreference.SECONDARY - client = MongoReplicaSetClient(GSSAPI_HOST, - replicaSet=set_name, - read_preference=preference) + client = MongoClient(GSSAPI_HOST, + replicaSet=set_name, + read_preference=preference) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) self.assertTrue(client.test.command('dbstats')) @@ -201,9 +201,9 @@ class TestSASLPlain(unittest.TestCase): set_name = client.admin.command('ismaster').get('setName') if set_name: - client = MongoReplicaSetClient(SASL_HOST, - port=SASL_PORT, - replicaSet=set_name) + client = MongoClient(SASL_HOST, + port=SASL_PORT, + replicaSet=set_name) self.assertTrue(client.ldap.authenticate(SASL_USER, SASL_PASS, SASL_DB, 'PLAIN')) client.ldap.test.find_one() @@ -213,7 +213,7 @@ class TestSASLPlain(unittest.TestCase): quote_plus(SASL_PASS), SASL_HOST, SASL_PORT, SASL_DB, str(set_name))) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) client.ldap.test.find_one() def test_sasl_plain_bad_credentials(self): @@ -283,8 +283,8 @@ class TestSCRAMSHA1(unittest.TestCase): client.pymongo_test.command('dbstats') if self.set_name: - client = MongoReplicaSetClient(host, port, - replicaSet='%s' % (self.set_name,)) + client = MongoClient(host, port, + replicaSet='%s' % (self.set_name,)) self.assertTrue(client.pymongo_test.authenticate( 'user', 'pass', mechanism='SCRAM-SHA-1')) client.pymongo_test.command('dbstats') @@ -292,7 +292,7 @@ class TestSCRAMSHA1(unittest.TestCase): uri = ('mongodb://user:pass' '@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1' '&replicaSet=%s' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) client.pymongo_test.command('dbstats') client.read_preference = ReadPreference.SECONDARY client.pymongo_test.command('dbstats') @@ -321,8 +321,7 @@ class TestAuthURIOptions(unittest.TestCase): # GLE requires authentication. client.admin.authenticate('admin', 'pass') # Make sure the admin user is replicated after calling add_user - # above. This avoids a race in the MRSC tests below. Adding a - # user is just an insert into system.users. + # above. This avoids a race in the replica set tests below. client.admin.command('getLastError', w=len(response['hosts'])) self.client = client @@ -342,7 +341,7 @@ class TestAuthURIOptions(unittest.TestCase): if self.set_name: uri = ('mongodb://admin:pass' '@%s:%d/?replicaSet=%s' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertTrue(client.admin.command('dbstats')) client.read_preference = ReadPreference.SECONDARY self.assertTrue(client.admin.command('dbstats')) @@ -356,7 +355,7 @@ class TestAuthURIOptions(unittest.TestCase): if self.set_name: uri = ('mongodb://user:pass@%s:%d' '/pymongo_test?replicaSet=%s' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertRaises(OperationFailure, client.admin.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) @@ -374,7 +373,7 @@ class TestAuthURIOptions(unittest.TestCase): if self.set_name: uri = ('mongodb://user:pass@%s:%d/pymongo_test2?replicaSet=' '%s;authSource=pymongo_test' % (host, port, self.set_name)) - client = MongoReplicaSetClient(uri) + client = MongoClient(uri) self.assertRaises(OperationFailure, client.pymongo_test2.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) diff --git a/test/test_client.py b/test/test_client.py index c8e89a7b2..b376fc648 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -59,16 +59,18 @@ from test import (client_context, from test.pymongo_mocks import MockClient from test.utils import (assertRaisesExactly, delay, - get_client, + ignore_deprecations, remove_all_users, server_is_master_with_slave, TestRequestMixin, _TestLazyConnectMixin, - lazy_client_trial, get_pool, one, connected, - wait_until) + wait_until, + rs_or_single_client, + rs_or_single_client_noauth, + lazy_client_trial) class ClientUnitTest(unittest.TestCase, TestRequestMixin): @@ -129,7 +131,7 @@ class TestClient(IntegrationTest, TestRequestMixin): @classmethod def setUpClass(cls): super(TestClient, cls).setUpClass() - cls.client = client_context.client + cls.client = client_context.rs_or_standalone_client def test_constants(self): # Set bad defaults. @@ -156,13 +158,16 @@ class TestClient(IntegrationTest, TestRequestMixin): self.fail(self._formatMessage(msg, standardMsg)) def test_init_disconnected(self): - c = MongoClient(host, port, connect=False) + c = rs_or_single_client(connect=False) self.assertIsInstance(c.is_primary, bool) self.assertIsInstance(c.is_mongos, bool) self.assertIsInstance(c.max_pool_size, int) self.assertIsInstance(c.nodes, frozenset) - self.assertEqual(dict, c.get_document_class()) + + with ignore_deprecations(): + self.assertEqual(dict, c.get_document_class()) + self.assertIsInstance(c.tz_aware, bool) self.assertIsInstance(c.max_bson_size, int) self.assertIsInstance(c.min_wire_version, int) @@ -170,6 +175,8 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertIsInstance(c.max_write_batch_size, int) self.assertEqual(None, c.host) self.assertEqual(None, c.port) + self.assertFalse(c.primary) + self.assertFalse(c.secondaries) c.pymongo_test.command('ismaster') # Auto-connect. self.assertEqual(host, c.host) @@ -193,13 +200,11 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) def test_equality(self): - c = connected(MongoClient(host, port)) - - # ClientContext.client is constructed as MongoClient(host, port) - self.assertEqual(self.client, c) + c = connected(rs_or_single_client()) + self.assertEqual(client_context.rs_or_standalone_client, c) # Explicitly test inequality - self.assertFalse(self.client != c) + self.assertFalse(client_context.rs_or_standalone_client != c) def test_host_w_port(self): with client_knobs(server_wait_time=0.01): @@ -215,7 +220,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_getters(self): self.assertEqual(self.client.host, host) self.assertEqual(self.client.port, port) - self.assertEqual(set([(host, port)]), self.client.nodes) + self.assertEqual(client_context.nodes, self.client.nodes) def test_database_names(self): self.client.pymongo_test.test.save({"dummy": u("object")}) @@ -298,7 +303,7 @@ class TestClient(IntegrationTest, TestRequestMixin): c.drop_database("pymongo_test1") c.admin.add_user("admin", "password") - auth_c = MongoClient(host, port) + auth_c = rs_or_single_client_noauth() auth_c.admin.authenticate("admin", "password") try: auth_c.pymongo_test.add_user("mike", "password") @@ -339,7 +344,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_from_uri(self): self.assertEqual( self.client, - connected(MongoClient("mongodb://%s:%d" % (host, port)))) + connected(rs_or_single_client("mongodb://%s:%d" % (host, port)))) @client_context.require_auth def test_auth_from_uri(self): @@ -349,27 +354,29 @@ class TestClient(IntegrationTest, TestRequestMixin): "user", "pass", roles=['userAdmin', 'readWrite']) with self.assertRaises(OperationFailure): - connected(MongoClient("mongodb://a:b@%s:%d" % (host, port))) + connected(rs_or_single_client_noauth( + "mongodb://a:b@%s:%d" % (host, port))) # No error. - connected(MongoClient("mongodb://admin:pass@%s:%d" % (host, port))) + connected(rs_or_single_client_noauth( + "mongodb://admin:pass@%s:%d" % (host, port))) # Wrong database. uri = "mongodb://admin:pass@%s:%d/pymongo_test" % (host, port) with self.assertRaises(OperationFailure): - connected(MongoClient(uri)) + connected(rs_or_single_client_noauth(uri)) # No error. - connected(MongoClient( + connected(rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port))) # Auth with lazy connection. - MongoClient( + rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port), connect=False).pymongo_test.test.find_one() # Wrong password. - bad_client = MongoClient( + bad_client = rs_or_single_client_noauth( "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port), connect=False) @@ -387,8 +394,8 @@ class TestClient(IntegrationTest, TestRequestMixin): self.client.pymongo_test.add_user('user2', 'pass', roles=['readWrite']) try: - client = MongoClient("mongodb://user1:pass@%s:%d/pymongo_test" % ( - host, port)) + client = rs_or_single_client_noauth( + "mongodb://user1:pass@%s:%d/pymongo_test" % (host, port)) client.pymongo_test.test.find_one() with self.assertRaises(OperationFailure): @@ -413,9 +420,8 @@ class TestClient(IntegrationTest, TestRequestMixin): @client_context.require_auth def test_lazy_auth_raises_operation_failure(self): - lazy_client = MongoClient( - "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port), - connect=False) + lazy_client = rs_or_single_client( + "mongodb://user:wrong@%s/pymongo_test" % host, connect=False) assertRaisesExactly( OperationFailure, lazy_client.test.collection.find_one) @@ -495,7 +501,7 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) - c = get_client(pair, document_class=SON) + c = rs_or_single_client(document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) @@ -511,34 +517,34 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertRaises(DeprecationWarning, c.get_document_class) def test_timeouts(self): - client = MongoClient(host, port, connectTimeoutMS=10500) + client = rs_or_single_client(connectTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.connect_timeout) - client = MongoClient(host, port, socketTimeoutMS=10500) + client = rs_or_single_client(socketTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.socket_timeout) def test_socket_timeout_ms_validation(self): - c = get_client(pair, socketTimeoutMS=10 * 1000) + c = rs_or_single_client(socketTimeoutMS=10 * 1000) self.assertEqual(10, get_pool(c).opts.socket_timeout) - c = connected(get_client(pair, socketTimeoutMS=None)) + c = connected(rs_or_single_client(socketTimeoutMS=None)) self.assertEqual(None, get_pool(c).opts.socket_timeout) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS=0) + rs_or_single_client, socketTimeoutMS=0) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS=-1) + rs_or_single_client, socketTimeoutMS=-1) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS=1e10) + rs_or_single_client, socketTimeoutMS=1e10) self.assertRaises(ConfigurationError, - get_client, pair, socketTimeoutMS='foo') + rs_or_single_client, socketTimeoutMS='foo') def test_socket_timeout(self): no_timeout = self.client timeout_sec = 1 - timeout = get_client(pair, socketTimeoutMS=1000 * timeout_sec) + timeout = rs_or_single_client(socketTimeoutMS=1000 * timeout_sec) no_timeout.pymongo_test.drop_collection("test") no_timeout.pymongo_test.test.insert({"x": 1}) @@ -553,11 +559,11 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertRaises(NetworkTimeout, get_x, timeout.pymongo_test) def test_waitQueueTimeoutMS(self): - client = MongoClient(host, port, waitQueueTimeoutMS=2000) + client = rs_or_single_client(waitQueueTimeoutMS=2000) self.assertEqual(get_pool(client).opts.wait_queue_timeout, 2) def test_waitQueueMultiple(self): - client = MongoClient(host, port, max_pool_size=3, waitQueueMultiple=2) + client = rs_or_single_client(max_pool_size=3, waitQueueMultiple=2) pool = get_pool(client) self.assertEqual(pool.opts.wait_queue_multiple, 2) self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6) @@ -565,7 +571,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_tz_aware(self): self.assertRaises(ConfigurationError, MongoClient, tz_aware='foo') - aware = get_client(pair, tz_aware=True) + aware = rs_or_single_client(tz_aware=True) naive = self.client aware.pymongo_test.drop_collection("test") @@ -578,22 +584,18 @@ class TestClient(IntegrationTest, TestRequestMixin): aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), naive.pymongo_test.test.find_one()["x"]) + @client_context.require_ipv6 def test_ipv6(self): - with client_knobs(server_wait_time=0.01): - try: - connected(MongoClient("[::1]")) - except: - # Either mongod was started without --ipv6 - # or the OS doesn't support it (or both). - raise SkipTest("No IPv6") - if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%s[::1]:%d" % (auth_str, port) - client = MongoClient(uri) + if client_context.is_rs: + uri += '/?replicaSet=' + client_context.setname + + client = rs_or_single_client_noauth(uri) client.pymongo_test.test.save({"dummy": u("object")}) client.pymongo_test_bernie.test.save({"dummy": u("object")}) @@ -627,7 +629,7 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertFalse(locked) def test_contextlib(self): - client = get_client(pair) + client = rs_or_single_client(pair) client.pymongo_test.drop_collection("test") client.pymongo_test.test.insert({"foo": "bar"}) @@ -677,33 +679,6 @@ class TestClient(IntegrationTest, TestRequestMixin): self.assertNoRequest(pool) self.assertDifferentSock(pool) - def test_nested_request(self): - pool = get_pool(self.client) - self.assertFalse(self.client.in_request()) - - # Start and end request - self.client.start_request() - self.assertInRequestAndSameSock(self.client, pool) - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - - # Double-nesting - self.client.start_request() - self.client.start_request() - self.client.end_request() - self.assertInRequestAndSameSock(self.client, pool) - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - - # Extra end_request calls have no effect - count stays at zero - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - - self.client.start_request() - self.assertInRequestAndSameSock(self.client, pool) - self.client.end_request() - self.assertNotInRequestAndDifferentSock(self.client, pool) - def test_request_threads(self): client = self.client pool = get_pool(client) @@ -810,7 +785,7 @@ class TestClient(IntegrationTest, TestRequestMixin): def test_operation_failure_with_request(self): # Ensure MongoClient doesn't close socket after it gets an error # response to getLastError. PYTHON-395. - c = get_client(pair) + c = rs_or_single_client(pair) c.start_request() pool = get_pool(c) @@ -854,32 +829,21 @@ class TestClient(IntegrationTest, TestRequestMixin): with self.assertRaises(CursorNotFound): list(cursor) - @client_context.require_replica_set - def test_replica_set(self): - name = client_context.setname - connected(MongoClient(host, port, replicaSet=name)) # No error. - - with client_knobs(server_wait_time=0.01): - client = MongoClient(host, port, replicaSet='bad' + name) - - with self.assertRaises(AutoReconnect): - connected(client) - def test_lazy_connect_w0(self): - client = get_client(connection_string(), connect=False) + client = rs_or_single_client(connect=False) client.pymongo_test.test.insert({}, w=0) - client = get_client(connection_string(), connect=False) + client = rs_or_single_client(connect=False) client.pymongo_test.test.update({}, {'$set': {'x': 1}}, w=0) - client = get_client(connection_string(), connect=False) + client = rs_or_single_client(connect=False) client.pymongo_test.test.remove(w=0) @client_context.require_no_mongos def test_exhaust_network_error(self): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid semaphore leaks. - client = get_client(max_pool_size=1) + client = rs_or_single_client(max_pool_size=1) collection = client.pymongo_test.test pool = get_pool(client) pool._check_interval_seconds = None # Never check. @@ -905,7 +869,7 @@ class TestClient(IntegrationTest, TestRequestMixin): # when authenticating a new socket with cached credentials. # Get a client with one socket so we detect if it's leaked. - c = get_client(max_pool_size=1, waitQueueTimeoutMS=1) + c = rs_or_single_client(max_pool_size=1, waitQueueTimeoutMS=1) # Simulate an authenticate() call on a different socket. credentials = auth._build_credentials_tuple( @@ -925,62 +889,17 @@ class TestClient(IntegrationTest, TestRequestMixin): # No semaphore leak, the pool is allowed to make a new socket. c.test.collection.find_one() + @client_context.require_no_replica_set + def test_connect_to_standalone_using_replica_set_name(self): + with client_knobs(server_wait_time=0.1): + client = MongoClient(pair, replicaSet='anything') + + with self.assertRaises(AutoReconnect): + client.test.test.find_one() + class TestClientProperties(MockClientTest): - @client_context.require_connection - def test_wire_version(self): - c = MockClient( - standalones=[], - members=['a:1', 'b:2', 'c:3'], - mongoses=[], - host='b:2', # Pass a secondary. - replicaSet='rs', - connect=False) - - c.set_wire_version_range('a:1', 1, 5) - c._get_topology().select_servers(writable_server_selector) # Connect. - self.assertEqual(c.min_wire_version, 1) - self.assertEqual(c.max_wire_version, 5) - - c.set_wire_version_range('a:1', 10, 11) - c.disconnect() - c._get_topology() - self.assertRaises(ConfigurationError, c.db.collection.find_one) - - def test_max_wire_version(self): - # Disable periodic monitoring. - with client_knobs(heartbeat_frequency=1e6): - c = MockClient( - standalones=[], - members=['a:1', 'b:2', 'c:3'], - mongoses=[], - host='b:2', # Pass a secondary. - replicaSet='rs', - connect=False) - - c.set_max_write_batch_size('a:1', 1) - c.set_max_write_batch_size('b:2', 2) - - # Starts with default max batch size. - self.assertEqual(1000, c.max_write_batch_size) - connected(c) - wait_until(lambda: len(c.nodes) == 3, 'connect') - - # Uses primary's max batch size. - self.assertEqual(c.max_write_batch_size, 1) - - # b becomes primary. - c.mock_primary = 'b:2' - c.disconnect() - - # While disconnected, return to default max batch size. - self.assertEqual(1000, c.max_write_batch_size) - - connected(c) - wait_until(lambda: len(c.nodes) == 3, 'connect') - self.assertEqual(c.max_write_batch_size, 2) - def test_wire_version_mongos_ha(self): # TODO: Reimplement Mongos HA with PyMongo 3's MongoClient. raise SkipTest('Mongos HA must be reimplemented in PyMongo 3') @@ -1015,19 +934,15 @@ class TestClientProperties(MockClientTest): class TestClientLazyConnect(IntegrationTest, _TestLazyConnectMixin): - - def _get_client(self, **kwargs): - return get_client(connection_string(), **kwargs) + def _get_client(self): + return rs_or_single_client(connection_string(), connect=False) class TestClientLazyConnectBadSeeds(IntegrationTest): - - def _get_client(self, **kwargs): - kwargs.setdefault('connectTimeoutMS', 100) - + def _get_client(self): # Assume there are no open mongods listening on a.com, b.com, .... bad_seeds = ['%s.com' % chr(ord('a') + i) for i in range(10)] - return get_client(bad_seeds, **kwargs) + return rs_or_single_client(bad_seeds, connect=False) def test_connect(self): def reset(dummy): diff --git a/test/test_collection.py b/test/test_collection.py index 928222c1d..f15e04d50 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -48,9 +48,10 @@ from pymongo.errors import (DocumentTooLarge, InvalidOperation, OperationFailure, WTimeoutError) -from test.test_client import get_client, IntegrationTest +from test.test_client import IntegrationTest from test.utils import (is_mongos, joinall, enable_text_search, get_pool, - oid_generated_on_client, one) + oid_generated_on_client, one, ignore_deprecations, + get_client) from test import client_context, host, port, pair, qcheck, unittest @@ -232,8 +233,7 @@ class TestCollection(IntegrationTest): self.assertRaises(DeprecationWarning, lambda: db.test.ensure_index("goodbye", ttl=10)) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): self.assertEqual("goodbye_1", db.test.ensure_index("goodbye", ttl=10)) self.assertEqual(None, db.test.ensure_index("goodbye")) @@ -2187,8 +2187,7 @@ class TestCollection(IntegrationTest): for j in range(5): c.insert({'j': j, 'i': 0}) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): sort={'j': DESCENDING} self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, diff --git a/test/test_cursor.py b/test/test_cursor.py index 4611599c0..5492ff192 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -18,7 +18,6 @@ import itertools import random import re import sys -import warnings sys.path[0:0] = [""] @@ -36,7 +35,7 @@ from pymongo.errors import (InvalidOperation, OperationFailure, ExecutionTimeout) from test import client_context, SkipTest, unittest, host, port, IntegrationTest -from test.utils import server_started_with_auth +from test.utils import ignore_deprecations, server_started_with_auth if PY3: long = int @@ -1115,8 +1114,7 @@ class TestCursor(IntegrationTest): client = self.db.connection try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): client.set_cursor_manager(CManager) docs = [] cursor = self.db.test.find().batch_size(10) @@ -1130,8 +1128,7 @@ class TestCursor(IntegrationTest): docs.extend(ccursor) self.assertEqual(len(docs), 200) finally: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with ignore_deprecations(): client.set_cursor_manager(CursorManager) if __name__ == "__main__": diff --git a/test/test_read_preferences.py b/test/test_read_preferences.py index 0c6504ca1..a13a7817b 100644 --- a/test/test_read_preferences.py +++ b/test/test_read_preferences.py @@ -33,7 +33,6 @@ from pymongo.server_type import SERVER_TYPE from pymongo.errors import ConfigurationError from test.test_replica_set_client import TestReplicaSetClientBase -from test.test_client import get_client from test import (client_context, host, port, @@ -43,7 +42,7 @@ from test import (client_context, IntegrationTest, db_user, db_pwd) -from test.utils import connected, one, wait_until +from test.utils import connected, get_client, one, wait_until from test.version import Version diff --git a/test/test_replica_set_client.py b/test/test_replica_set_client.py index 3b58db55d..e7aa457c8 100644 --- a/test/test_replica_set_client.py +++ b/test/test_replica_set_client.py @@ -14,30 +14,21 @@ """Test the mongo_replica_set_client module.""" -import datetime -import signal import socket import sys import time -import threading import warnings sys.path[0:0] = [""] -from bson.py3compat import thread, u, _unicode +from bson.py3compat import u from bson.son import SON -from bson.tz_util import utc -from pymongo import auth -from pymongo.database import Database from pymongo.errors import (AutoReconnect, ConfigurationError, - ConnectionFailure, - InvalidName, OperationFailure, - NetworkTimeout) -from pymongo.mongo_client import _partition_node + NetworkTimeout, ConnectionFailure) +from pymongo.mongo_client import MongoClient, _partition_node from pymongo.mongo_replica_set_client import MongoReplicaSetClient -from pymongo.pool import SocketInfo from pymongo.read_preferences import ReadPreference, Secondary, Nearest from test import (client_context, client_knobs, @@ -51,30 +42,12 @@ from test import (client_context, MockClientTest) from test.pymongo_mocks import MockClient from test.utils import ( - delay, assertReadFrom, assertReadFromAll, read_from_which_host, - remove_all_users, assertRaisesExactly, TestRequestMixin, one, - server_started_with_auth, pools_from_rs_client, get_pool, - get_rs_client, _TestLazyConnectMixin, connected, wait_until) + delay, assertReadFrom, assertReadFromAll, ignore_deprecations, + read_from_which_host, assertRaisesExactly, TestRequestMixin, get_pools, + connected, wait_until, get_client, rs_or_single_client) from test.version import Version -class TestReplicaSetClientAgainstStandalone(unittest.TestCase): - """This is a funny beast -- we want to run tests for MongoReplicaSetClient - but only if the database at DB_IP and DB_PORT is a standalone. - """ - @client_context.require_connection - def setUp(self): - if client_context.setname: - raise SkipTest("Connected to a replica set, not a standalone mongod") - - def test_connect(self): - with client_knobs(server_wait_time=0.1): - client = MongoReplicaSetClient(pair, replicaSet='anything') - - with self.assertRaises(AutoReconnect): - client.test.test.find_one() - - class TestReplicaSetClientBase(unittest.TestCase): @classmethod @@ -100,8 +73,8 @@ class TestReplicaSetClientBase(unittest.TestCase): if m['stateStr'] == 'SECONDARY') def _get_client(self, **kwargs): - return get_rs_client(connection_string(), - replicaSet=self.name, **kwargs) + return get_client(connection_string(), + replicaSet=self.name, **kwargs) class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): @@ -115,126 +88,43 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self.fail(msg) - def assertIsInstance(self, obj, cls, msg=None): - """Backport from Python 2.7.""" - if not isinstance(obj, cls): - standardMsg = '%r is not an instance of %r' % (obj, cls) - self.fail(self._formatMessage(msg, standardMsg)) - def test_deprecated(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) with self.assertRaises(DeprecationWarning): MongoReplicaSetClient() - def test_init_disconnected(self): - c = self._get_client(connect=False) - - self.assertIsInstance(c.is_mongos, bool) - self.assertIsInstance(c.max_pool_size, int) - self.assertIsInstance(c.tz_aware, bool) - self.assertIsInstance(c.max_bson_size, int) - self.assertIsInstance(c.min_wire_version, int) - self.assertIsInstance(c.max_wire_version, int) - self.assertIsInstance(c.arbiters, set) - self.assertEqual(dict, c.get_document_class()) - self.assertFalse(c.primary) - self.assertFalse(c.secondaries) - - connected(c) - self.assertTrue(c.primary) - wait_until(lambda: c.secondaries, "found secondaries") - - if Version.from_client(c).at_least(2, 5, 4, -1): - self.assertTrue(c.max_wire_version > 0) - else: - self.assertEqual(c.max_wire_version, 0) - self.assertTrue(c.min_wire_version >= 0) - - c = self._get_client(connect=False) - c.pymongo_test.test.update({}, {}) # Auto-connect for write. - self.assertTrue(c.primary) - - c = self._get_client(connect=False) - c.pymongo_test.test.insert({}) # Auto-connect for write. - self.assertTrue(c.primary) - - c = self._get_client(connect=False) - c.pymongo_test.test.remove({}) # Auto-connect for write. - self.assertTrue(c.primary) - - with client_knobs(server_wait_time=0.1): - c = MongoReplicaSetClient( - "somedomainthatdoesntexist.org", replicaSet="rs", - connectTimeoutMS=1, connect=False) - - self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) - - def test_init_disconnected_with_auth_failure(self): - with client_knobs(server_wait_time=0.1): - c = MongoReplicaSetClient( - "mongodb://user:pass@somedomainthatdoesntexist", - replicaSet="rs", connectTimeoutMS=1, connect=False) - - self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) - - @client_context.require_auth - def test_init_disconnected_with_auth(self): - c = client_context.rs_client - try: - c.pymongo_test.add_user("user", "pass", - roles=['readWrite', 'userAdmin']) - - # Auth with lazy connection. - host = one(self.hosts) - uri = "mongodb://user:pass@%s:%d/pymongo_test?replicaSet=%s" % ( - host[0], host[1], self.name) - - authenticated_client = MongoReplicaSetClient(uri, connect=False) - authenticated_client.pymongo_test.test.find_one() - - # Wrong password. - bad_uri = "mongodb://user:wrong@%s:%d/pymongo_test?replicaSet=%s" % ( - host[0], host[1], self.name) - - bad_client = MongoReplicaSetClient(bad_uri, connect=False) - self.assertRaises( - OperationFailure, bad_client.pymongo_test.test.find_one) - - finally: - # Clean up. - remove_all_users(c.pymongo_test) - def test_connect(self): with client_knobs(server_wait_time=0.1): - client = MongoReplicaSetClient( - "somedomainthatdoesntexist.org:27017", - replicaSet=self.name) - - with self.assertRaises(AutoReconnect): - client.test.test.find_one() - - client = MongoReplicaSetClient(pair, replicaSet='fdlksjfdslkjfd') + client = MongoClient(pair, replicaSet='fdlksjfdslkjfd') with self.assertRaises(ConnectionFailure): client.test.test.find_one() def test_repr(self): - client = client_context.rs_client + with ignore_deprecations(): + client = MongoReplicaSetClient(connection_string(), + replicaSet=self.name) - # Quirk: the RS client makes a frozenset of hosts from a dict's keys, - # so we must do the same to achieve the same order. - host_dict = dict([(host, 1) for host in self.hosts]) - hosts_set = frozenset(host_dict) - hosts_repr = ', '.join([ - repr(_unicode('%s:%s' % host)) for host in hosts_set]) + wait_until(lambda: client.primary == self.primary, "discover primary") + wait_until(lambda: client.secondaries == self.secondaries, + "discover secondaries") - self.assertEqual(repr(client), - "MongoReplicaSetClient([%s])" % hosts_repr) + # repr should be something like + # MongoReplicaSetClient(["localhost:27017", "localhost:27018"]). + self.assertIn("MongoReplicaSetClient([", repr(client)) + for host in self.hosts: + self.assertIn("%s:%d" % host, repr(client)) def test_properties(self): c = client_context.rs_client c.admin.command('ping') + + wait_until(lambda: c.primary == self.primary, "discover primary") + wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") + wait_until(lambda: c.secondaries == self.secondaries, + "discover secondaries") + self.assertEqual(c.primary, self.primary) self.assertEqual(c.secondaries, self.secondaries) self.assertEqual(c.arbiters, self.arbiters) @@ -242,7 +132,8 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self.assertEqual(c.document_class, dict) self.assertEqual(c.tz_aware, False) - # Make sure MRSC's properties are copied to Database and Collection + # Make sure MongoClient's properties are copied to Database and + # Collection. for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, {}) @@ -253,17 +144,12 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets=tag_sets) - c = MongoReplicaSetClient( + c = MongoClient( pair, replicaSet=self.name, max_pool_size=25, document_class=SON, tz_aware=True, read_preference=secondary, secondaryacceptablelatencyms=77) - wait_until(lambda: c.primary == self.primary, "discover primary") - wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") - wait_until(lambda: c.secondaries == self.secondaries, - "discover secondaries") - self.assertEqual(c.max_pool_size, 25) self.assertEqual(c.document_class, SON) self.assertEqual(c.tz_aware, True) @@ -287,26 +173,9 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self.assertEqual(c.max_bson_size, 4194304) c.close() - def test_get_db(self): - client = client_context.rs_client - - def make_db(base, name): - return base[name] - - self.assertRaises(InvalidName, make_db, client, "") - self.assertRaises(InvalidName, make_db, client, "te$t") - self.assertRaises(InvalidName, make_db, client, "te.t") - self.assertRaises(InvalidName, make_db, client, "te\\t") - self.assertRaises(InvalidName, make_db, client, "te/t") - self.assertRaises(InvalidName, make_db, client, "te st") - - self.assertTrue(isinstance(client.test, Database)) - self.assertEqual(client.test, client["test"]) - self.assertEqual(client.test, Database(client, "test")) - def test_auto_reconnect_exception_when_read_preference_is_secondary(self): with client_knobs(server_wait_time=0.1): - c = MongoReplicaSetClient(pair, replicaSet=self.name) + c = MongoClient(pair, replicaSet=self.name) db = c.pymongo_test def raise_socket_error(*args, **kwargs): @@ -321,276 +190,8 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): finally: socket.socket.sendall = old_sendall - @client_context.require_auth - def test_lazy_auth_raises_operation_failure(self): - # Check if we have the prerequisites to run this test. - lazy_client = MongoReplicaSetClient( - "mongodb://user:wrong@%s/pymongo_test" % pair, - replicaSet=self.name, - connect=False) - - assertRaisesExactly( - OperationFailure, lazy_client.test.collection.find_one) - - def test_operations(self): - c = client_context.rs_client - - # Check explicitly for a case we've commonly hit in tests: - # a replica set is started with a tiny oplog, a previous - # test does a big insert that leaves the secondaries - # permanently "RECOVERING", and our insert(w=self.w) hangs - # forever. - rs_status = c.admin.command('replSetGetStatus') - members = rs_status['members'] - self.assertFalse( - [m for m in members if m['stateStr'] == 'RECOVERING'], - "Replica set is recovering, try a larger oplogSize next time" - ) - - db = c.pymongo_test - db.test.remove({}) - self.assertEqual(0, db.test.count()) - db.test.insert({'foo': 'x'}, w=self.w, wtimeout=10000) - self.assertEqual(1, db.test.count()) - - cursor = db.test.find() - doc = next(cursor) - self.assertEqual('x', doc['foo']) - # Ensure we read from the primary - self.assertEqual(c.primary, cursor._Cursor__connection_id) - - cursor = db.test.find(read_preference=ReadPreference.SECONDARY) - doc = next(cursor) - self.assertEqual('x', doc['foo']) - # Ensure we didn't read from the primary - self.assertTrue(cursor._Cursor__connection_id in c.secondaries) - - self.assertEqual(1, db.test.count()) - db.test.remove({}) - self.assertEqual(0, db.test.count()) - db.test.drop() - - def test_database_names(self): - client = client_context.rs_client - - client.pymongo_test.test.save({"dummy": u("object")}) - client.pymongo_test_mike.test.save({"dummy": u("object")}) - - dbs = client.database_names() - self.assertTrue("pymongo_test" in dbs) - self.assertTrue("pymongo_test_mike" in dbs) - - def test_drop_database(self): - client = client_context.rs_client - - self.assertRaises(TypeError, client.drop_database, 5) - self.assertRaises(TypeError, client.drop_database, None) - - client.pymongo_test.test.save({"dummy": u("object")}) - dbs = client.database_names() - self.assertTrue("pymongo_test" in dbs) - client.drop_database("pymongo_test") - dbs = client.database_names() - self.assertTrue("pymongo_test" not in dbs) - - client.pymongo_test.test.save({"dummy": u("object")}) - dbs = client.database_names() - self.assertTrue("pymongo_test" in dbs) - client.drop_database(client.pymongo_test) - dbs = client.database_names() - self.assertTrue("pymongo_test" not in dbs) - - def test_copy_db(self): - c = client_context.rs_client - # We test copy twice; once starting in a request and once not. In - # either case the copy should succeed (because it starts a request - # internally) and should leave us in the same state as before the copy. - with c.start_request(): - self.assertRaises(TypeError, c.copy_database, 4, "foo") - self.assertRaises(TypeError, c.copy_database, "foo", 4) - - self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") - - c.pymongo_test.test.drop() - c.drop_database("pymongo_test1") - c.drop_database("pymongo_test2") - - c.pymongo_test.test.insert({"foo": "bar"}) - - self.assertFalse("pymongo_test1" in c.database_names()) - self.assertFalse("pymongo_test2" in c.database_names()) - - c.copy_database("pymongo_test", "pymongo_test1") - # copy_database() didn't accidentally end the request - self.assertTrue(c.in_request()) - - self.assertTrue("pymongo_test1" in c.database_names()) - self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) - - self.assertFalse(c.in_request()) - - c.copy_database("pymongo_test", "pymongo_test2") - # copy_database() didn't accidentally restart the request - self.assertFalse(c.in_request()) - - time.sleep(1) - - self.assertTrue("pymongo_test2" in c.database_names()) - self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) - - if (Version.from_client(c).at_least(1, 3, 3, 1) and - server_started_with_auth(c)): - c.drop_database("pymongo_test1") - - try: - c.pymongo_test.add_user("mike", "password") - - self.assertRaises(OperationFailure, c.copy_database, - "pymongo_test", "pymongo_test1", - username="foo", password="bar") - self.assertFalse("pymongo_test1" in c.database_names()) - - self.assertRaises(OperationFailure, c.copy_database, - "pymongo_test", "pymongo_test1", - username="mike", password="bar") - self.assertFalse("pymongo_test1" in c.database_names()) - - c.copy_database("pymongo_test", "pymongo_test1", - username="mike", password="password") - self.assertTrue("pymongo_test1" in c.database_names()) - res = c.pymongo_test1.test.find_one( - read_preference=ReadPreference.PRIMARY) - self.assertEqual("bar", res["foo"]) - finally: - # Cleanup - remove_all_users(c.pymongo_test) - - def test_get_default_database(self): - host = one(self.hosts) - uri = "mongodb://%s:%d/foo?replicaSet=%s" % ( - host[0], host[1], self.name) - - c = MongoReplicaSetClient(uri, connect=False) - self.assertEqual(Database(c, 'foo'), c.get_default_database()) - - def test_get_default_database_error(self): - host = one(self.hosts) - # URI with no database. - uri = "mongodb://%s:%d/?replicaSet=%s" % ( - host[0], host[1], self.name) - - c = MongoReplicaSetClient(uri, connect=False) - self.assertRaises(ConfigurationError, c.get_default_database) - - def test_get_default_database_with_authsource(self): - # Ensure we distinguish database name from authSource. - host = one(self.hosts) - uri = "mongodb://%s:%d/foo?replicaSet=%s&authSource=src" % ( - host[0], host[1], self.name) - - c = MongoReplicaSetClient(uri, connect=False) - self.assertEqual(Database(c, 'foo'), c.get_default_database()) - - def test_iteration(self): - client = client_context.rs_client - - def iterate(): - [a for a in client] - - self.assertRaises(TypeError, iterate) - - def test_disconnect(self): - c = client_context.rs_client - coll = c.pymongo_test.bar - - c.disconnect() - c.disconnect() - - coll.count() - - c.disconnect() - c.disconnect() - - coll.count() - - def test_close(self): - # Multiple threads can call close() at once without error. - # Subsequent operations reopen the client. - c = self._get_client() - nthreads = 10 - outcomes = [] - - def close(): - c.close() - outcomes.append(True) - - threads = [threading.Thread(target=close) for _ in range(nthreads)] - for t in threads: - t.start() - - for t in threads: - t.join(10) - - self.assertEqual(nthreads, len(outcomes)) - - # No error. - c.db.collection.insert({}) - c.db.collection.find_one() - - def test_socket_timeout_ms_validation(self): - c = self._get_client(socketTimeoutMS=10 * 1000) - self.assertEqual(10, - c._MongoClient__options.pool_options.socket_timeout) - - c = self._get_client(socketTimeoutMS=None) - self.assertEqual(None, - c._MongoClient__options.pool_options.socket_timeout) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS=0) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS=-1) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS=1e10) - - self.assertRaises(ConfigurationError, - self._get_client, socketTimeoutMS='foo') - - def test_socket_timeout(self): - no_timeout = client_context.rs_client - timeout_sec = 1 - timeout = self._get_client(socketTimeoutMS=timeout_sec*1000) - - no_timeout.pymongo_test.drop_collection("test") - no_timeout.pymongo_test.test.insert({"x": 1}) - - # A $where clause that takes a second longer than the timeout. - query = {'$where': delay(1 + timeout_sec)} - no_timeout.pymongo_test.test.find_one(query) # No error. - - try: - timeout.pymongo_test.test.find_one(query) - except AutoReconnect as e: - self.assertTrue('%d: timed out' % (port,) in e.args[0]) - else: - self.fail('RS client should have raised timeout error') - - try: - timeout.pymongo_test.test.find_one( - query, - read_preference=ReadPreference.SECONDARY) - except NetworkTimeout as e: - self.assertTrue( - any('%d: timed out' % address[1] in e.args[0] - for address in self.secondaries), - "%r does not mention any secondary's port" % e.args[0]) - else: - self.fail('RS client should have raised timeout error') - def test_timeout_does_not_mark_member_down(self): - # If a query times out, the RS client shouldn't mark the member "down". + # If a query times out, the client shouldn't mark the member "down". # Disable background refresh. with client_knobs(heartbeat_frequency=999999): @@ -619,51 +220,20 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): # No error. collection.find_one(read_preference=ReadPreference.SECONDARY) - def test_waitQueueTimeoutMS(self): - client = self._get_client(waitQueueTimeoutMS=2000) - pool = get_pool(client) - self.assertEqual(pool.opts.wait_queue_timeout, 2) - - def test_waitQueueMultiple(self): - client = self._get_client(max_pool_size=3, waitQueueMultiple=2) - pool = get_pool(client) - self.assertEqual(pool.opts.wait_queue_multiple, 2) - self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6) - - def test_tz_aware(self): - self.assertRaises(ConfigurationError, MongoReplicaSetClient, - tz_aware='foo', replicaSet=self.name) - - aware = self._get_client(tz_aware=True) - naive = client_context.rs_client - aware.pymongo_test.drop_collection("test") - - now = datetime.datetime.utcnow() - aware.pymongo_test.test.insert({"x": now}) - time.sleep(1) - - self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo) - self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo) - self.assertEqual( - aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), - naive.pymongo_test.test.find_one()["x"]) - + @client_context.require_replica_set + @client_context.require_ipv6 def test_ipv6(self): - try: - MongoReplicaSetClient("[::1]:%d" % (port,), replicaSet=self.name) - except: - # Either mongod was started without --ipv6 - # or the OS doesn't support it (or both). - raise SkipTest("No IPv6") + c = MongoClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name) - # Try a few simple things - MongoReplicaSetClient("mongodb://[::1]:%d" % (port,), - replicaSet=self.name) - MongoReplicaSetClient("mongodb://[::1]:%d/?w=0;" - "replicaSet=%s" % (port, self.name)) - MongoReplicaSetClient("[::1]:%d,localhost:" - "%d" % (port, port), - replicaSet=self.name) + # Client switches to IPv4 once it has first ismaster response. + msg = 'discovered primary with IPv4 address "%r"' % (self.primary,) + wait_until(lambda: c.primary == self.primary, msg) + + # Same outcome with both IPv4 and IPv6 seeds. + c = MongoClient("[::1]:%d,localhost:%d" % (port, port), + replicaSet=self.name) + + wait_until(lambda: c.primary == self.primary, msg) if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) @@ -671,7 +241,7 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): auth_str = "" uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port) - client = MongoReplicaSetClient(uri, replicaSet=self.name) + client = MongoClient(uri, replicaSet=self.name) client.pymongo_test.test.save({"dummy": u("object")}) client.pymongo_test_bernie.test.save({"dummy": u("object")}) @@ -727,103 +297,12 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): def test_kill_cursor_explicit_secondary(self): self._test_kill_cursor_explicit(ReadPreference.SECONDARY) - def test_interrupt_signal(self): - if sys.platform.startswith('java'): - raise SkipTest("Can't test interrupts in Jython") - - # Test fix for PYTHON-294 -- make sure client closes its socket if it - # gets an interrupt while waiting to recv() from it. - c = client_context.rs_client - db = c.pymongo_test - - # A $where clause which takes 1.5 sec to execute - where = delay(1.5) - - # Need exactly 1 document so find() will execute its $where clause once - db.drop_collection('foo') - db.foo.insert({'_id': 1}) - - old_signal_handler = None - - try: - def interrupter(): - time.sleep(0.25) - - # Raises KeyboardInterrupt in the main thread - thread.interrupt_main() - - thread.start_new_thread(interrupter, ()) - - raised = False - try: - # Will be interrupted by a KeyboardInterrupt. - next(db.foo.find({'$where': where})) - except KeyboardInterrupt: - raised = True - - # Can't use self.assertRaises() because it doesn't catch system - # exceptions - self.assertTrue(raised, "Didn't raise expected ConnectionFailure") - - # Raises AssertionError due to PYTHON-294 -- Mongo's response to the - # previous find() is still waiting to be read on the socket, so the - # request id's don't match. - self.assertEqual( - {'_id': 1}, - next(db.foo.find()) - ) - finally: - if old_signal_handler: - signal.signal(signal.SIGALRM, old_signal_handler) - - def test_operation_failure_without_request(self): - # Ensure MongoReplicaSetClient doesn't close socket after it gets an - # error response to getLastError. PYTHON-395. - c = self._get_client() - connected(c) - pool = get_pool(c) - self.assertEqual(1, len(pool.sockets)) - old_sock_info = next(iter(pool.sockets)) - c.pymongo_test.test.drop() - c.pymongo_test.test.insert({'_id': 'foo'}) - self.assertRaises( - OperationFailure, - c.pymongo_test.test.insert, {'_id': 'foo'}) - - self.assertEqual(1, len(pool.sockets)) - new_sock_info = next(iter(pool.sockets)) - - self.assertEqual(old_sock_info, new_sock_info) - c.close() - - def test_operation_failure_with_request(self): - # Ensure MongoReplicaSetClient doesn't close socket after it gets an - # error response to getLastError. PYTHON-395. - c = self._get_client() - c.start_request() - c.pymongo_test.test.find_one() - pool = get_pool(c) - - # Client reserved a socket for this thread - self.assertTrue(isinstance(pool._get_request_state(), SocketInfo)) - - old_sock_info = pool._get_request_state() - c.pymongo_test.test.drop() - c.pymongo_test.test.insert({'_id': 'foo'}) - self.assertRaises( - OperationFailure, - c.pymongo_test.test.insert, {'_id': 'foo'}) - - # OperationFailure doesn't affect the request socket - self.assertEqual(old_sock_info, pool._get_request_state()) - c.close() - def test_nested_request(self): - client = self._get_client() + client = rs_or_single_client() connected(client) client.start_request() try: - pools = pools_from_rs_client(client) + pools = get_pools(client) self.assertTrue(client.in_request()) # Start and end request - we're still in "outer" original request @@ -837,8 +316,8 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): client.start_request() for pool in pools: - # MRSC only called start_request() once per pool, although its - # own counter is 3. + # Client only called start_request() once per pool, although + # its own counter is 3. self.assertEqual(1, pool._request_counter.get()) client.end_request() @@ -857,45 +336,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): finally: client.close() - def test_request_threads(self): - client = client_context.rs_client - - pools = pools_from_rs_client(client) - self.assertNotInRequestAndDifferentSock(client, pools) - - started_request, ended_request = threading.Event(), threading.Event() - checked_request = threading.Event() - thread_done = [False] - - # Starting a request in one thread doesn't put the other thread in a - # request - def f(): - self.assertNotInRequestAndDifferentSock(client, pools) - client.start_request() - self.assertInRequestAndSameSock(client, pools) - started_request.set() - checked_request.wait() - checked_request.clear() - self.assertInRequestAndSameSock(client, pools) - client.end_request() - self.assertNotInRequestAndDifferentSock(client, pools) - ended_request.set() - checked_request.wait() - thread_done[0] = True - - t = threading.Thread(target=f) - t.setDaemon(True) - t.start() - started_request.wait() - self.assertNotInRequestAndDifferentSock(client, pools) - checked_request.set() - ended_request.wait() - self.assertNotInRequestAndDifferentSock(client, pools) - checked_request.set() - t.join() - self.assertNotInRequestAndDifferentSock(client, pools) - self.assertTrue(thread_done[0], "Thread didn't complete") - def test_pinned_member(self): raise SkipTest("Secondary pinning not implemented in PyMongo 3") @@ -939,43 +379,6 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin): self, client, list(client.secondaries) + [client.primary], ReadPreference.NEAREST, None) - def test_alive(self): - client = client_context.rs_client - self.assertTrue(client.alive()) - - client = MongoReplicaSetClient( - 'doesnt exist', replicaSet='rs', connect=False) - - self.assertFalse(client.alive()) - - @client_context.require_auth - def test_auth_network_error(self): - # Make sure there's no semaphore leak if we get a network error - # when authenticating a new socket with cached credentials. - - # Get a client with one socket so we detect if it's leaked. - c = self._get_client(max_pool_size=1, waitQueueTimeoutMS=1) - - # Simulate an authenticate() call on a different socket. - credentials = auth._build_credentials_tuple( - 'MONGODB-CR', 'admin', db_user, db_pwd, {}) - - c._cache_credentials('test', credentials, connect=False) - - # Cause a network error on the actual socket. - connected(c) - pool = get_pool(c) - socket_info = one(pool.sockets) - socket_info.sock.close() - - # In __check_auth, the client authenticates its socket with the - # new credential, but gets a socket.error. Should be reraised as - # AutoReconnect. - self.assertRaises(AutoReconnect, c.test.collection.find_one) - - # No semaphore leak, the pool is allowed to make a new socket. - c.test.collection.find_one() - class TestReplicaSetWireVersion(MockClientTest): @@ -1017,46 +420,6 @@ class TestReplicaSetWireVersion(MockClientTest): self.assertRaises(ConfigurationError, c.db.collection.insert, {}) -# Test concurrent access to a lazily-connecting RS client. -class TestReplicaSetClientLazyConnect( - TestReplicaSetClientBase, - _TestLazyConnectMixin): - - @classmethod - def setUpClass(cls): - TestReplicaSetClientBase.setUpClass() - - def test_read_mode_secondary(self): - client = MongoReplicaSetClient( - connection_string(), replicaSet=self.name, connect=False, - read_preference=ReadPreference.SECONDARY) - - # No error. - client.pymongo_test.test_collection.find_one() - - -class TestReplicaSetClientLazyConnectBadSeeds( - TestReplicaSetClientBase, - _TestLazyConnectMixin): - - @classmethod - def setUpClass(cls): - TestReplicaSetClientBase.setUpClass() - - def _get_client(self, **kwargs): - kwargs.setdefault('connectTimeoutMS', 500) - - # Assume there are no open mongods listening on a.com, b.com, .... - bad_seeds = ['%s.com' % chr(ord('a') + i) for i in range(5)] - client = MongoReplicaSetClient( - connection_string(seeds=(bad_seeds + [pair])), - replicaSet=self.name, **kwargs) - - # In case of a slow test machine. - client._refresh_timeout_sec = 30 - return client - - class TestReplicaSetClientInternalIPs(MockClientTest): @client_context.require_connection diff --git a/test/test_ssl.py b/test/test_ssl.py index badd9b6b8..0ce8a9aac 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -32,7 +32,7 @@ except ImportError: # Python 2 from urllib import quote_plus -from pymongo import MongoClient, MongoReplicaSetClient +from pymongo import MongoClient from pymongo.errors import (ConfigurationError, ConnectionFailure, OperationFailure) @@ -65,8 +65,8 @@ MONGODB_X509_USERNAME = ( # --sslWeakCertificateValidation # Also, make sure you have 'server' as an alias for localhost in /etc/hosts # -# Note: For all tests to pass with MongoReplicaSetClient the replica -# set configuration must use 'server' for the hostname of all hosts. +# Note: For all replica set tests to pass, the replica set configuration must +# use 'server' for the hostname of all hosts. def is_server_resolvable(): """Returns True if 'server' is resolvable.""" @@ -127,15 +127,10 @@ class TestClientSSL(unittest.TestCase): # Explicit self.assertRaises(ConfigurationError, MongoClient, ssl=True) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, replicaSet='rs', ssl=True) + # Implied self.assertRaises(ConfigurationError, MongoClient, ssl_certfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_certfile=CLIENT_PEM) def test_config_ssl(self): # Tests various ssl configurations @@ -148,20 +143,6 @@ class TestClientSSL(unittest.TestCase): self.assertRaises(TypeError, MongoClient, ssl=5.5) self.assertRaises(TypeError, MongoClient, ssl=[]) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, replicaSet='rs', ssl='foo') - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_certfile=CLIENT_PEM) - self.assertRaises(TypeError, - MongoReplicaSetClient, replicaSet='rs', ssl=0) - self.assertRaises(TypeError, - MongoReplicaSetClient, replicaSet='rs', ssl=5.5) - self.assertRaises(TypeError, - MongoReplicaSetClient, replicaSet='rs', ssl=[]) - self.assertRaises(IOError, MongoClient, ssl_certfile="NoSuchFile") self.assertRaises(TypeError, MongoClient, ssl_certfile=True) self.assertRaises(TypeError, MongoClient, ssl_certfile=[]) @@ -169,19 +150,6 @@ class TestClientSSL(unittest.TestCase): self.assertRaises(TypeError, MongoClient, ssl_keyfile=True) self.assertRaises(TypeError, MongoClient, ssl_keyfile=[]) - self.assertRaises(IOError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_keyfile="NoSuchFile") - self.assertRaises(IOError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_certfile="NoSuchFile") - self.assertRaises(TypeError, - MongoReplicaSetClient, - replicaSet='rs', - ssl_certfile=True) - # Test invalid combinations self.assertRaises(ConfigurationError, MongoClient, @@ -197,23 +165,6 @@ class TestClientSSL(unittest.TestCase): ssl_keyfile=CLIENT_PEM, ssl_certfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_keyfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_certfile=CLIENT_PEM) - self.assertRaises(ConfigurationError, - MongoReplicaSetClient, - replicaSet='rs', - ssl=False, - ssl_keyfile=CLIENT_PEM, - ssl_certfile=CLIENT_PEM) - class TestSSL(unittest.TestCase): @@ -235,10 +186,10 @@ class TestSSL(unittest.TestCase): client = MongoClient(host, port, ssl=True) response = client.admin.command('ismaster') if 'setName' in response: - client = MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True) + client = MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True) db = client.pymongo_ssl_test db.test.drop() @@ -261,10 +212,10 @@ class TestSSL(unittest.TestCase): client = ssl_client response = ssl_client.admin.command('ismaster') if 'setName' in response: - client = MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, ssl_certfile=CLIENT_PEM) + client = MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, ssl_certfile=CLIENT_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -287,10 +238,10 @@ class TestSSL(unittest.TestCase): client = ssl_client response = ssl_client.admin.command('ismaster') if 'setName' in response: - client = MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl_certfile=CLIENT_PEM) + client = MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl_certfile=CLIENT_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -325,13 +276,13 @@ class TestSSL(unittest.TestCase): raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") - client = MongoReplicaSetClient('server', - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, - ssl_certfile=CLIENT_PEM, - ssl_cert_reqs=ssl.CERT_REQUIRED, - ssl_ca_certs=CA_PEM) + client = MongoClient('server', + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, + ssl_certfile=CLIENT_PEM, + ssl_cert_reqs=ssl.CERT_REQUIRED, + ssl_ca_certs=CA_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -367,13 +318,13 @@ class TestSSL(unittest.TestCase): raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") - client = MongoReplicaSetClient('server', - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, - ssl_certfile=CLIENT_PEM, - ssl_cert_reqs=ssl.CERT_OPTIONAL, - ssl_ca_certs=CA_PEM) + client = MongoClient('server', + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, + ssl_certfile=CLIENT_PEM, + ssl_cert_reqs=ssl.CERT_OPTIONAL, + ssl_ca_certs=CA_PEM) db = client.pymongo_ssl_test db.test.drop() @@ -405,13 +356,13 @@ class TestSSL(unittest.TestCase): if 'setName' in response: try: - MongoReplicaSetClient(pair, - replicaSet=response['setName'], - w=len(response['hosts']), - ssl=True, - ssl_certfile=CLIENT_PEM, - ssl_cert_reqs=ssl.CERT_REQUIRED, - ssl_ca_certs=CA_PEM) + MongoClient(pair, + replicaSet=response['setName'], + w=len(response['hosts']), + ssl=True, + ssl_certfile=CLIENT_PEM, + ssl_cert_reqs=ssl.CERT_REQUIRED, + ssl_ca_certs=CA_PEM) self.fail("Invalid hostname should have failed") except CertificateError: pass diff --git a/test/test_threads.py b/test/test_threads.py index afd6374ee..f910a66a4 100644 --- a/test/test_threads.py +++ b/test/test_threads.py @@ -17,9 +17,11 @@ import threading import traceback -from test import unittest, client_context, pair -from test.utils import joinall, remove_all_users, RendezvousThread -from test.test_client import get_client +from test import unittest, client_context, IntegrationTest +from test.utils import (joinall, + remove_all_users, + RendezvousThread, + rs_or_single_client) from test.utils import get_pool from pymongo.pool import SocketInfo, _closed from pymongo.errors import AutoReconnect, OperationFailure @@ -163,30 +165,9 @@ class FindPauseFind(RendezvousThread): assert self.request_sock != pool._get_request_state().sock -class BaseTestThreads(object): - """ - Base test class for TestThreads and TestThreadsReplicaSet. (This - is not itself a unittest.TestCase, otherwise it'd be run twice -- - once when unittest imports this module, and once when unittest - imports test_threads_replica_set_connection.py, which imports this - module.) - """ +class TestThreads(IntegrationTest): def setUp(self): - self.db = self._get_client().pymongo_test - - def tearDown(self): - # Clear client reference so that RSC's monitor thread - # dies. - self.db = None - - def _get_client(self): - """ - Intended for overriding in TestThreadsReplicaSet. This method - returns a MongoClient here, and a MongoReplicaSetClient in - test_threads_replica_set_connection.py. - """ - # Regular test client - return get_client(pair) + self.db = client_context.rs_or_standalone_client.pymongo_test def test_threading(self): self.db.drop_collection("test") @@ -254,7 +235,7 @@ class BaseTestThreads(object): # # If we've fixed PYTHON-345, then only one AutoReconnect is raised, # and all the threads get new request sockets. - cx = get_client(pair) + cx = rs_or_single_client() cx.start_request() collection = cx.db.pymongo_test @@ -305,37 +286,22 @@ class BaseTestThreads(object): self.assertTrue(t.passed, "%s threw exception" % t) -class BaseTestThreadsAuth(object): - """ - Base test class for TestThreadsAuth and TestThreadsAuthReplicaSet. (This is - not itself a unittest.TestCase, otherwise it'd be run twice -- once when - unittest imports this module, and once when unittest imports - test_threads_replica_set_connection.py, which imports this module.) - """ +class TestThreadsAuth(IntegrationTest): @classmethod @client_context.require_auth def setUpClass(cls): - pass - - def _get_client(self): - """ - Intended for overriding in TestThreadsAuthReplicaSet. This method - returns a MongoClient here, and a MongoReplicaSetClient in - test_threads_replica_set_connection.py. - """ - # Regular test client - return get_client(pair) + super(TestThreadsAuth, cls).setUpClass() def setUp(self): - client = self._get_client() - self.client = client + self.client = rs_or_single_client() self.client.admin.add_user('admin-user', 'password', roles=['clusterAdmin', 'dbAdminAnyDatabase', 'readWriteAnyDatabase', 'userAdminAnyDatabase']) - # client returned from self._get_client() may already be authenticated - # to get around restricted localhost exception in MongoDB >= 2.7.1. + + # client is already authenticated to get around restricted + # localhost exception in MongoDB >= 2.7.1. self.client.admin.logout() self.client.admin.authenticate("admin-user", "password") self.client.auth_test.add_user("test-user", "password", @@ -348,18 +314,13 @@ class BaseTestThreadsAuth(object): self.client.drop_database('auth_test') self.client.admin.remove_user('admin-user') self.client.admin.logout() - # Clear client reference so that RSC's monitor thread - # dies. - self.client = None def test_auto_auth_login(self): - client = self._get_client() + client = self.client client.admin.logout() self.assertRaises(OperationFailure, client.auth_test.test.find_one) # Admin auth - client = self._get_client() - client.admin.logout() client.admin.authenticate("admin-user", "password") nthreads = 10 @@ -375,7 +336,6 @@ class BaseTestThreadsAuth(object): self.assertTrue(t.success) # Database-specific auth - client = self._get_client() client.admin.logout() client.auth_test.authenticate("test-user", "password") @@ -390,12 +350,6 @@ class BaseTestThreadsAuth(object): for t in threads: self.assertTrue(t.success) -class TestThreads(BaseTestThreads, unittest.TestCase): - pass - -class TestThreadsAuth(BaseTestThreadsAuth, unittest.TestCase): - pass - if __name__ == "__main__": unittest.main() diff --git a/test/test_threads_replica_set_client.py b/test/test_threads_replica_set_client.py deleted file mode 100644 index f67a5b073..000000000 --- a/test/test_threads_replica_set_client.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2011-2014 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test that pymongo is thread safe.""" - -from test import unittest -from test.test_threads import BaseTestThreads, BaseTestThreadsAuth -from test.test_replica_set_client import TestReplicaSetClientBase, pair -from test.utils import get_rs_client - - -class TestThreadsReplicaSet(TestReplicaSetClientBase, BaseTestThreads): - def setUp(self): - """ - Prepare to test all the same things that TestThreads tests, but do it - with a replica-set client - """ - TestReplicaSetClientBase.setUp(self) - BaseTestThreads.setUp(self) - - def tearDown(self): - TestReplicaSetClientBase.tearDown(self) - BaseTestThreads.tearDown(self) - - def _get_client(self, **kwargs): - return TestReplicaSetClientBase._get_client(self, **kwargs) - - -class TestThreadsAuthReplicaSet(TestReplicaSetClientBase, BaseTestThreadsAuth): - - @classmethod - def setUpClass(cls): - TestReplicaSetClientBase.setUpClass() - BaseTestThreadsAuth.setUpClass() - - @classmethod - def tearDownClass(cls): - TestReplicaSetClientBase.tearDownClass() - - def setUp(self): - """ - Prepare to test all the same things that TestThreads tests, but do it - with a replica-set client - """ - TestReplicaSetClientBase.setUp(self) - BaseTestThreadsAuth.setUp(self) - - def tearDown(self): - TestReplicaSetClientBase.tearDown(self) - BaseTestThreadsAuth.tearDown(self) - - def _get_client(self): - """ - Override TestThreadsAuth, so its tests run on a MongoReplicaSetClient - instead of a regular MongoClient. - """ - return get_rs_client(pair, replicaSet=self.name) - - -if __name__ == "__main__": - suite = unittest.TestSuite([ - unittest.makeSuite(TestThreadsReplicaSet), - unittest.makeSuite(TestThreadsAuthReplicaSet) - ]) - unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/test/utils.py b/test/utils.py index 68bbb40cd..1f8ec9248 100644 --- a/test/utils.py +++ b/test/utils.py @@ -15,20 +15,24 @@ """Utilities for testing pymongo """ +import contextlib import os import struct import sys import threading import time +import warnings -from pymongo import MongoClient, MongoReplicaSetClient +from pymongo import MongoClient from pymongo.errors import AutoReconnect, OperationFailure from pymongo.pool import NO_REQUEST, NO_SOCKET_YET, SocketInfo from pymongo.server_selectors import (any_server_selector, writable_server_selector) from test import (client_context, db_user, - db_pwd) + db_pwd, + host, + port) from test.version import Version @@ -39,11 +43,27 @@ def get_client(*args, **kwargs): return client -def get_rs_client(*args, **kwargs): - client = MongoReplicaSetClient(*args, **kwargs) - if client_context.auth_enabled and kwargs.get("_connect", True): - client.admin.authenticate(db_user, db_pwd) - return client +def rs_or_single_client(h=host, p=port, **kwargs): + """Connect to the replica set if there is one, otherwise the standalone. + + Authenticates if necessary. + """ + if client_context.setname: + return get_client(h, p, replicaSet=client_context.setname, **kwargs) + else: + return get_client(h, p, **kwargs) + + +def rs_or_single_client_noauth(h=host, p=port, **kwargs): + """Connect to the replica set if there is one, otherwise the standalone. + + Like rs_or_single_client, but does not authenticate. + """ + # Just call MongoClient(), not get_client() which does auth. + if client_context.setname: + return MongoClient(h, p, replicaSet=client_context.setname, **kwargs) + else: + return MongoClient(h, p, **kwargs) # No functools in Python 2.4 @@ -160,7 +180,12 @@ def joinall(threads): def connected(client): """Convenience to wait for a newly-constructed client to connect.""" - client.admin.command('ismaster') # Force connection. + with warnings.catch_warnings(): + # Ignore warning that "ismaster" is always routed to primary even + # if client's read preference isn't PRIMARY. + warnings.simplefilter("ignore", UserWarning) + client.admin.command('ismaster') # Force connection. + return client def wait_until(predicate, success_description, timeout=10): @@ -187,12 +212,11 @@ def enable_text_search(client): client.admin.command( 'setParameter', textSearchEnabled=True) - if isinstance(client, MongoReplicaSetClient): - for host, port in client.secondaries: - client = MongoClient(host, port) - if client_context.auth_enabled: - client.admin.authenticate(db_user, db_pwd) - client.admin.command('setParameter', textSearchEnabled=True) + for host, port in client.secondaries: + client = MongoClient(host, port) + if client_context.auth_enabled: + client.admin.authenticate(db_user, db_pwd) + client.admin.command('setParameter', textSearchEnabled=True) def assertRaisesExactly(cls, fn, *args, **kwargs): """ @@ -208,6 +232,11 @@ def assertRaisesExactly(cls, fn, *args, **kwargs): else: raise AssertionError("%s not raised" % cls) +@contextlib.contextmanager +def ignore_deprecations(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + yield class RendezvousThread(threading.Thread): """A thread that starts and pauses at a rendezvous point before resuming. @@ -306,20 +335,22 @@ class RendezvousThread(threading.Thread): self.passed = True def read_from_which_host( - rsc, + client, pref, tag_sets=None, secondary_acceptable_latency_ms=None ): - """Read from a MongoReplicaSetClient with the given Read Preference mode, - tags, and acceptable latency. Return the 'host:port' which was read from. + """Read from a client with the given Read Preference. + + Return the 'host:port' which was read from. :Parameters: - - `rsc`: A MongoReplicaSetClient + - `client`: A MongoClient - `mode`: A ReadPreference - `tag_sets`: List of dicts of tags for data-center-aware reads + - `secondary_acceptable_latency_ms`: Size of latency window """ - db = rsc.pymongo_test + db = client.pymongo_test if isinstance(tag_sets, dict): tag_sets = [tag_sets] @@ -342,27 +373,27 @@ def read_from_which_host( except AutoReconnect: return None -def assertReadFrom(testcase, rsc, member, *args, **kwargs): +def assertReadFrom(testcase, client, member, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from the expected replica-set member. :Parameters: - `testcase`: A unittest.TestCase - - `rsc`: A MongoReplicaSetClient + - `client`: A MongoClient - `member`: A host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads """ for _ in range(10): - testcase.assertEqual(member, read_from_which_host(rsc, *args, **kwargs)) + testcase.assertEqual(member, read_from_which_host(client, *args, **kwargs)) -def assertReadFromAll(testcase, rsc, members, *args, **kwargs): +def assertReadFromAll(testcase, client, members, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from all members in a set, and only members in that set. :Parameters: - `testcase`: A unittest.TestCase - - `rsc`: A MongoReplicaSetClient + - `client`: A MongoClient - `members`: Sequence of host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads @@ -370,18 +401,18 @@ def assertReadFromAll(testcase, rsc, members, *args, **kwargs): members = set(members) used = set() for _ in range(100): - used.add(read_from_which_host(rsc, *args, **kwargs)) + used.add(read_from_which_host(client, *args, **kwargs)) testcase.assertEqual(members, used) def get_pool(client): + """Get the standalone, primary, or mongos pool.""" topology = client._get_topology() server = topology.select_server(writable_server_selector) return server.pool -def pools_from_rs_client(client): - """Get Pool instances from a MongoReplicaSetClient. - """ +def get_pools(client): + """Get all pools.""" return [ server.pool for server in client._get_topology().select_servers(any_server_selector)] @@ -479,7 +510,7 @@ def lazy_client_trial(reset, target, test, get_client): try: for i in range(NTRIALS): reset(collection) - lazy_client = get_client(connect=False) + lazy_client = get_client() lazy_collection = lazy_client.pymongo_test.test run_threads(lazy_collection, target) test(lazy_collection) @@ -495,9 +526,8 @@ def lazy_client_trial(reset, target, test, get_client): class _TestLazyConnectMixin(object): """Test concurrent operations on a lazily-connecting client. - Inherit from this class and from unittest.TestCase, and override - _get_client(self, **kwargs), for testing a lazily-connecting - client, i.e. a client initialized with connect=False. + Inherit from this class and from IntegrationTest, and override + self._get_client() to return a client initialized with connect=False. """ NTRIALS = 5 NTHREADS = 10 @@ -572,7 +602,7 @@ class _TestLazyConnectMixin(object): def test_max_bson_size(self): # Client should have sane defaults before connecting, and should update # its configuration once connected. - c = self._get_client(connect=False) + c = self._get_client() self.assertEqual(16 * (1024 ** 2), c.max_bson_size) self.assertEqual(2 * c.max_bson_size, c.max_message_size)