mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-777 Make copy_database work with SCRAM-SHA-1.

Browse Source 
See doc/examples/copydb.rst for details.
This commit is contained in:
A. Jesse Jiryu Davis 2014-11-06 13:41:41 -05:00
parent f787165d43
commit 0b7b51975e
10 changed files with 355 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
doc/examples/copydb.rst Normal file
View File

@ -0,0 +1,63 @@
Copying a Database
==================
Raw command
-----------
To copy a database within a single mongod process, or between mongod
servers, simply connect to the target mongod and use the
:meth:`~pymongo.database.Database.command` method::
>>> from pymongo import MongoClient
>>> client = MongoClient('target.example.com')
>>> client.admin.command('copydb',
fromdb='source_db_name',
todb='target_db_name')
To copy from a different mongod server that is not password-protected::
>>> client.admin.command('copydb',
fromdb='source_db_name',
todb='target_db_name',
fromhost='source.example.com')
If the target server is password-protected, authenticate to the "admin"
database first::
>>> client.admin.authenticate('administrator', 'pwd')
True
>>> client.admin.command('copydb',
fromdb='source_db_name',
todb='target_db_name',
fromhost='source.example.com')
See the :doc:`authentication examples </examples/authentication>`.
``copy_database`` method
------------------------
The current version of PyMongo provides a helper method,
:meth:`~pymongo.mongo_client.MongoClient.copy_database`, to copy a database
from a password-protected mongod server to the target server::
>>> client = MongoClient('target.example.com')
>>> client.copy_database(from_name='source_db_name',
to_name='target_db_name',
from_host='source.example.com',
username='jesse',
password='pwd',
mechanism='SCRAM-SHA-1')
Provide the username and password of a user who is authorized to read the
source database on the source host. Again, if the target database is also
password-protected, authenticate to the "admin" database first.
The mechanism can be "MONGODB-CR" or "SCRAM-SHA-1". Use SCRAM-SHA-1 if the
target and source hosts are both MongoDB 2.8 or later, otherwise use
MONGODB-CR.
If no mechanism is specified, PyMongo tries to use MONGODB-CR when
connected to a pre-2.8 version of MongoDB, and SCRAM-SHA-1 when connected to
a recent version. However, since PyMongo cannot determine the MongoDB
version of the **source** host, it is better if you specify a mechanism
yourself.

1
doc/examples/index.rst
View File

@ -18,6 +18,7 @@ MongoDB, you can start it like so:
aggregation
authentication
copydb
bulk
custom_type
geo

89
pymongo/auth.py
View File

@ -144,8 +144,26 @@ def _parse_scram_response(response):
"""Split a scram response into key, value pairs."""
return dict([item.split(_EQUAL, 1) for item in response.split(_COMMA)])
def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
"""Authenticate using SCRAM-SHA-1."""
def _scram_sha1_conversation(
credentials,
sock_info,
cmd_func,
sasl_start,
sasl_continue):
"""Authenticate or copydb using SCRAM-SHA-1.
sasl_start and sasl_continue are SONs, the base command documents for
beginning and continuing the SASL conversation. They may be modified
by the callee.
:Parameters:
- `credentials`: A credentials tuple from _build_credentials_tuple.
- `sock_info`: A SocketInfo instance.
- `cmd_func`: A callback taking args sock_info, database, command doc.
- `sasl_start`: A SON.
- `sasl_continue`: A SON.
"""
source, username, password = credentials
# Make local
@ -159,11 +177,8 @@ def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
(("%s" % (SystemRandom().random(),))[2:]).encode("utf-8"))
first_bare = b("n=") + user + b(",r=") + nonce
cmd = SON([('saslStart', 1),
('mechanism', 'SCRAM-SHA-1'),
('payload', Binary(b("n,,") + first_bare)),
('autoAuthorize', 1)])
res, _ = cmd_func(sock_info, source, cmd)
sasl_start['payload'] = Binary(b("n,,") + first_bare)
res, _ = cmd_func(sock_info, source, sasl_start)
server_first = res['payload']
parsed = _parse_scram_response(server_first)
@ -187,9 +202,9 @@ def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
server_sig = standard_b64encode(
_hmac(server_key, auth_msg, _SHA1MOD).digest())
cmd = SON([('saslContinue', 1),
('conversationId', res['conversationId']),
('payload', Binary(client_final))])
cmd = sasl_continue.copy()
cmd['conversationId'] = res['conversationId']
cmd['payload'] = Binary(client_final)
res, _ = cmd_func(sock_info, source, cmd)
parsed = _parse_scram_response(res['payload'])
@ -198,14 +213,62 @@ def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
# Depending on how it's configured, Cyrus SASL (which the server uses)
# requires a third empty challenge.
if not res['done']:
cmd = SON([('saslContinue', 1),
('conversationId', res['conversationId']),
('payload', Binary(_EMPTY))])
cmd = sasl_continue.copy()
cmd['conversationId'] = res['conversationId']
cmd['payload'] = Binary(_EMPTY)
res, _ = cmd_func(sock_info, source, cmd)
if not res['done']:
raise OperationFailure('SASL conversation failed to complete.')
def _authenticate_scram_sha1(credentials, sock_info, cmd_func):
"""Authenticate using SCRAM-SHA-1."""
# Base commands for starting and continuing SASL authentication.
sasl_start = SON([('saslStart', 1),
('mechanism', 'SCRAM-SHA-1'),
('autoAuthorize', 1)])
sasl_continue = SON([('saslContinue', 1)])
_scram_sha1_conversation(credentials, sock_info, cmd_func,
sasl_start, sasl_continue)
def _copydb_scram_sha1(
credentials,
sock_info,
cmd_func,
fromdb,
todb,
fromhost):
"""Copy a database using SCRAM-SHA-1 authentication.
:Parameters:
- `credentials`: A tuple, (mechanism, source, username, password).
- `sock_info`: A SocketInfo instance.
- `cmd_func`: A callback taking args sock_info, database, command doc.
- `fromdb`: Source database.
- `todb`: Target database.
- `fromhost`: Source host or None.
"""
assert credentials[0] == 'SCRAM-SHA-1'
sasl_start = SON([('copydbsaslstart', 1),
('mechanism', 'SCRAM-SHA-1'),
('autoAuthorize', 1),
('fromdb', fromdb),
('fromhost', fromhost)])
sasl_continue = SON([('copydb', 1),
('fromdb', fromdb),
('fromhost', fromhost),
('todb', todb)])
_scram_sha1_conversation(credentials[1:],
sock_info,
cmd_func,
sasl_start,
sasl_continue)
def _password_digest(username, password):
"""Get a password digest to use for authentication.
"""

15
pymongo/database.py
View File

@ -24,7 +24,6 @@ from pymongo import auth, common, helpers
from pymongo.collection import Collection
from pymongo.errors import (CollectionInvalid,
ConfigurationError,
InvalidName,
OperationFailure)
from pymongo.read_preferences import (modes,
secondary_ok_commands,
@ -32,18 +31,6 @@ from pymongo.read_preferences import (modes,
from pymongo.son_manipulator import SONManipulator
def _check_name(name):
"""Check if a database name is valid.
"""
if not name:
raise InvalidName("database name cannot be the empty string")
for invalid_char in [" ", ".", "$", "/", "\\", "\x00"]:
if invalid_char in name:
raise InvalidName("database names cannot contain the "
"character %r" % invalid_char)
class Database(common.BaseObject):
"""A Mongo database.
"""
@ -77,7 +64,7 @@ class Database(common.BaseObject):
"of %s" % (basestring.__name__,))
if name != '$external':
_check_name(name)
helpers._check_database_name(name)
self.__name = unicode(name)
self.__connection = connection

113
pymongo/helpers.py
View File

@ -22,9 +22,12 @@ import pymongo
from bson.binary import OLD_UUID_SUBTYPE
from bson.son import SON
from pymongo import auth
from pymongo.errors import (AutoReconnect,
CursorNotFound,
DuplicateKeyError,
InvalidName,
InvalidOperation,
OperationFailure,
ExecutionTimeout,
WTimeoutError)
@ -223,6 +226,116 @@ def _fields_list_to_dict(fields):
return as_dict
def _check_database_name(name):
"""Check if a database name is valid."""
if not name:
raise InvalidName("database name cannot be the empty string")
for invalid_char in [" ", ".", "$", "/", "\\", "\x00"]:
if invalid_char in name:
raise InvalidName("database names cannot contain the "
"character %r" % invalid_char)
def _copy_database(
fromdb,
todb,
fromhost,
mechanism,
username,
password,
sock_info,
cmd_func):
"""Copy a database, perhaps from a remote host.
:Parameters:
- `fromdb`: Source database.
- `todb`: Target database.
- `fromhost`: Source host like 'foo.com', 'foo.com:27017', or None.
- `mechanism`: An authentication mechanism.
- `username`: A str or unicode, or None.
- `password`: A str or unicode, or None.
- `sock_info`: A SocketInfo instance.
- `cmd_func`: A callback taking args sock_info, database, command doc.
"""
if not isinstance(fromdb, basestring):
raise TypeError('from_name must be an instance '
'of %s' % (basestring.__name__,))
if not isinstance(todb, basestring):
raise TypeError('to_name must be an instance '
'of %s' % (basestring.__name__,))
_check_database_name(todb)
# It would be better if the user told us what mechanism to use, but for
# backwards compatibility with earlier PyMongos we don't require the
# mechanism. Hope 'fromhost' runs the same version as the target.
if mechanism == 'DEFAULT':
if sock_info.max_wire_version >= 3:
mechanism = 'SCRAM-SHA-1'
else:
mechanism = 'MONGODB-CR'
if username is not None:
if mechanism == 'SCRAM-SHA-1':
credentials = auth._build_credentials_tuple(mech=mechanism,
source='admin',
user=username,
passwd=password,
extra=None)
try:
auth._copydb_scram_sha1(credentials=credentials,
sock_info=sock_info,
cmd_func=cmd_func,
fromdb=fromdb,
todb=todb,
fromhost=fromhost)
except OperationFailure, exc:
errmsg = exc.details and exc.details.get('errmsg') or ''
if 'no such cmd: saslStart' in errmsg:
explanation = (
"%s doesn't support SCRAM-SHA-1, pass"
" mechanism='MONGODB-CR' to copy_database" % fromhost)
raise OperationFailure(explanation,
exc.code,
exc.details)
else:
raise
elif mechanism == 'MONGODB-CR':
get_nonce_cmd = SON([('copydbgetnonce', 1),
('fromhost', fromhost)])
get_nonce_response, _ = cmd_func(sock_info, 'admin', get_nonce_cmd)
nonce = get_nonce_response['nonce']
copydb_cmd = SON([('copydb', 1),
('fromdb', fromdb),
('todb', todb)])
copydb_cmd['username'] = username
copydb_cmd['nonce'] = nonce
copydb_cmd['key'] = auth._auth_key(nonce, username, password)
if fromhost is not None:
copydb_cmd['fromhost'] = fromhost
cmd_func(sock_info, 'admin', copydb_cmd)
else:
raise InvalidOperation('Authentication mechanism %r not supported'
' for copy_database' % mechanism)
else:
# No username.
copydb_cmd = SON([('copydb', 1),
('fromdb', fromdb),
('todb', todb)])
if fromhost:
copydb_cmd['fromhost'] = fromhost
cmd_func(sock_info, 'admin', copydb_cmd)
def shuffled(sequence):
"""Returns a copy of the sequence (as a :class:`list`) which has been
shuffled by :func:`random.shuffle`.

54
pymongo/mongo_client.py
View File

@ -1398,7 +1398,8 @@ class MongoClient(common.BaseObject):
read_preference=ReadPreference.PRIMARY)
def copy_database(self, from_name, to_name,
from_host=None, username=None, password=None):
from_host=None, username=None, password=None,
mechanism='DEFAULT'):
"""Copy a database, potentially from another host.
Raises :class:`TypeError` if `from_name` or `to_name` is not
@ -1410,7 +1411,9 @@ class MongoClient(common.BaseObject):
source. Otherwise the database is copied from `from_host`.
If the source database requires authentication, `username` and
`password` must be specified.
`password` must be specified. By default, use SCRAM-SHA-1 with
MongoDB 2.8 and later, MONGODB-CR (MongoDB Challenge Response
protocol) for older servers.
:Parameters:
- `from_name`: the name of the source database
@ -1418,42 +1421,27 @@ class MongoClient(common.BaseObject):
- `from_host` (optional): host name to copy from
- `username` (optional): username for source database
- `password` (optional): password for source database
- `mechanism` (optional): auth method, 'MONGODB-CR' or 'SCRAM-SHA-1'
.. note:: Specifying `username` and `password` requires server
version **>= 1.3.3+**.
.. seealso:: The :doc:`copy_database examples </examples/copydb>`.
.. versionadded:: 1.5
.. versionadded:: 2.8
SCRAM-SHA-1 support.
"""
if not isinstance(from_name, basestring):
raise TypeError("from_name must be an instance "
"of %s" % (basestring.__name__,))
if not isinstance(to_name, basestring):
raise TypeError("to_name must be an instance "
"of %s" % (basestring.__name__,))
database._check_name(to_name)
command = {"fromdb": from_name, "todb": to_name}
if from_host is not None:
command["fromhost"] = from_host
member = self.__ensure_member()
sock_info = self.__socket(member)
try:
self.start_request()
if username is not None:
nonce = self.admin.command("copydbgetnonce",
read_preference=ReadPreference.PRIMARY,
fromhost=from_host)["nonce"]
command["username"] = username
command["nonce"] = nonce
command["key"] = auth._auth_key(nonce, username, password)
return self.admin.command("copydb",
read_preference=ReadPreference.PRIMARY,
**command)
helpers._copy_database(
fromdb=from_name,
todb=to_name,
fromhost=from_host,
mechanism=mechanism,
username=username,
password=password,
sock_info=sock_info,
cmd_func=self.__simple_command)
finally:
self.end_request()
member.pool.maybe_return_socket(sock_info)
def get_default_database(self):
"""Get the database named in the MongoDB connection URI.

54
pymongo/mongo_replica_set_client.py
View File

@ -1885,7 +1885,8 @@ class MongoReplicaSetClient(common.BaseObject):
read_preference=ReadPreference.PRIMARY)
def copy_database(self, from_name, to_name,
from_host=None, username=None, password=None):
from_host=None, username=None, password=None,
mechanism='DEFAULT'):
"""Copy a database, potentially from another host.
Raises :class:`TypeError` if `from_name` or `to_name` is not
@ -1897,7 +1898,9 @@ class MongoReplicaSetClient(common.BaseObject):
source. Otherwise the database is copied from `from_host`.
If the source database requires authentication, `username` and
`password` must be specified.
`password` must be specified. By default, use SCRAM-SHA-1 with
MongoDB 2.8 and later, MONGODB-CR (MongoDB Challenge Response
protocol) for older servers.
:Parameters:
- `from_name`: the name of the source database
@ -1905,40 +1908,27 @@ class MongoReplicaSetClient(common.BaseObject):
- `from_host` (optional): host name to copy from
- `username` (optional): username for source database
- `password` (optional): password for source database
- `mechanism` (optional): auth method, 'MONGODB-CR' or 'SCRAM-SHA-1'
.. note:: Specifying `username` and `password` requires server
version **>= 1.3.3+**.
.. seealso:: The :doc:`copy_database examples </examples/copydb>`.
.. versionadded:: 2.8
SCRAM-SHA-1 support.
"""
if not isinstance(from_name, basestring):
raise TypeError("from_name must be an instance "
"of %s" % (basestring.__name__,))
if not isinstance(to_name, basestring):
raise TypeError("to_name must be an instance "
"of %s" % (basestring.__name__,))
database._check_name(to_name)
command = {"fromdb": from_name, "todb": to_name}
if from_host is not None:
command["fromhost"] = from_host
member = self.__find_primary()
sock_info = self.__socket(member)
try:
self.start_request()
if username is not None:
nonce = self.admin.command("copydbgetnonce",
read_preference=ReadPreference.PRIMARY,
fromhost=from_host)["nonce"]
command["username"] = username
command["nonce"] = nonce
command["key"] = auth._auth_key(nonce, username, password)
return self.admin.command("copydb",
read_preference=ReadPreference.PRIMARY,
**command)
helpers._copy_database(
fromdb=from_name,
todb=to_name,
fromhost=from_host,
mechanism=mechanism,
username=username,
password=password,
sock_info=sock_info,
cmd_func=self.__simple_command)
finally:
self.end_request()
member.pool.maybe_return_socket(sock_info)
def get_default_database(self):
"""Get the database named in the MongoDB connection URI.

55
test/test_auth.py
View File

@ -36,7 +36,7 @@ from pymongo.errors import (OperationFailure,
ConnectionFailure,
AutoReconnect)
from pymongo.read_preferences import ReadPreference
from test import version, host, port, pair, auth_context
from test import version, host, port, pair, auth_context, db_user, db_pwd
from test.test_bulk import BulkTestBase
from test.test_client import get_client
from test.test_pooling_base import get_pool
@ -279,11 +279,13 @@ class TestSCRAMSHA1(unittest.TestCase):
ismaster = client.admin.command('ismaster')
self.set_name = ismaster.get('setName')
cmd_line = get_command_line(client)
if 'SCRAM-SHA-1' not in cmd_line.get(
'parsed', {}).get('setParameter',
{}).get('authenticationMechanisms', ''):
raise SkipTest('SCRAM-SHA-1 mechanism not enabled')
# SCRAM-SHA-1 is always enabled beginning in 2.7.8.
if not version.at_least(client, (2, 7, 8)):
cmd_line = get_command_line(client)
if 'SCRAM-SHA-1' not in cmd_line.get(
'parsed', {}).get('setParameter',
{}).get('authenticationMechanisms', ''):
raise SkipTest('SCRAM-SHA-1 mechanism not enabled')
if self.set_name:
client.pymongo_test.add_user('user', 'pass',
@ -305,8 +307,8 @@ class TestSCRAMSHA1(unittest.TestCase):
client.pymongo_test.command('dbstats')
if self.set_name:
client = MongoReplicaSetClient(host, port,
replicaSet='%s' % (self.set_name,))
client = MongoReplicaSetClient(
'mongodb://localhost:%d/?replicaSet=%s' % (port, self.set_name))
self.assertTrue(client.pymongo_test.authenticate(
'user', 'pass', mechanism='SCRAM-SHA-1'))
client.pymongo_test.command('dbstats')
@ -319,6 +321,43 @@ class TestSCRAMSHA1(unittest.TestCase):
client.read_preference = ReadPreference.SECONDARY
client.pymongo_test.command('dbstats')
def test_copy_db_scram_sha_1(self):
auth_context.client.drop_database('pymongo_test2')
if self.set_name:
client = MongoReplicaSetClient(
'mongodb://localhost:%d/?replicaSet=%s' % (port, self.set_name))
else:
client = MongoClient(host, port)
client.admin.authenticate(db_user, db_pwd, mechanism='SCRAM-SHA-1')
try:
client.pymongo_test.collection.insert({})
# No from_host.
client.copy_database(from_name='pymongo_test',
to_name='pymongo_test2',
username='user',
password='pass')
self.assertTrue('pymongo_test2'
in auth_context.client.database_names())
# With from_host.
client.copy_database(from_name='pymongo_test',
to_name='pymongo_test3',
from_host='%s:%s' % (host, port),
username='user',
password='pass')
self.assertTrue('pymongo_test3'
in auth_context.client.database_names())
finally:
auth_context.client.drop_database('pymongo_test')
auth_context.client.drop_database('pymongo_test2')
auth_context.client.drop_database('pymongo_test3')
def tearDown(self):
auth_context.client.pymongo_test.remove_user('user')

30
test/test_client.py
View File

@ -261,46 +261,34 @@ class TestClient(unittest.TestCase, TestRequestMixin):
# from a master in a master-slave pair.
if server_is_master_with_slave(c):
raise SkipTest("SERVER-2329")
if (not version.at_least(c, (2, 6, 0)) and
is_mongos(c) and server_started_with_auth(c)):
raise SkipTest("Need mongos >= 2.6.0 to test with authentication")
# We test copy twice; once starting in a request and once not. In
# either case the copy should succeed (because it starts a request
# internally) and should leave us in the same state as before the copy.
c.start_request()
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.drop_database("pymongo_test1")
c.drop_database("pymongo_test2")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertFalse("pymongo_test2" in c.database_names())
c.pymongo_test.test.insert({"foo": "bar"})
c.copy_database("pymongo_test", "pymongo_test1")
# copy_database() didn't accidentally end the request
self.assertTrue(c.in_request())
c.drop_database("pymongo_test1")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.end_request()
c.drop_database("pymongo_test1")
# XXX - SERVER-15318
if not (version.at_least(c, (2, 6, 4)) and is_mongos(c)):
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test2",
c.copy_database("pymongo_test", "pymongo_test1",
"%s:%d" % (host, port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test2" in c.database_names())
self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.drop_database("pymongo_test1")
def test_iteration(self):
client = MongoClient(host, port)

27
test/test_replica_set_client.py
View File

@ -417,43 +417,20 @@ class TestReplicaSetClient(TestReplicaSetClientBase, TestRequestMixin):
def test_copy_db(self):
c = self._get_client()
# We test copy twice; once starting in a request and once not. In
# either case the copy should succeed (because it starts a request
# internally) and should leave us in the same state as before the copy.
c.start_request()
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.drop_database("pymongo_test1")
c.drop_database("pymongo_test2")
c.pymongo_test.test.insert({"foo": "bar"})
c.drop_database("pymongo_test1")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertFalse("pymongo_test2" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1")
# copy_database() didn't accidentally end the request
self.assertTrue(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.end_request()
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test2", pair)
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
time.sleep(1)
self.assertTrue("pymongo_test2" in c.database_names())
self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
c.drop_database("pymongo_test1")
def test_get_default_database(self):
host = one(self.hosts)