mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-1378 - Implement DNS seedlist discovery

Browse Source
This commit is contained in:
Bernie Hackett 2017-10-19 09:36:20 -07:00
parent 6721e0157b
commit 732b0f15df
14 changed files with 297 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.rst
View File

@ -90,7 +90,7 @@ Dependencies
PyMongo supports CPython 2.6, 2.7, 3.4+, PyPy, and PyPy3.
Optional dependencies for GSSAPI and TLS:
Optional dependencies:
GSSAPI authentication requires `pykerberos
<https://pypi.python.org/pypi/pykerberos>`_ on Unix or `WinKerberos
@ -99,6 +99,11 @@ dependency can be installed automatically along with PyMongo::
$ python -m pip install pymongo[gssapi]
Support for mongodb+srv:// URIs requires `dnspython
<https://pypi.python.org/pypi/dnspython>`_::
$ python -m pip install pymongo[srv]
TLS / SSL support may require `ipaddress
<https://pypi.python.org/pypi/ipaddress>`_ and `certifi
<https://pypi.python.org/pypi/certifi>`_ or `wincertstore
@ -108,10 +113,10 @@ PyMongo::
$ python -m pip install pymongo[tls]
You can install both dependencies automatically with the following
You can install all dependencies automatically with the following
command::
$ python -m pip install pymongo[gssapi,tls]
$ python -m pip install pymongo[gssapi,srv,tls]
Other optional packages:

2
doc/changelog.rst
View File

@ -17,6 +17,8 @@ Highlights include:
- New methods :meth:`~pymongo.collection.Collection.find_raw_batches` and
:meth:`~pymongo.collection.Collection.aggregate_raw_batches` for use with
external libraries that can parse raw batches of BSON data.
- Support for mongodb+srv:// URIs. See
:class:`~pymongo.mongo_client.MongoClient` for details.
Breaking changes include:

11
doc/installation.rst
View File

@ -47,7 +47,7 @@ Dependencies
PyMongo supports CPython 2.6, 2.7, 3.4+, PyPy, and PyPy3.
Optional dependencies for GSSAPI and TLS:
Optional dependencies:
GSSAPI authentication requires `pykerberos
<https://pypi.python.org/pypi/pykerberos>`_ on Unix or `WinKerberos
@ -56,6 +56,11 @@ dependency can be installed automatically along with PyMongo::
$ python -m pip install pymongo[gssapi]
Support for mongodb+srv:// URIs requires `dnspython
<https://pypi.python.org/pypi/dnspython>`_::
$ python -m pip install pymongo[srv]
TLS / SSL support may require `ipaddress
<https://pypi.python.org/pypi/ipaddress>`_ and `certifi
<https://pypi.python.org/pypi/certifi>`_ or `wincertstore
@ -65,10 +70,10 @@ PyMongo::
$ python -m pip install pymongo[tls]
You can install both dependencies automatically with the following
You can install all dependencies automatically with the following
command::
$ python -m pip install pymongo[gssapi,tls]
$ python -m pip install pymongo[gssapi,srv,tls]
Other optional packages:

35
pymongo/mongo_client.py
View File

@ -126,6 +126,21 @@ class MongoClient(common.BaseObject):
client = MongoClient('/tmp/mongodb-27017.sock')
Starting with version 3.6, PyMongo supports mongodb+srv:// URIs. The
URI must include one, and only one, hostname. The hostname will be
resolved to one or more DNS `SRV records
<https://en.wikipedia.org/wiki/SRV_record>`_ which will be used
as the seed list for connecting to the MongoDB deployment. When using
SRV support configuration options can be specified using `TXT records
<https://en.wikipedia.org/wiki/TXT_record>`_. See the
`Initial DNS Seedlist Discovery spec
<https://github.com/mongodb/specifications/blob/master/source/
initial-dns-seedlist-discovery/initial-dns-seedlist-discovery.rst>`_
for more details.
.. note:: MongoClient creation will block waiting for answers from
DNS when mongodb+srv:// URIs are used.
.. note:: Starting with version 3.0 the :class:`MongoClient`
constructor no longer blocks while connecting to the server or
servers, and it no longer raises
@ -338,6 +353,9 @@ class MongoClient(common.BaseObject):
.. mongodoc:: connections
.. versionchanged:: 3.6
Added support for mongodb+srv:// URIs.
.. versionchanged:: 3.5
Add ``username`` and ``password`` options. Document the
``authSource``, ``authMechanism``, and ``authMechanismProperties ``
@ -420,17 +438,12 @@ class MongoClient(common.BaseObject):
opts = {}
for entity in host:
if "://" in entity:
if entity.startswith("mongodb://"):
res = uri_parser.parse_uri(entity, port, warn=True)
seeds.update(res["nodelist"])
username = res["username"] or username
password = res["password"] or password
dbase = res["database"] or dbase
opts = res["options"]
else:
idx = entity.find("://")
raise InvalidURI("Invalid URI scheme: "
"%s" % (entity[:idx],))
res = uri_parser.parse_uri(entity, port, warn=True)
seeds.update(res["nodelist"])
username = res["username"] or username
password = res["password"] or password
dbase = res["database"] or dbase
opts = res["options"]
else:
seeds.update(uri_parser.split_hosts(entity, port))
if not seeds:

78
pymongo/uri_parser.py
View File

@ -17,7 +17,14 @@
import re
import warnings
from bson.py3compat import PY3, iteritems, string_type
try:
from dns import rdata, resolver
from dns.exception import DNSException
_HAVE_DNSPYTHON = True
except ImportError:
_HAVE_DNSPYTHON = False
from bson.py3compat import PY3, string_type
if PY3:
from urllib.parse import unquote_plus
@ -30,6 +37,8 @@ from pymongo.errors import ConfigurationError, InvalidURI
SCHEME = 'mongodb://'
SCHEME_LEN = len(SCHEME)
SRV_SCHEME = 'mongodb+srv://'
SRV_SCHEME_LEN = len(SRV_SCHEME)
DEFAULT_PORT = 27017
@ -258,6 +267,26 @@ def split_hosts(hosts, default_port=DEFAULT_PORT):
_BAD_DB_CHARS = re.compile('[' + re.escape(r'/ "$') + ']')
def _get_dns_srv_hosts(hostname):
try:
results = resolver.query('_mongodb._tcp.' + hostname, 'SRV')
return [(res.target.to_text(omit_final_dot=True), res.port)
for res in results]
except DNSException as exc:
raise ConfigurationError(str(exc))
def _get_dns_txt_options(hostname):
try:
results = resolver.query(hostname, 'TXT')
return '&'.join(['&'.join(["%s" % (rdata._escapify(ent),)
for ent in res.strings])
for res in results])
except resolver.NoAnswer:
# No TXT records
return None
def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False):
"""Parse and validate a MongoDB URI.
@ -272,6 +301,9 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False):
'options': <dict of MongoDB URI options>
}
If the URI scheme is "mongodb+srv://" DNS SRV and TXT lookups will be done
to build nodelist and options.
:Parameters:
- `uri`: The MongoDB URI to parse.
- `default_port`: The port number to use when one wasn't specified
@ -283,6 +315,9 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False):
validation will error when options are unsupported or values are
invalid.
.. versionchanged:: 3.6
Added support for mongodb+srv:// URIs
.. versionchanged:: 3.5
Return the original value of the ``readPreference`` MongoDB URI option
instead of the validated read preference mode.
@ -290,11 +325,18 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False):
.. versionchanged:: 3.1
``warn`` added so invalid options can be ignored.
"""
if not uri.startswith(SCHEME):
raise InvalidURI("Invalid URI scheme: URI "
"must begin with '%s'" % (SCHEME,))
scheme_free = uri[SCHEME_LEN:]
if uri.startswith(SCHEME):
is_srv = False
scheme_free = uri[SCHEME_LEN:]
elif uri.startswith(SRV_SCHEME):
if not _HAVE_DNSPYTHON:
raise ConfigurationError('The "dnspython" module must be '
'installed to use mongodb+srv:// URIs')
is_srv = True
scheme_free = uri[SRV_SCHEME_LEN:]
else:
raise InvalidURI("Invalid URI scheme: URI must "
"begin with '%s' or '%s'", (SCHEME, SRV_SCHEME))
if not scheme_free:
raise InvalidURI("Must provide at least one hostname or IP.")
@ -325,7 +367,23 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False):
" percent-encoded: %s" % host_part)
hosts = unquote_plus(hosts)
nodes = split_hosts(hosts, default_port=default_port)
if is_srv:
nodes = split_hosts(hosts, default_port=None)
if len(nodes) != 1:
raise InvalidURI(
"%s URIs must include one, "
"and only one, hostname" % (SRV_SCHEME,))
hostname, port = nodes[0]
if port is not None:
raise InvalidURI(
"%s URIs must not include a port number" % (SRV_SCHEME,))
nodes = _get_dns_srv_hosts(hostname)
dns_options = _get_dns_txt_options(hostname)
if dns_options:
options = split_options(dns_options, validate, warn)
else:
nodes = split_hosts(hosts, default_port=default_port)
if path_part:
if path_part[0] == '?':
@ -339,7 +397,7 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False):
raise InvalidURI('Bad database name "%s"' % dbase)
if opts:
options = split_options(opts, validate, warn)
options.update(split_options(opts, validate, warn))
if dbase is not None:
dbase = unquote_plus(dbase)
@ -361,6 +419,6 @@ if __name__ == '__main__':
import sys
try:
pprint.pprint(parse_uri(sys.argv[1]))
except InvalidURI as e:
print(e)
except InvalidURI as exc:
print(exc)
sys.exit(0)

5
setup.py
View File

@ -316,10 +316,11 @@ ext_modules = [Extension('bson._cbson',
sources=['pymongo/_cmessagemodule.c',
'bson/buffer.c'])]
extras_require = {'tls': []}
vi = sys.version_info
if vi[0] == 2:
extras_require['tls'].append("ipaddress")
extras_require = {'tls': ["ipaddress"], 'srv': ["dnspython>=1.8.0,<2.0.0"]}
else:
extras_require = {'tls': [], 'srv': ["dnspython>=1.15.0,<2.0.0"]}
if sys.platform == 'win32':
extras_require['gssapi'] = ["winkerberos>=0.5.0"]
if vi[0] == 2 and vi < (2, 7, 9) or vi[0] == 3 and vi < (3, 4):

5
test/dns/no-results.json Normal file
View File

@ -0,0 +1,5 @@
{
"uri": "mongodb+srv://test4.test.build.10gen.cc/",
"seeds": [],
"hosts": []
}

11
test/dns/one-result-default-port.json Normal file
View File

@ -0,0 +1,11 @@
{
"uri": "mongodb+srv://test3.test.build.10gen.cc/?replicaSet=repl0",
"seeds": [
"localhost.build.10gen.cc:27017"
],
"hosts": [
"localhost:27017",
"localhost:27018",
"localhost:27019"
]
}

16
test/dns/one-txt-record.json Normal file
View File

@ -0,0 +1,16 @@
{
"uri": "mongodb+srv://test5.test.build.10gen.cc/?replicaSet=repl0",
"seeds": [
"localhost.build.10gen.cc:27017"
],
"hosts": [
"localhost:27017",
"localhost:27018",
"localhost:27019"
],
"options": {
"connectTimeoutMS": 300000,
"replicaSet": "repl0",
"socketTimeoutMS": 300000
}
}

12
test/dns/two-results-default-port.json Normal file
View File

@ -0,0 +1,12 @@
{
"uri": "mongodb+srv://test1.test.build.10gen.cc/?replicaSet=repl0",
"seeds": [
"localhost.build.10gen.cc:27017",
"localhost.build.10gen.cc:27018"
],
"hosts": [
"localhost:27017",
"localhost:27018",
"localhost:27019"
]
}

12
test/dns/two-results-nonstandard-port.json Normal file
View File

@ -0,0 +1,12 @@
{
"uri": "mongodb+srv://test2.test.build.10gen.cc/?replicaSet=repl0",
"seeds": [
"localhost.build.10gen.cc:27018",
"localhost.build.10gen.cc:27019"
],
"hosts": [
"localhost:27017",
"localhost:27018",
"localhost:27019"
]
}

16
test/dns/two-txt-records-with-override.json Normal file
View File

@ -0,0 +1,16 @@
{
"uri": "mongodb+srv://test6.test.build.10gen.cc/?replicaSet=repl0&connectTimeoutMS=250000",
"seeds": [
"localhost.build.10gen.cc:27017"
],
"hosts": [
"localhost:27017",
"localhost:27018",
"localhost:27019"
],
"options": {
"connectTimeoutMS": 250000,
"replicaSet": "repl0",
"socketTimeoutMS": 200000
}
}

16
test/dns/two-txt-records.json Normal file
View File

@ -0,0 +1,16 @@
{
"uri": "mongodb+srv://test6.test.build.10gen.cc/?replicaSet=repl0",
"seeds": [
"localhost.build.10gen.cc:27017"
],
"hosts": [
"localhost:27017",
"localhost:27018",
"localhost:27019"
],
"options": {
"connectTimeoutMS": 200000,
"replicaSet": "repl0",
"socketTimeoutMS": 200000
}
}

96
test/test_dns.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright 2017 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run the SRV support tests."""
import glob
import json
import os
import sys
sys.path[0:0] = [""]
from pymongo.errors import ConfigurationError
from pymongo.mongo_client import MongoClient
from pymongo.uri_parser import parse_uri, split_hosts, _HAVE_DNSPYTHON
from test import client_context, unittest
from test.utils import wait_until
_TEST_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'dns')
_SSL_OPTS = client_context.ssl_client_options.copy()
if client_context.ssl is True:
# Our test certs don't support the SRV hosts used in these tests.
_SSL_OPTS['ssl_match_hostname'] = False
class TestDNS(unittest.TestCase):
pass
def create_test(test_case):
@client_context.require_replica_set
def run_test(self):
if not _HAVE_DNSPYTHON:
raise unittest.SkipTest("DNS tests require the dnspython module")
uri = test_case['uri']
seeds = test_case['seeds']
hosts = test_case['hosts']
options = test_case.get('options')
if seeds:
seeds = split_hosts(','.join(seeds))
if hosts:
hosts = frozenset(split_hosts(','.join(hosts)))
if options:
for key, value in options.items():
# Convert numbers to strings for comparison
options[key] = str(value)
if seeds:
result = parse_uri(uri, validate=False)
self.assertEqual(sorted(result['nodelist']), sorted(seeds))
if options:
self.assertEqual(result['options'], options)
hostname = next(iter(client_context.client.nodes))[0]
# The replica set members must be configured as 'localhost'.
if hostname == 'localhost':
client = MongoClient(uri, **_SSL_OPTS)
# Force server selection
client.admin.command('ismaster')
wait_until(
lambda: hosts == client.nodes,
'match test hosts to client nodes')
else:
self.assertRaises(
ConfigurationError, parse_uri, uri, validate=False)
return run_test
def create_tests():
for filename in glob.glob(os.path.join(_TEST_PATH, '*.json')):
test_suffix, _ = os.path.splitext(os.path.basename(filename))
with open(filename) as dns_test_file:
test_method = create_test(json.load(dns_test_file))
setattr(TestDNS, 'test_' + test_suffix, test_method)
create_tests()
if __name__ == '__main__':
unittest.main()