From e26dc96e31789e7c6ea1991ff1be44c4065a3403 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 31 Jan 2020 16:24:31 -0800 Subject: [PATCH] PYTHON-2034 Support MONGODB-AWS authentication mechanism Use botocore to perform the manual Signature Version 4 Signing Process. Test MONGODB-AWS in Evergreen. Properly unquote URI option values in authMechanismProperties and readPreferenceTags. --- .evergreen/config.yml | 194 ++++++++++++++++++++++++ .evergreen/run-mongodb-aws-ecs-test.sh | 48 ++++++ .evergreen/run-mongodb-aws-test.sh | 45 ++++++ README.rst | 15 +- doc/changelog.rst | 5 +- doc/examples/authentication.rst | 119 +++++++++++++++ doc/examples/encryption.rst | 2 + doc/installation.rst | 13 +- pymongo/auth.py | 39 ++++- pymongo/auth_aws.py | 201 +++++++++++++++++++++++++ pymongo/common.py | 22 ++- pymongo/database.py | 4 +- pymongo/mongo_client.py | 6 +- pymongo/uri_parser.py | 21 ++- setup.py | 1 + test/auth/connection-string.json | 46 +++++- test/auth_aws/test_auth_aws.py | 60 ++++++++ test/test_auth_spec.py | 10 +- test/test_uri_parser.py | 37 +++++ 19 files changed, 860 insertions(+), 28 deletions(-) create mode 100755 .evergreen/run-mongodb-aws-ecs-test.sh create mode 100755 .evergreen/run-mongodb-aws-test.sh create mode 100644 pymongo/auth_aws.py create mode 100644 test/auth_aws/test_auth_aws.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 757a57480..443b305ac 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -429,6 +429,172 @@ functions: # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) PYTHON_BINARY=${PYTHON_BINARY} ATLAS_REPL='${atlas_repl}' ATLAS_SHRD='${atlas_shrd}' ATLAS_FREE='${atlas_free}' ATLAS_TLS11='${atlas_tls11}' ATLAS_TLS12='${atlas_tls12}' sh ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh + "add aws auth variables to file": + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + cat < ${DRIVERS_TOOLS}/.evergreen/auth_aws/aws_e2e_setup.json + { + "iam_auth_ecs_account" : "${iam_auth_ecs_account}", + "iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}", + "iam_auth_ecs_account_arn": "arn:aws:iam::557821124784:user/authtest_fargate_user", + "iam_auth_ecs_cluster": "${iam_auth_ecs_cluster}", + "iam_auth_ecs_task_definition": "${iam_auth_ecs_task_definition}", + "iam_auth_ecs_subnet_a": "${iam_auth_ecs_subnet_a}", + "iam_auth_ecs_subnet_b": "${iam_auth_ecs_subnet_b}", + "iam_auth_ecs_security_group": "${iam_auth_ecs_security_group}", + + "iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}", + "iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}", + "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}", + + "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}", + "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", + "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}" + } + EOF + + "run aws auth test with regular aws credentials": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/auth_aws + mongo aws_e2e_regular_aws.js + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + alias urlencode='python -c "import sys, urllib as ul; print ul.quote_plus(sys.argv[1])"' + USER=$(urlencode ${iam_auth_ecs_account}) + PASS=$(urlencode ${iam_auth_ecs_secret_access_key}) + MONGODB_URI="mongodb://$USER:$PASS@localhost" + EOF + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + .evergreen/run-mongodb-aws-test.sh + + "run aws auth test with assume role credentials": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/auth_aws + mongo aws_e2e_assume_role.js + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + alias urlencode='python -c "import sys, urllib as ul; print ul.quote_plus(sys.argv[1])"' + USER=$(jq -r '.AccessKeyId' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json) + USER=$(urlencode $USER) + PASS=$(jq -r '.SecretAccessKey' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json) + PASS=$(urlencode $PASS) + SESSION_TOKEN=$(jq -r '.SessionToken' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json) + SESSION_TOKEN=$(urlencode $SESSION_TOKEN) + MONGODB_URI="mongodb://$USER:$PASS@localhost" + EOF + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + .evergreen/run-mongodb-aws-test.sh + + "run aws auth test with aws EC2 credentials": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/auth_aws + mongo aws_e2e_ec2.js + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + .evergreen/run-mongodb-aws-test.sh + + "run aws auth test with aws credentials as environment variables": + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + export AWS_ACCESS_KEY_ID=${iam_auth_ecs_account} + export AWS_SECRET_ACCESS_KEY=${iam_auth_ecs_secret_access_key} + EOF + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + PROJECT_DIRECTORY=${PROJECT_DIRECTORY} .evergreen/run-mongodb-aws-test.sh + + "run aws auth test with aws credentials and session token as environment variables": + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + export AWS_ACCESS_KEY_ID=$(jq -r '.AccessKeyId' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json) + export AWS_SECRET_ACCESS_KEY=$(jq -r '.SecretAccessKey' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json) + export AWS_SESSION_TOKEN=$(jq -r '.SessionToken' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json) + EOF + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + .evergreen/run-mongodb-aws-test.sh + + "run aws ECS auth test": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/auth_aws + + cat < setup.js + const mongo_binaries = "$MONGODB_BINARIES"; + const project_dir = "$PROJECT_DIRECTORY"; + EOF + + mongo --nodb setup.js aws_e2e_ecs.js + cd - + "cleanup": - command: shell.exec params: @@ -968,6 +1134,22 @@ tasks: vars: OCSP_TLS_SHOULD_SUCCEED: "0" + - name: "aws-auth-test" + commands: + - func: "bootstrap mongo-orchestration" + vars: + AUTH: "auth" + # TODO: SSL?? + ORCHESTRATION_FILE: "auth-aws.json" + TOPOLOGY: "server" + - func: "add aws auth variables to file" + - func: "run aws auth test with regular aws credentials" + - func: "run aws auth test with assume role credentials" + - func: "run aws auth test with aws credentials as environment variables" + - func: "run aws auth test with aws credentials and session token as environment variables" + - func: "run aws auth test with aws EC2 credentials" + - func: "run aws ECS auth test" + # }}} - name: "coverage-report" tags: ["coverage"] @@ -1064,6 +1246,10 @@ axes: batchtime: 10080 # 7 days variables: libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1604/master/latest/libmongocrypt.tar.gz + - id: ubuntu-18.04 + display_name: "Ubuntu 18.04" + run_on: ubuntu1804-test + batchtime: 10080 # 7 days - id: ubuntu1604-arm64-small display_name: "Ubuntu 16.04 (ARM64)" run_on: ubuntu1604-arm64-small @@ -1920,6 +2106,14 @@ buildvariants: tasks: - name: ".ocsp" +- matrix_name: "aws-auth-test" + matrix_spec: + platform: ubuntu-18.04 + display_name: "MONGODB-AWS Auth test" + run_on: ubuntu1804-test + tasks: + - name: "aws-auth-test" + # Platform notes # i386 builds of OpenSSL or Cyrus SASL are not available # Ubuntu16.04 ppc64le is only supported by MongoDB 3.4+ diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh new file mode 100755 index 000000000..6be913eff --- /dev/null +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Don't trace since the URI contains a password that shouldn't show up in the logs +set -o errexit # Exit the script with error if any of the commands fail + +############################################ +# Main Program # +############################################ + +if [[ -z "$1" ]]; then + echo "usage: $0 " + exit 1 +fi +export MONGODB_URI="$1" + +if echo "$MONGODB_URI" | grep -q "@"; then + echo "MONGODB_URI unexpectedly contains user credentials in ECS test!"; + exit 1 +fi +# Now we can safely enable xtrace +set -o xtrace + +if command -v virtualenv ; then + VIRTUALENV=$(command -v virtualenv) +else + echo "Installing virtualenv..." + apt install python3-pip -y + pip3 install --user virtualenv + VIRTUALENV='python3 -m virtualenv' +fi + +authtest () { + echo "Running MONGODB-AWS ECS authentication tests with $PYTHON" + $PYTHON --version + + $VIRTUALENV -p $PYTHON --system-site-packages --never-download venvaws + . venvaws/bin/activate + pip install requests botocore + + cd src + python test/auth_aws/test_auth_aws.py + cd - + deactivate + rm -rf venvaws +} + +PYTHON=$(command -v python) authtest +PYTHON=$(command -v python3) authtest diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh new file mode 100755 index 000000000..5febd1302 --- /dev/null +++ b/.evergreen/run-mongodb-aws-test.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -o xtrace +set -o errexit # Exit the script with error if any of the commands fail + +############################################ +# Main Program # +############################################ + +echo "Running MONGODB-AWS authentication tests" +# ensure no secrets are printed in log files +set +x + +# load the script +shopt -s expand_aliases # needed for `urlencode` alias +[ -s "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" ] && source "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + +MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"} +MONGODB_URI="${MONGODB_URI}/aws?authMechanism=MONGODB-AWS" +if [[ -n ${SESSION_TOKEN} ]]; then + MONGODB_URI="${MONGODB_URI}&authMechanismProperties=AWS_SESSION_TOKEN:${SESSION_TOKEN}" +fi + +export MONGODB_URI="$MONGODB_URI" + +# show test output +set -x + +VIRTUALENV=$(command -v virtualenv) + +authtest () { + echo "Running MONGODB-AWS authentication tests with $PYTHON" + $PYTHON --version + + $VIRTUALENV -p $PYTHON --system-site-packages --never-download venvaws + . venvaws/bin/activate + pip install requests botocore + + python test/auth_aws/test_auth_aws.py + deactivate + rm -rf venvaws +} + +PYTHON=$(command -v python) authtest +PYTHON=$(command -v python3) authtest diff --git a/README.rst b/README.rst index 7e4d16b19..585c51f1c 100644 --- a/README.rst +++ b/README.rst @@ -99,6 +99,12 @@ dependency can be installed automatically along with PyMongo:: $ python -m pip install pymongo[gssapi] +MONGODB-AWS authentication requires `botocore +`_ and `requests +`_:: + + $ python -m pip install pymongo[aws] + Support for mongodb+srv:// URIs requires `dnspython `_:: @@ -116,7 +122,7 @@ PyMongo:: .. note:: Users of Python versions older than 2.7.9 will also receive the dependencies for OCSP when using the tls extra. -:ref:`OCSP` requires `PyOpenSSL +OCSP (Online Certificate Status Protocol) requires `PyOpenSSL `_, `requests `_ and `service_identity `_:: @@ -133,10 +139,15 @@ Wire protocol compression with zstandard requires `zstandard $ python -m pip install pymongo[zstd] +Client-Side Field Level Encryption requires `pymongocrypt +`_:: + + $ python -m pip install pymongo[encryption] + You can install all dependencies automatically with the following command:: - $ python -m pip install pymongo[gssapi,ocsp,snappy,srv,tls,zstd] + $ python -m pip install pymongo[gssapi,aws,ocsp,snappy,srv,tls,zstd,encryption] Other optional packages: diff --git a/doc/changelog.rst b/doc/changelog.rst index 9aedb72fc..9cbf8eebc 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -6,16 +6,17 @@ Changes in Version 3.11.0 Version 3.11 adds support for MongoDB 4.4. Highlights include: -- Added the ``allow_disk_use`` parameters to - :meth:`pymongo.collection.Collection.find`. - Support for :ref:`OCSP` (Online Certificate Status Protocol) - Support for `PyOpenSSL `_ as an alternative TLS implementation. PyOpenSSL is required for :ref:`OCSP` support. It will also be installed when using the "tls" extra if the version of Python in use is older than 2.7.9. +- Support for the :ref:`MONGODB-AWS` authentication mechanism. - Added the ``background`` parameter to :meth:`pymongo.database.Database.validate_collection`. For a description of this parameter see the MongoDB documentation for the `validate command`_. +- Added the ``allow_disk_use`` parameters to + :meth:`pymongo.collection.Collection.find`. .. _validate command: https://docs.mongodb.com/manual/reference/command/validate/ diff --git a/doc/examples/authentication.rst b/doc/examples/authentication.rst index dabf06957..bf5fff130 100644 --- a/doc/examples/authentication.rst +++ b/doc/examples/authentication.rst @@ -5,6 +5,8 @@ MongoDB supports several different authentication mechanisms. These examples cover all authentication methods currently supported by PyMongo, documenting Python module and MongoDB version dependencies. +.. _percent escaped: + Percent-Escaping Username and Password -------------------------------------- @@ -252,3 +254,120 @@ the SASL PLAIN mechanism:: ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> + +.. _MONGODB-AWS: + +MONGODB-AWS +----------- +.. versionadded:: 3.11 + +The MONGODB-AWS authentication mechanism is available in MongoDB 4.4+ and +requires extra pymongo dependencies. To use it, install pymongo with the +``aws`` extra:: + + $ python -m pip install 'pymongo[aws]' + +The MONGODB-AWS mechanism authenticates using AWS IAM credentials (an access +key ID and a secret access key), `temporary AWS IAM credentials`_ obtained +from an `AWS Security Token Service (STS)`_ `Assume Role`_ request, +AWS Lambda `environment variables`_, or temporary AWS IAM credentials assigned +to an `EC2 instance`_ or ECS task. The use of temporary credentials, in +addition to an access key ID and a secret access key, also requires a +security (or session) token. + +Credentials can be configured through the MongoDB URI, environment variables, +or the local EC2 or ECS endpoint. The order in which the client searches for +credentials is: + +#. Credentials passed through the URI +#. Environment variables +#. ECS endpoint if and only if ``AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`` is set. +#. EC2 endpoint + +MONGODB-AWS authenticates against the "$external" virtual database, so none of +the URIs in this section need to include the ``authSource`` URI option. + +AWS IAM credentials +~~~~~~~~~~~~~~~~~~~ + +Applications can authenticate using AWS IAM credentials by providing a valid +access key id and secret access key pair as the username and password, +respectively, in the MongoDB URI. A sample URI would be:: + + >>> from pymongo import MongoClient + >>> uri = "mongodb://:@localhost/?authMechanism=MONGODB-AWS" + >>> client = MongoClient(uri) + +.. note:: The access_key_id and secret_access_key passed into the URI MUST + be `percent escaped`_. + +AssumeRole +~~~~~~~~~~ + +Applications can authenticate using temporary credentials returned from an +assume role request. These temporary credentials consist of an access key +ID, a secret access key, and a security token passed into the URI. +A sample URI would be:: + + >>> from pymongo import MongoClient + >>> uri = "mongodb://:@example.com/?authMechanism=MONGODB-AWS&authMechanismProperties=AWS_SESSION_TOKEN:" + >>> client = MongoClient(uri) + +.. note:: The access_key_id, secret_access_key, and session_token passed into + the URI MUST be `percent escaped`_. + +AWS Lambda (Environment Variables) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the username and password are not provided and the MONGODB-AWS mechanism +is set, the client will fallback to using the `environment variables`_ +``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``, and ``AWS_SESSION_TOKEN`` +for the access key ID, secret access key, and session token, respectively:: + + $ export AWS_ACCESS_KEY_ID= + $ export AWS_SECRET_ACCESS_KEY= + $ export AWS_SESSION_TOKEN= + $ python + >>> from pymongo import MongoClient + >>> uri = "mongodb://example.com/?authMechanism=MONGODB-AWS" + >>> client = MongoClient(uri) + +.. note:: No username, password, or session token is passed into the URI. + PyMongo will use credentials set via the environment variables. + These environment variables MUST NOT be `percent escaped`_. + +ECS Container +~~~~~~~~~~~~~ + +Applications can authenticate from an ECS container via temporary +credentials assigned to the machine. A sample URI on an ECS container +would be:: + + >>> from pymongo import MongoClient + >>> uri = "mongodb://localhost/?authMechanism=MONGODB-AWS" + >>> client = MongoClient(uri) + +.. note:: No username, password, or session token is passed into the URI. + PyMongo will query the ECS container endpoint to obtain these + credentials. + +EC2 Instance +~~~~~~~~~~~~ + +Applications can authenticate from an EC2 instance via temporary +credentials assigned to the machine. A sample URI on an EC2 machine +would be:: + + >>> from pymongo import MongoClient + >>> uri = "mongodb://localhost/?authMechanism=MONGODB-AWS" + >>> client = MongoClient(uri) + +.. note:: No username, password, or session token is passed into the URI. + PyMongo will query the EC2 instance endpoint to obtain these + credentials. + +.. _temporary AWS IAM credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html +.. _AWS Security Token Service (STS): https://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html +.. _Assume Role: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html +.. _EC2 instance: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html +.. _environment variables: https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime diff --git a/doc/examples/encryption.rst b/doc/examples/encryption.rst index 2f8e2c7a9..ba07b3722 100644 --- a/doc/examples/encryption.rst +++ b/doc/examples/encryption.rst @@ -1,3 +1,5 @@ +.. _Client-Side Field Level Encryption: + Client-Side Field Level Encryption ================================== diff --git a/doc/installation.rst b/doc/installation.rst index 4875d72aa..aa7ec6a41 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -56,6 +56,12 @@ dependency can be installed automatically along with PyMongo:: $ python -m pip install pymongo[gssapi] +:ref:`MONGODB-AWS` authentication requires `botocore +`_ and `requests +`_:: + + $ python -m pip install pymongo[aws] + Support for mongodb+srv:// URIs requires `dnspython `_:: @@ -90,10 +96,15 @@ Wire protocol compression with zstandard requires `zstandard $ python -m pip install pymongo[zstd] +:ref:`Client-Side Field Level Encryption` requires `pymongocrypt +`_:: + + $ python -m pip install pymongo[encryption] + You can install all dependencies automatically with the following command:: - $ python -m pip install pymongo[gssapi,ocsp,snappy,srv,tls,zstd] + $ python -m pip install pymongo[gssapi,aws,ocsp,snappy,srv,tls,zstd,encryption] Other optional packages: diff --git a/pymongo/auth.py b/pymongo/auth.py index 455717a6a..b52c6b0af 100644 --- a/pymongo/auth.py +++ b/pymongo/auth.py @@ -43,6 +43,7 @@ from collections import namedtuple from bson.binary import Binary from bson.py3compat import string_type, _unicode, PY3 from bson.son import SON +from pymongo.auth_aws import _HAVE_MONGODB_AWS, _auth_aws, _AWSCredential from pymongo.errors import ConfigurationError, OperationFailure from pymongo.saslprep import saslprep @@ -51,6 +52,7 @@ MECHANISMS = frozenset( ['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', + 'MONGODB-AWS', 'PLAIN', 'SCRAM-SHA-1', 'SCRAM-SHA-256', @@ -100,10 +102,14 @@ GSSAPIProperties = namedtuple('GSSAPIProperties', """Mechanism properties for GSSAPI authentication.""" +_AWSProperties = namedtuple('AWSProperties', ['aws_session_token']) +"""Mechanism properties for MONGODB-AWS authentication.""" + + def _build_credentials_tuple(mech, source, user, passwd, extra, database): """Build and return a mechanism specific credentials tuple. """ - if mech != 'MONGODB-X509' and user is None: + if mech not in ('MONGODB-X509', 'MONGODB-AWS') and user is None: raise ConfigurationError("%s requires a username." % (mech,)) if mech == 'GSSAPI': if source is not None and source != '$external': @@ -126,8 +132,22 @@ def _build_credentials_tuple(mech, source, user, passwd, extra, database): raise ValueError( "authentication source must be " "$external or None for MONGODB-X509") - # user can be None. + # Source is always $external, user can be None. return MongoCredential(mech, '$external', user, None, None, None) + elif mech == 'MONGODB-AWS': + if user is not None and passwd is None: + raise ConfigurationError( + "username without a password is not supported by MONGODB-AWS") + if source is not None and source != '$external': + raise ConfigurationError( + "authentication source must be " + "$external or None for MONGODB-AWS") + + properties = extra.get('authmechanismproperties', {}) + aws_session_token = properties.get('AWS_SESSION_TOKEN') + props = _AWSProperties(aws_session_token=aws_session_token) + # user can be None for temporary link-local EC2 credentials. + return MongoCredential(mech, '$external', user, passwd, props, None) elif mech == 'PLAIN': source_database = source or database or '$external' return MongoCredential(mech, source_database, user, passwd, None, None) @@ -507,6 +527,20 @@ def _authenticate_x509(credentials, sock_info): sock_info.command('$external', query) +def _authenticate_aws(credentials, sock_info): + """Authenticate using MONGODB-AWS. + """ + if not _HAVE_MONGODB_AWS: + raise ConfigurationError( + "MONGODB-AWS authentication requires botocore and requests: " + "install these libraries with: " + "python -m pip install 'pymongo[aws]'") + + _auth_aws(_AWSCredential( + credentials.username, credentials.password, + credentials.mechanism_properties.aws_session_token), sock_info) + + def _authenticate_mongo_cr(credentials, sock_info): """Authenticate using MONGODB-CR. """ @@ -549,6 +583,7 @@ _AUTH_MAP = { 'GSSAPI': _authenticate_gssapi, 'MONGODB-CR': _authenticate_mongo_cr, 'MONGODB-X509': _authenticate_x509, + 'MONGODB-AWS': _authenticate_aws, 'PLAIN': _authenticate_plain, 'SCRAM-SHA-1': functools.partial( _authenticate_scram, mechanism='SCRAM-SHA-1'), diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py new file mode 100644 index 000000000..fdb6ec88f --- /dev/null +++ b/pymongo/auth_aws.py @@ -0,0 +1,201 @@ +# Copyright 2020-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MONGODB-AWS Authentication helpers.""" + +import os + +try: + + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + import requests + + _HAVE_MONGODB_AWS = True +except ImportError: + _HAVE_MONGODB_AWS = False + +import bson + + +from base64 import standard_b64encode +from collections import namedtuple + +from bson.binary import Binary +from bson.son import SON +from pymongo.errors import ConfigurationError, OperationFailure + + +_AWS_REL_URI = 'http://169.254.170.2/' +_AWS_EC2_URI = 'http://169.254.169.254/' +_AWS_EC2_PATH = 'latest/meta-data/iam/security-credentials/' +_AWS_HTTP_TIMEOUT = 10 + + +_AWSCredential = namedtuple('_AWSCredential', + ['username', 'password', 'token']) +"""MONGODB-AWS credentials.""" + + +def _aws_temp_credentials(): + """Construct temporary MONGODB-AWS credentials.""" + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + if access_key and secret_key: + return _AWSCredential( + access_key, secret_key, os.environ.get('AWS_SESSION_TOKEN')) + # If the environment variable + # AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is set then drivers MUST + # assume that it was set by an AWS ECS agent and use the URI + # http://169.254.170.2/$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI to + # obtain temporary credentials. + relative_uri = os.environ.get('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI') + if relative_uri is not None: + try: + res = requests.get(_AWS_REL_URI+relative_uri, + timeout=_AWS_HTTP_TIMEOUT) + res_json = res.json() + except (ValueError, requests.exceptions.RequestException): + raise OperationFailure( + 'temporary MONGODB-AWS credentials could not be obtained') + else: + # If the environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is + # not set drivers MUST assume we are on an EC2 instance and use the + # endpoint + # http://169.254.169.254/latest/meta-data/iam/security-credentials + # / + # whereas role-name can be obtained from querying the URI + # http://169.254.169.254/latest/meta-data/iam/security-credentials/. + try: + # Get token + headers = {'X-aws-ec2-metadata-token-ttl-seconds': "30"} + res = requests.post(_AWS_EC2_URI+'latest/api/token', + headers=headers, timeout=_AWS_HTTP_TIMEOUT) + token = res.content + # Get role name + headers = {'X-aws-ec2-metadata-token': token} + res = requests.get(_AWS_EC2_URI+_AWS_EC2_PATH, headers=headers, + timeout=_AWS_HTTP_TIMEOUT) + role = res.text + # Get temp creds + res = requests.get(_AWS_EC2_URI+_AWS_EC2_PATH+role, + headers=headers, timeout=_AWS_HTTP_TIMEOUT) + res_json = res.json() + except (ValueError, requests.exceptions.RequestException): + raise OperationFailure( + 'temporary MONGODB-AWS credentials could not be obtained') + + try: + temp_user = res_json['AccessKeyId'] + temp_password = res_json['SecretAccessKey'] + token = res_json['Token'] + except KeyError: + # If temporary credentials cannot be obtained then drivers MUST + # fail authentication and raise an error. + raise OperationFailure( + 'temporary MONGODB-AWS credentials could not be obtained') + + return _AWSCredential(temp_user, temp_password, token) + + +_AWS4_HMAC_SHA256 = 'AWS4-HMAC-SHA256' +_AWS_SERVICE = 'sts' + + +def _get_region(sts_host): + """""" + parts = sts_host.split('.') + if len(parts) == 1 or sts_host == 'sts.amazonaws.com': + return 'us-east-1' # Default + + if len(parts) > 2 or not all(parts): + raise OperationFailure("Server returned an invalid sts host") + + return parts[1] + + +def _aws_auth_header(credentials, server_nonce, sts_host): + """Signature Version 4 Signing Process to construct the authorization header + """ + region = _get_region(sts_host) + + request_parameters = 'Action=GetCallerIdentity&Version=2011-06-15' + encoded_nonce = standard_b64encode(server_nonce).decode('utf8') + request_headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': str(len(request_parameters)), + 'Host': sts_host, + 'X-MongoDB-Server-Nonce': encoded_nonce, + 'X-MongoDB-GS2-CB-Flag': 'n', + } + request = AWSRequest(method="POST", url="/", data=request_parameters, + headers=request_headers) + boto_creds = Credentials(credentials.username, credentials.password, + token=credentials.token) + auth = SigV4Auth(boto_creds, "sts", region) + auth.add_auth(request) + final = { + 'a': request.headers['Authorization'], + 'd': request.headers['X-Amz-Date'] + } + if credentials.token: + final['t'] = credentials.token + return final + + +def _auth_aws(credentials, sock_info): + """Authenticate using MONGODB-AWS. + """ + if not _HAVE_MONGODB_AWS: + raise ConfigurationError( + "MONGODB-AWS authentication requires botocore and requests: " + "install these libraries with: " + "python -m pip install 'pymongo[aws]'") + + if sock_info.max_wire_version < 9: + raise ConfigurationError( + "MONGODB-AWS authentication requires MongoDB version 4.4 or later") + + # If a username and password are not provided, drivers MUST query + # a link-local AWS address for temporary credentials. + if credentials.username is None: + credentials = _aws_temp_credentials() + + # Client first. + client_nonce = os.urandom(32) + payload = {'r': Binary(client_nonce), 'p': 110} + client_first = SON([('saslStart', 1), + ('mechanism', 'MONGODB-AWS'), + ('payload', Binary(bson.encode(payload)))]) + server_first = sock_info.command('$external', client_first) + + server_payload = bson.decode(server_first['payload']) + server_nonce = server_payload['s'] + if len(server_nonce) != 64 or not server_nonce.startswith(client_nonce): + raise OperationFailure("Server returned an invalid nonce.") + sts_host = server_payload['h'] + if len(sts_host) < 1 or len(sts_host) > 255 or '..' in sts_host: + # Drivers must also validate that the host is greater than 0 and less + # than or equal to 255 bytes per RFC 1035. + raise OperationFailure("Server returned an invalid sts host.") + + payload = _aws_auth_header(credentials, server_nonce, sts_host) + client_second = SON([('saslContinue', 1), + ('conversationId', server_first['conversationId']), + ('payload', Binary(bson.encode(payload)))]) + res = sock_info.command('$external', client_second) + if not res['done']: + raise OperationFailure('MONGODB-AWS conversation failed to complete.') diff --git a/pymongo/common.py b/pymongo/common.py index f208e83d0..12777b7ac 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -22,7 +22,7 @@ from bson import SON from bson.binary import (STANDARD, PYTHON_LEGACY, JAVA_LEGACY, CSHARP_LEGACY) from bson.codec_options import CodecOptions, TypeRegistry -from bson.py3compat import abc, integer_types, iteritems, string_type +from bson.py3compat import abc, integer_types, iteritems, string_type, PY3 from bson.raw_bson import RawBSONDocument from pymongo.auth import MECHANISMS from pymongo.compression_support import (validate_compressors, @@ -43,6 +43,10 @@ try: except ImportError: ORDERED_TYPES = (SON,) +if PY3: + from urllib.parse import unquote_plus +else: + from urllib import unquote_plus # Defaults until we connect to a server and get updated limits. MAX_BSON_SIZE = 16 * (1024 ** 2) @@ -391,8 +395,11 @@ def validate_read_preference_tags(name, value): tag_sets.append({}) continue try: - tag_sets.append(dict([tag.split(":") - for tag in tag_set.split(",")])) + tags = {} + for tag in tag_set.split(","): + key, val = tag.split(":") + tags[unquote_plus(key)] = unquote_plus(val) + tag_sets.append(tags) except Exception: raise ValueError("%r not a valid " "value for %s" % (tag_set, name)) @@ -401,7 +408,8 @@ def validate_read_preference_tags(name, value): _MECHANISM_PROPS = frozenset(['SERVICE_NAME', 'CANONICALIZE_HOST_NAME', - 'SERVICE_REALM']) + 'SERVICE_REALM', + 'AWS_SESSION_TOKEN']) def validate_auth_mechanism_properties(option, value): @@ -412,6 +420,10 @@ def validate_auth_mechanism_properties(option, value): try: key, val = opt.split(':') except ValueError: + # Try not to leak the token. + if 'AWS_SESSION_TOKEN' in opt: + opt = ('AWS_SESSION_TOKEN:, did you forget ' + 'to percent-escape the token with quote_plus?') raise ValueError("auth mechanism properties must be " "key:value pairs like SERVICE_NAME:" "mongodb, not %s." % (opt,)) @@ -422,7 +434,7 @@ def validate_auth_mechanism_properties(option, value): if key == 'CANONICALIZE_HOST_NAME': props[key] = validate_boolean_or_string(key, val) else: - props[key] = val + props[key] = unquote_plus(val) return props diff --git a/pymongo/database.py b/pymongo/database.py index d363d3b90..144a1f6b4 100644 --- a/pymongo/database.py +++ b/pymongo/database.py @@ -1437,7 +1437,9 @@ class Database(common.BaseObject): - `authMechanismProperties` (optional): Used to specify authentication mechanism specific options. To specify the service name for GSSAPI authentication pass - authMechanismProperties='SERVICE_NAME:' + ``authMechanismProperties='SERVICE_NAME:'``. + To specify the session token for MONGODB-AWS authentication pass + ``authMechanismProperties='AWS_SESSION_TOKEN:'``. .. versionchanged:: 3.7 Added support for SCRAM-SHA-256 with MongoDB 4.0 and later. diff --git a/pymongo/mongo_client.py b/pymongo/mongo_client.py index 8294af14e..9b33fa2cd 100644 --- a/pymongo/mongo_client.py +++ b/pymongo/mongo_client.py @@ -411,7 +411,9 @@ class MongoClient(common.BaseObject): - `authMechanismProperties`: Used to specify authentication mechanism specific options. To specify the service name for GSSAPI authentication pass authMechanismProperties='SERVICE_NAME:' + name>'. + To specify the session token for MONGODB-AWS authentication pass + ``authMechanismProperties='AWS_SESSION_TOKEN:'``. .. seealso:: :doc:`/examples/authentication` @@ -520,7 +522,7 @@ class MongoClient(common.BaseObject): .. versionchanged:: 3.5 Add ``username`` and ``password`` options. Document the - ``authSource``, ``authMechanism``, and ``authMechanismProperties `` + ``authSource``, ``authMechanism``, and ``authMechanismProperties`` options. Deprecated the ``socketKeepAlive`` keyword argument and URI option. ``socketKeepAlive`` now defaults to ``True``. diff --git a/pymongo/uri_parser.py b/pymongo/uri_parser.py index 131f54d8f..a6b0a46cd 100644 --- a/pymongo/uri_parser.py +++ b/pymongo/uri_parser.py @@ -146,7 +146,11 @@ def _parse_options(opts, delim): else: if key in options: warnings.warn("Duplicate URI option '%s'." % (key,)) - options[key] = unquote_plus(value) + if key.lower() == 'authmechanismproperties': + val = value + else: + val = unquote_plus(value) + options[key] = val return options @@ -417,24 +421,19 @@ def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False, "the host list and any options.") if path_part: - if path_part[0] == '?': - opts = unquote_plus(path_part[1:]) - else: - dbase, _, opts = map(unquote_plus, path_part.partition('?')) + dbase, _, opts = path_part.partition('?') + if dbase: + dbase = unquote_plus(dbase) if '.' in dbase: dbase, collection = dbase.split('.', 1) - if _BAD_DB_CHARS.search(dbase): raise InvalidURI('Bad database name "%s"' % dbase) + else: + dbase = None if opts: options.update(split_options(opts, validate, warn, normalize)) - if dbase is not None: - dbase = unquote_plus(dbase) - if collection is not None: - collection = unquote_plus(collection) - if '@' in host_part: userinfo, _, hosts = host_part.rpartition('@') user, passwd = parse_userinfo(userinfo) diff --git a/setup.py b/setup.py index 4c8491505..ba051f73b 100755 --- a/setup.py +++ b/setup.py @@ -329,6 +329,7 @@ extras_require = { 'snappy': ['python-snappy'], 'tls': [], 'zstd': ['zstandard'], + 'aws': ['requests<3.0.0', 'botocore'], } # https://jira.mongodb.org/browse/PYTHON-2117 diff --git a/test/auth/connection-string.json b/test/auth/connection-string.json index 2005a090a..5452912e8 100644 --- a/test/auth/connection-string.json +++ b/test/auth/connection-string.json @@ -376,6 +376,50 @@ "description": "should throw an exception if no username/password provided (userinfo implies default mechanism)", "uri": "mongodb://:@localhost.com/", "valid": false + }, + { + "description": "should recognise the mechanism (MONGODB-AWS)", + "uri": "mongodb://localhost/?authMechanism=MONGODB-AWS", + "valid": true, + "credential": { + "username": null, + "password": null, + "source": "$external", + "mechanism": "MONGODB-AWS", + "mechanism_properties": null + } + }, + { + "description": "should throw an exception if username and no password (MONGODB-AWS)", + "uri": "mongodb://user@localhost/?authMechanism=MONGODB-AWS", + "valid": false, + "credential": null + }, + { + "description": "should use username and password if specified (MONGODB-AWS)", + "uri": "mongodb://user%21%40%23%24%25%5E%26%2A%28%29_%2B:pass%21%40%23%24%25%5E%26%2A%28%29_%2B@localhost/?authMechanism=MONGODB-AWS", + "valid": true, + "credential": { + "username": "user!@#$%^&*()_+", + "password": "pass!@#$%^&*()_+", + "source": "$external", + "mechanism": "MONGODB-AWS", + "mechanism_properties": null + } + }, + { + "description": "should use username, password and session token if specified (MONGODB-AWS)", + "uri": "mongodb://user:password@localhost/?authMechanism=MONGODB-AWS&authMechanismProperties=AWS_SESSION_TOKEN:token%21%40%23%24%25%5E%26%2A%28%29_%2B", + "valid": true, + "credential": { + "username": "user", + "password": "password", + "source": "$external", + "mechanism": "MONGODB-AWS", + "mechanism_properties": { + "AWS_SESSION_TOKEN": "token!@#$%^&*()_+" + } + } } ] -} \ No newline at end of file +} diff --git a/test/auth_aws/test_auth_aws.py b/test/auth_aws/test_auth_aws.py new file mode 100644 index 000000000..d17ebb5aa --- /dev/null +++ b/test/auth_aws/test_auth_aws.py @@ -0,0 +1,60 @@ +# Copyright 2020-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test MONGODB-AWS Authentication.""" + +import os +import sys +import unittest + +sys.path[0:0] = [""] + +from pymongo import MongoClient +from pymongo.errors import OperationFailure +from pymongo.uri_parser import parse_uri + + +if not hasattr(unittest.TestCase, 'assertRaisesRegex'): + unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + + +class TestAuthAWS(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.uri = os.environ['MONGODB_URI'] + + def test_should_fail_without_credentials(self): + if '@' not in self.uri: + self.skipTest('MONGODB_URI already has no credentials') + + hosts = ['%s:%s' % addr for addr in parse_uri(self.uri)['nodelist']] + self.assertTrue(hosts) + with MongoClient(hosts) as client: + with self.assertRaises(OperationFailure): + client.aws.test.find_one() + + def test_should_fail_incorrect_credentials(self): + with MongoClient(self.uri, username='fake', password='fake', + authMechanism='MONGODB-AWS') as client: + with self.assertRaises(OperationFailure): + client.get_database().test.find_one() + + def test_connect_uri(self): + with MongoClient(self.uri) as client: + client.get_database().test.find_one() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_auth_spec.py b/test/test_auth_spec.py index 4e76a97dd..1b6b919e8 100644 --- a/test/test_auth_spec.py +++ b/test/test_auth_spec.py @@ -72,10 +72,18 @@ def create_test(test_case): self.assertEqual( actual.service_realm, expected['SERVICE_REALM']) + elif 'AWS_SESSION_TOKEN' in expected: + self.assertEqual( + actual.aws_session_token, + expected['AWS_SESSION_TOKEN']) else: self.fail('Unhandled property: %s' % (key,)) else: - self.assertIsNone(credentials.mechanism_properties) + if credential['mechanism'] == 'MONGODB-AWS': + self.assertIsNone( + credentials.mechanism_properties.aws_session_token) + else: + self.assertIsNone(credentials.mechanism_properties) return run_test diff --git a/test/test_uri_parser.py b/test/test_uri_parser.py index 4fb60c843..a921c6656 100644 --- a/test/test_uri_parser.py +++ b/test/test_uri_parser.py @@ -498,6 +498,43 @@ class TestURI(unittest.TestCase): self.assertEqual(len(ctx), 1) self.assertTrue(issubclass(ctx[0].category, DeprecationWarning)) + def test_unquote_after_parsing(self): + quoted_val = "val%21%40%23%24%25%5E%26%2A%28%29_%2B%2C%3A+etc" + unquoted_val = "val!@#$%^&*()_+,: etc" + uri = ("mongodb://user:password@localhost/?authMechanism=MONGODB-AWS" + "&authMechanismProperties=AWS_SESSION_TOKEN:"+quoted_val) + res = parse_uri(uri) + options = { + 'authmechanism': 'MONGODB-AWS', + 'authmechanismproperties': { + 'AWS_SESSION_TOKEN': unquoted_val}} + self.assertEqual(options, res['options']) + + uri = (("mongodb://localhost/foo?readpreference=secondary&" + "readpreferencetags=dc:west,"+quoted_val+":"+quoted_val+"&" + "readpreferencetags=dc:east,use:"+quoted_val)) + res = parse_uri(uri) + options = { + 'readpreference': ReadPreference.SECONDARY.mongos_mode, + 'readpreferencetags': [ + {'dc': 'west', unquoted_val: unquoted_val}, + {'dc': 'east', 'use': unquoted_val} + ] + } + self.assertEqual(options, res['options']) + + def test_redact_AWS_SESSION_TOKEN(self): + unquoted_colon = "token:" + uri = ("mongodb://user:password@localhost/?authMechanism=MONGODB-AWS" + "&authMechanismProperties=AWS_SESSION_TOKEN:"+unquoted_colon) + with self.assertRaisesRegex( + ValueError, + 'auth mechanism properties must be key:value pairs like ' + 'SERVICE_NAME:mongodb, not AWS_SESSION_TOKEN:' + ', did you forget to percent-escape the token with ' + 'quote_plus?'): + parse_uri(uri) + if __name__ == "__main__": unittest.main()