mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-3004 Support kmip FLE KMS provider (#786)

Browse Source 
Resync CSFLE spec tests.
This commit is contained in:
Shane Harvey 2021-11-15 16:23:59 -08:00 committed by GitHub
parent 754e52890f
commit a7fb3281ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 5328 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
.evergreen/config.yml
View File

@ -359,6 +359,49 @@ functions:
PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/run-doctests.sh
"run tests":
# If testing FLE, start the KMS mock servers, first create the virtualenv.
- command: shell.exec
params:
script: |
if [ -n "${test_encryption}" ]; then
${PREPARE_SHELL}
cd ${DRIVERS_TOOLS}/.evergreen/csfle
. ./activate_venv.sh
fi
# Run in the background so the mock servers don't block the EVG task.
- command: shell.exec
params:
background: true
script: |
if [ -n "${test_encryption}" ]; then
${PREPARE_SHELL}
cd ${DRIVERS_TOOLS}/.evergreen/csfle
. ./activate_venv.sh
# The -u options forces the stdout and stderr streams to be unbuffered.
# TMPDIR is required to avoid "AF_UNIX path too long" errors.
TMPDIR="$(dirname $DRIVERS_TOOLS)" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 5698 &
python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
fi
# Wait up to 10 seconds for the KMIP server to start.
- command: shell.exec
params:
script: |
if [ -n "${test_encryption}" ]; then
${PREPARE_SHELL}
cd ${DRIVERS_TOOLS}/.evergreen/csfle
. ./activate_venv.sh
for i in $(seq 1 1 10); do
sleep 1
if python -u kms_kmip_client.py; then
echo 'KMS KMIP server started!'
exit 0
fi
done
echo 'Failed to start KMIP server!'
exit 1
fi
- command: shell.exec
type: test
params:

7
.evergreen/run-tests.sh
View File

@ -146,13 +146,6 @@ if [ -n "$TEST_ENCRYPTION" ]; then
# Get access to the AWS temporary credentials:
# CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
. $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
# Start the mock KMS servers.
pushd ${DRIVERS_TOOLS}/.evergreen/csfle
python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
trap 'kill $(jobs -p)' EXIT HUP
popd
fi
if [ -z "$DATA_LAKE" ]; then

23
pymongo/encryption.py
View File

@ -109,7 +109,7 @@ class _EncryptionIO(MongoCryptCallback):
message = kms_context.message
provider = kms_context.kms_provider
ctx = self.opts._kms_ssl_contexts.get(provider)
if not ctx:
if ctx is None:
# Enable strict certificate verification, OCSP, match hostname, and
# SNI using the system default CA certificates.
ctx = get_ssl_context(
@ -378,9 +378,8 @@ class ClientEncryption(object):
See :ref:`explicit-client-side-encryption` for an example.
:Parameters:
- `kms_providers`: Map of KMS provider options. Two KMS providers
are supported: "aws" and "local". The kmsProviders map values
differ by provider:
- `kms_providers`: Map of KMS provider options. The `kms_providers`
map values differ by provider:
- `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
These are the AWS access key ID and AWS secret access key used
@ -396,6 +395,8 @@ class ClientEncryption(object):
Additionally, "endpoint" may also be specified as a string
(defaults to 'oauth2.googleapis.com'). These are the
credentials used to generate Google Cloud KMS messages.
- `kmip`: Map with "endpoint" as a host with required port.
For example: ``{"endpoint": "example.com:443"}``.
- `local`: Map with "key" as `bytes` (96 bytes in length) or
a base64 encoded string which decodes
to 96 bytes. "key" is the master key used to encrypt/decrypt
@ -424,7 +425,7 @@ class ClientEncryption(object):
kms_tls_options={'kmip': {'tlsCAFile': certifi.where()}}
.. versionchanged:: 4.0
Added the `kms_tls_options` parameter.
Added the `kms_tls_options` parameter and the "kmip" KMS provider.
.. versionadded:: 3.9
"""
@ -458,7 +459,7 @@ class ClientEncryption(object):
:Parameters:
- `kms_provider`: The KMS provider to use. Supported values are
"aws" and "local".
"aws", "azure", "gcp", "kmip", and "local".
- `master_key`: Identifies a KMS-specific key used to encrypt the
new data key. If the kmsProvider is "local" the `master_key` is
not applicable and may be omitted.
@ -493,6 +494,16 @@ class ClientEncryption(object):
- `endpoint` (string): Optional. Host with optional port.
Defaults to "cloudkms.googleapis.com".
If the `kms_provider` is "kmip" it is optional and has the
following fields::
- `keyId` (string): Optional. `keyId` is the KMIP Unique
Identifier to a 96 byte KMIP Secret Data managed object. If
keyId is omitted, the driver creates a random 96 byte KMIP
Secret Data managed object.
- `endpoint` (string): Optional. Host with optional
port, e.g. "example.vault.azure.net:".
- `key_alt_names` (optional): An optional list of string alternate
names used to reference a key. If a key is created with alternate
names, then encryption may refer to the key by the unique alternate

9
pymongo/encryption_options.py
View File

@ -55,9 +55,8 @@ class AutoEncryptionOpts(object):
See :ref:`automatic-client-side-encryption` for an example.
:Parameters:
- `kms_providers`: Map of KMS provider options. Two KMS providers
are supported: "aws" and "local". The kmsProviders map values
differ by provider:
- `kms_providers`: Map of KMS provider options. The `kms_providers`
map values differ by provider:
- `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
These are the AWS access key ID and AWS secret access key used
@ -73,6 +72,8 @@ class AutoEncryptionOpts(object):
Additionally, "endpoint" may also be specified as a string
(defaults to 'oauth2.googleapis.com'). These are the
credentials used to generate Google Cloud KMS messages.
- `kmip`: Map with "endpoint" as a host with required port.
For example: ``{"endpoint": "example.com:443"}``.
- `local`: Map with "key" as `bytes` (96 bytes in length) or
a base64 encoded string which decodes
to 96 bytes. "key" is the master key used to encrypt/decrypt
@ -129,7 +130,7 @@ class AutoEncryptionOpts(object):
kms_tls_options={'kmip': {'tlsCAFile': certifi.where()}}
.. versionchanged:: 4.0
Added the `kms_tls_options` parameter.
Added the `kms_tls_options` parameter and the "kmip" KMS provider.
.. versionadded:: 3.9
"""

1830
test/client-side-encryption/corpus/corpus-encrypted.json
View File

File diff suppressed because it is too large Load Diff

32
test/client-side-encryption/corpus/corpus-key-kmip.json Normal file
View File

@ -0,0 +1,32 @@
{
"_id": {
"$binary": {
"base64": "KMIPAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "eUYDyB0HuWb+lQgUwO+6qJQyTTDTY2gp9FbemL7ZFo0pvr0x6rm6Ff9OVUTGH6HyMKipaeHdiIJU1dzsLwvqKvi7Beh+U4iaIWX/K0oEg1GOsJc0+Z/in8gNHbGUYLmycHViM3LES3kdt7FdFSUl5rEBHrM71yoNEXImz17QJWMGOuT4x6yoi2pvnaRJwfrI4DjpmnnTrDMac92jgZehbg==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1634220190041"
}
},
"updateDate": {
"$date": {
"$numberLong": "1634220190041"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "kmip",
"keyId": "1"
},
"keyAltNames": ["kmip"]
}

1266
test/client-side-encryption/corpus/corpus-schema.json
View File

File diff suppressed because it is too large Load Diff

1662
test/client-side-encryption/corpus/corpus.json
View File

File diff suppressed because it is too large Load Diff

14
test/client-side-encryption/spec/azureKMS.json
View File

@ -64,6 +64,20 @@
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_kmip": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "dBHpr8aITfeBQ15grpbLpQ==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"

14
test/client-side-encryption/spec/gcpKMS.json
View File

@ -64,6 +64,20 @@
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_kmip": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "dBHpr8aITfeBQ15grpbLpQ==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"

223
test/client-side-encryption/spec/kmipKMS.json Normal file
View File

@ -0,0 +1,223 @@
{
"runOn": [
{
"minServerVersion": "4.1.10"
}
],
"database_name": "default",
"collection_name": "default",
"data": [],
"json_schema": {
"properties": {
"encrypted_string_aws": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_azure": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_gcp": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_local": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_kmip": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "dBHpr8aITfeBQ15grpbLpQ==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"
},
"key_vault_data": [
{
"_id": {
"$binary": {
"base64": "dBHpr8aITfeBQ15grpbLpQ==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "eUYDyB0HuWb+lQgUwO+6qJQyTTDTY2gp9FbemL7ZFo0pvr0x6rm6Ff9OVUTGH6HyMKipaeHdiIJU1dzsLwvqKvi7Beh+U4iaIWX/K0oEg1GOsJc0+Z/in8gNHbGUYLmycHViM3LES3kdt7FdFSUl5rEBHrM71yoNEXImz17QJWMGOuT4x6yoi2pvnaRJwfrI4DjpmnnTrDMac92jgZehbg==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1634220190041"
}
},
"updateDate": {
"$date": {
"$numberLong": "1634220190041"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "kmip",
"keyId": "1"
},
"keyAltNames": [
"altname",
"kmip_altname"
]
}
],
"tests": [
{
"description": "Insert a document with auto encryption using KMIP KMS provider",
"clientOptions": {
"autoEncryptOpts": {
"kmsProviders": {
"kmip": {}
}
}
},
"operations": [
{
"name": "insertOne",
"arguments": {
"document": {
"_id": 1,
"encrypted_string_kmip": "string0"
}
}
}
],
"expectations": [
{
"command_started_event": {
"command": {
"listCollections": 1,
"filter": {
"name": "default"
}
},
"command_name": "listCollections"
}
},
{
"command_started_event": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "dBHpr8aITfeBQ15grpbLpQ==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault"
},
"command_name": "find"
}
},
{
"command_started_event": {
"command": {
"insert": "default",
"documents": [
{
"_id": 1,
"encrypted_string_kmip": {
"$binary": {
"base64": "AXQR6a/GiE33gUNeYK6Wy6UCKCwtKFIsL8eKObDVxvqGupJNUk7kXswHhB7G5j/C1D+6no+Asra0KgSU43bTL3ooIBLVyIzbV5CDJYqzAsa4WQ==",
"subType": "06"
}
}
}
],
"ordered": true
},
"command_name": "insert"
}
}
],
"outcome": {
"collection": {
"data": [
{
"_id": 1,
"encrypted_string_kmip": {
"$binary": {
"base64": "AXQR6a/GiE33gUNeYK6Wy6UCKCwtKFIsL8eKObDVxvqGupJNUk7kXswHhB7G5j/C1D+6no+Asra0KgSU43bTL3ooIBLVyIzbV5CDJYqzAsa4WQ==",
"subType": "06"
}
}
}
]
}
}
}
]
}

244
test/test_encryption.py
View File

@ -17,11 +17,12 @@
import base64
import copy
import os
import re
import ssl
import traceback
import socket
import sys
import textwrap
import traceback
import uuid
sys.path[0:0] = [""]
@ -516,6 +517,10 @@ GCP_CREDS = {
'email': os.environ.get('FLE_GCP_EMAIL', ''),
'privateKey': os.environ.get('FLE_GCP_PRIVATEKEY', '')}
KMIP = {'endpoint': os.environ.get('FLE_KMIP_ENDPOINT', 'localhost:5698')}
KMS_TLS_OPTS = {'kmip': {'tlsCAFile': CA_PEM,
'tlsCertificateKeyFile': CLIENT_PEM}}
class TestSpec(SpecRunner):
@ -550,6 +555,9 @@ class TestSpec(SpecRunner):
kms_providers['gcp'] = GCP_CREDS
if not any(AZURE_CREDS.values()):
self.skipTest('GCP environment credentials are not set')
if 'kmip' in kms_providers:
kms_providers['kmip'] = KMIP
opts['kms_tls_options'] = KMS_TLS_OPTS
if 'key_vault_namespace' not in opts:
opts['key_vault_namespace'] = 'keyvault.datakeys'
opts = dict(opts)
@ -631,6 +639,13 @@ LOCAL_MASTER_KEY = base64.b64decode(
b'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ'
b'5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk')
ALL_KMS_PROVIDERS = {
'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS,
'kmip': KMIP,
'local': {'key': LOCAL_MASTER_KEY}}
LOCAL_KEY_ID = Binary(
base64.b64decode(b'LOCALAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
AWS_KEY_ID = Binary(
@ -639,6 +654,8 @@ AZURE_KEY_ID = Binary(
base64.b64decode(b'AZUREAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
GCP_KEY_ID = Binary(
base64.b64decode(b'GCPAAAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
KMIP_KEY_ID = Binary(
base64.b64decode(b'KMIPAAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
def create_with_schema(coll, json_schema):
@ -661,10 +678,7 @@ def create_key_vault(vault, *data_keys):
class TestDataKeyDoubleEncryption(EncryptionIntegrationTest):
KMS_PROVIDERS = {'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS,
'local': {'key': LOCAL_MASTER_KEY}}
KMS_PROVIDERS = ALL_KMS_PROVIDERS
MASTER_KEYS = {
'aws': {
@ -679,6 +693,7 @@ class TestDataKeyDoubleEncryption(EncryptionIntegrationTest):
'location': 'global',
'keyRing': 'key-ring-csfle',
'keyName': 'key-name-csfle'},
'kmip': {},
'local': None
}
@ -710,11 +725,13 @@ class TestDataKeyDoubleEncryption(EncryptionIntegrationTest):
}
}
opts = AutoEncryptionOpts(
cls.KMS_PROVIDERS, 'keyvault.datakeys', schema_map=schemas)
cls.KMS_PROVIDERS, 'keyvault.datakeys', schema_map=schemas,
kms_tls_options=KMS_TLS_OPTS)
cls.client_encrypted = rs_or_single_client(
auto_encryption_opts=opts, uuidRepresentation='standard')
cls.client_encryption = ClientEncryption(
cls.KMS_PROVIDERS, 'keyvault.datakeys', cls.client, OPTS)
cls.KMS_PROVIDERS, 'keyvault.datakeys', cls.client, OPTS,
kms_tls_options=KMS_TLS_OPTS)
@classmethod
def tearDownClass(cls):
@ -784,6 +801,9 @@ class TestDataKeyDoubleEncryption(EncryptionIntegrationTest):
def test_data_key_gcp(self):
self.run_test('gcp')
def test_data_key_kmip(self):
self.run_test('kmip')
class TestExternalKeyVault(EncryptionIntegrationTest):
@ -882,10 +902,7 @@ class TestCorpus(EncryptionIntegrationTest):
@staticmethod
def kms_providers():
return {'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS,
'local': {'key': LOCAL_MASTER_KEY}}
return ALL_KMS_PROVIDERS
@staticmethod
def fix_up_schema(json_schema):
@ -923,7 +940,8 @@ class TestCorpus(EncryptionIntegrationTest):
json_data('corpus', 'corpus-key-local.json'),
json_data('corpus', 'corpus-key-aws.json'),
json_data('corpus', 'corpus-key-azure.json'),
json_data('corpus', 'corpus-key-gcp.json'))
json_data('corpus', 'corpus-key-gcp.json'),
json_data('corpus', 'corpus-key-kmip.json'))
self.addCleanup(vault.drop)
client_encrypted = rs_or_single_client(
@ -932,7 +950,7 @@ class TestCorpus(EncryptionIntegrationTest):
client_encryption = ClientEncryption(
self.kms_providers(), 'keyvault.datakeys', client_context.client,
OPTS)
OPTS, kms_tls_options=KMS_TLS_OPTS)
self.addCleanup(client_encryption.close)
corpus = self.fix_up_curpus(json_data('corpus', 'corpus.json'))
@ -940,7 +958,7 @@ class TestCorpus(EncryptionIntegrationTest):
for key, value in corpus.items():
corpus_copied[key] = copy.deepcopy(value)
if key in ('_id', 'altname_aws', 'altname_azure', 'altname_gcp',
'altname_local'):
'altname_local', 'altname_kmip'):
continue
if value['method'] == 'auto':
continue
@ -948,7 +966,7 @@ class TestCorpus(EncryptionIntegrationTest):
identifier = value['identifier']
self.assertIn(identifier, ('id', 'altname'))
kms = value['kms']
self.assertIn(kms, ('local', 'aws', 'azure', 'gcp'))
self.assertIn(kms, ('local', 'aws', 'azure', 'gcp', 'kmip'))
if identifier == 'id':
if kms == 'local':
kwargs = dict(key_id=LOCAL_KEY_ID)
@ -956,8 +974,10 @@ class TestCorpus(EncryptionIntegrationTest):
kwargs = dict(key_id=AWS_KEY_ID)
elif kms == 'azure':
kwargs = dict(key_id=AZURE_KEY_ID)
else:
elif kms == 'gcp':
kwargs = dict(key_id=GCP_KEY_ID)
else:
kwargs = dict(key_id=KMIP_KEY_ID)
else:
kwargs = dict(key_alt_name=kms)
@ -990,7 +1010,7 @@ class TestCorpus(EncryptionIntegrationTest):
corpus_encrypted_actual = coll.find_one()
for key, value in corpus_encrypted_actual.items():
if key in ('_id', 'altname_aws', 'altname_azure',
'altname_gcp', 'altname_local'):
'altname_gcp', 'altname_local', 'altname_kmip'):
continue
if value['algo'] == 'det':
@ -1011,7 +1031,8 @@ class TestCorpus(EncryptionIntegrationTest):
self.assertEqual(value['value'], corpus[key]['value'], key)
def test_corpus(self):
opts = AutoEncryptionOpts(self.kms_providers(), 'keyvault.datakeys')
opts = AutoEncryptionOpts(self.kms_providers(), 'keyvault.datakeys',
kms_tls_options=KMS_TLS_OPTS)
self._test_corpus(opts)
def test_corpus_local_schema(self):
@ -1019,7 +1040,8 @@ class TestCorpus(EncryptionIntegrationTest):
schemas = {'db.coll': self.fix_up_schema(
json_data('corpus', 'corpus-schema.json'))}
opts = AutoEncryptionOpts(
self.kms_providers(), 'keyvault.datakeys', schema_map=schemas)
self.kms_providers(), 'keyvault.datakeys', schema_map=schemas,
kms_tls_options=KMS_TLS_OPTS)
self._test_corpus(opts)
@ -1142,21 +1164,26 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
def setUp(self):
kms_providers = {'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS}
'gcp': GCP_CREDS,
'kmip': KMIP}
self.client_encryption = ClientEncryption(
kms_providers=kms_providers,
key_vault_namespace='keyvault.datakeys',
key_vault_client=client_context.client,
codec_options=OPTS)
codec_options=OPTS,
kms_tls_options=KMS_TLS_OPTS)
kms_providers_invalid = copy.deepcopy(kms_providers)
kms_providers_invalid['azure']['identityPlatformEndpoint'] = 'example.com:443'
kms_providers_invalid['gcp']['endpoint'] = 'example.com:443'
kms_providers_invalid['kmip']['endpoint'] = 'doesnotexist.local:5698'
self.client_encryption_invalid = ClientEncryption(
kms_providers=kms_providers_invalid,
key_vault_namespace='keyvault.datakeys',
key_vault_client=client_context.client,
codec_options=OPTS)
codec_options=OPTS,
kms_tls_options=KMS_TLS_OPTS)
self._kmip_host_error = ''
def tearDown(self):
self.client_encryption.close()
@ -1289,6 +1316,41 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
self.client_encryption.create_data_key(
'gcp', master_key=master_key)
def kmip_host_error(self):
if self._kmip_host_error:
return self._kmip_host_error
# The full error should be something like:
# "[Errno 8] nodename nor servname provided, or not known"
try:
socket.getaddrinfo('doesnotexist.local', 5698, socket.AF_INET,
socket.SOCK_STREAM)
except Exception as exc:
self._kmip_host_error = re.escape(str(exc))
return self._kmip_host_error
def test_10_kmip_invalid_endpoint(self):
key = {'keyId': '1'}
self.run_test_expected_success('kmip', key)
with self.assertRaisesRegex(EncryptionError, self.kmip_host_error()):
self.client_encryption_invalid.create_data_key('kmip', key)
def test_11_kmip_master_key_endpoint(self):
key = {'keyId': '1', 'endpoint': KMIP['endpoint']}
self.run_test_expected_success('kmip', key)
# Override invalid endpoint:
data_key_id = self.client_encryption_invalid.create_data_key(
'kmip', master_key=key)
encrypted = self.client_encryption_invalid.encrypt(
'test', Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_id=data_key_id)
self.assertEqual(
'test', self.client_encryption_invalid.decrypt(encrypted))
def test_12_kmip_master_key_invalid_endpoint(self):
key = {'keyId': '1', 'endpoint': 'doesnotexist.local:5698'}
with self.assertRaisesRegex(EncryptionError, self.kmip_host_error()):
self.client_encryption.create_data_key('kmip', key)
class AzureGCPEncryptionTestMixin(object):
DEK = None
@ -1709,5 +1771,143 @@ class TestKmsTLSProse(EncryptionIntegrationTest):
self.client_encrypted.create_data_key('aws', master_key=key)
# https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.rst#kms-tls-options-tests
class TestKmsTLSOptions(EncryptionIntegrationTest):
@unittest.skipUnless(any(AWS_CREDS.values()),
'AWS environment credentials are not set')
@unittest.skipIf(sys.version_info[:2] >= (3, 10) and
sys.platform == 'win32',
'These tests hang with Python 3.10 on Windows')
def setUp(self):
super(TestKmsTLSOptions, self).setUp()
# 1, create client with only tlsCAFile.
providers = copy.deepcopy(ALL_KMS_PROVIDERS)
providers['azure']['identityPlatformEndpoint'] = '127.0.0.1:8002'
providers['gcp']['endpoint'] = '127.0.0.1:8002'
kms_tls_opts_ca_only = {
'aws': {'tlsCAFile': CA_PEM},
'azure': {'tlsCAFile': CA_PEM},
'gcp': {'tlsCAFile': CA_PEM},
'kmip': {'tlsCAFile': CA_PEM},
}
self.client_encryption_no_client_cert = ClientEncryption(
providers, 'keyvault.datakeys', self.client, OPTS,
kms_tls_options=kms_tls_opts_ca_only)
self.addCleanup(self.client_encryption_no_client_cert.close)
# 2, same providers as above but with tlsCertificateKeyFile.
kms_tls_opts = copy.deepcopy(kms_tls_opts_ca_only)
for p in kms_tls_opts:
kms_tls_opts[p]['tlsCertificateKeyFile'] = CLIENT_PEM
self.client_encryption_with_tls = ClientEncryption(
providers, 'keyvault.datakeys', self.client, OPTS,
kms_tls_options=kms_tls_opts)
self.addCleanup(self.client_encryption_with_tls.close)
# 3, update endpoints to expired host.
providers = copy.deepcopy(providers)
providers['azure']['identityPlatformEndpoint'] = '127.0.0.1:8000'
providers['gcp']['endpoint'] = '127.0.0.1:8000'
providers['kmip']['endpoint'] = '127.0.0.1:8000'
self.client_encryption_expired = ClientEncryption(
providers, 'keyvault.datakeys', self.client, OPTS,
kms_tls_options=kms_tls_opts_ca_only)
self.addCleanup(self.client_encryption_expired.close)
# 3, update endpoints to invalid host.
providers = copy.deepcopy(providers)
providers['azure']['identityPlatformEndpoint'] = '127.0.0.1:8001'
providers['gcp']['endpoint'] = '127.0.0.1:8001'
providers['kmip']['endpoint'] = '127.0.0.1:8001'
self.client_encryption_invalid_hostname = ClientEncryption(
providers, 'keyvault.datakeys', self.client, OPTS,
kms_tls_options=kms_tls_opts_ca_only)
self.addCleanup(self.client_encryption_invalid_hostname.close)
# Errors when client has no cert, some examples:
# [SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (_ssl.c:2623)
self.cert_error = 'certificate required|SSL handshake failed'
# On Windows this error might be:
# [WinError 10054] An existing connection was forcibly closed by the remote host
if sys.platform == 'win32':
self.cert_error += '|forcibly closed'
# On Windows Python 3.10+ this error might be:
# EOF occurred in violation of protocol (_ssl.c:2384)
if sys.version_info[:2] >= (3, 10):
self.cert_error += '|forcibly closed'
def test_01_aws(self):
key = {
'region': 'us-east-1',
'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0',
'endpoint': '127.0.0.1:8002',
}
with self.assertRaisesRegex(EncryptionError, self.cert_error):
self.client_encryption_no_client_cert.create_data_key('aws', key)
# "parse error" here means that the TLS handshake succeeded.
with self.assertRaisesRegex(EncryptionError, 'parse error'):
self.client_encryption_with_tls.create_data_key('aws', key)
# Some examples:
# certificate verify failed: certificate has expired (_ssl.c:1129)
# amazon1-2018 Python 3.6: certificate verify failed (_ssl.c:852)
key['endpoint'] = '127.0.0.1:8000'
with self.assertRaisesRegex(
EncryptionError, 'expired|certificate verify failed'):
self.client_encryption_expired.create_data_key('aws', key)
# Some examples:
# certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)"
# hostname '127.0.0.1' doesn't match 'wronghost.com'
key['endpoint'] = '127.0.0.1:8001'
with self.assertRaisesRegex(
EncryptionError, 'IP address mismatch|wronghost'):
self.client_encryption_invalid_hostname.create_data_key('aws', key)
def test_02_azure(self):
key = {'keyVaultEndpoint': 'doesnotexist.local', 'keyName': 'foo'}
# Missing client cert error.
with self.assertRaisesRegex(EncryptionError, self.cert_error):
self.client_encryption_no_client_cert.create_data_key('azure', key)
# "HTTP status=404" here means that the TLS handshake succeeded.
with self.assertRaisesRegex(EncryptionError, 'HTTP status=404'):
self.client_encryption_with_tls.create_data_key('azure', key)
# Expired cert error.
with self.assertRaisesRegex(
EncryptionError, 'expired|certificate verify failed'):
self.client_encryption_expired.create_data_key('azure', key)
# Invalid cert hostname error.
with self.assertRaisesRegex(
EncryptionError, 'IP address mismatch|wronghost'):
self.client_encryption_invalid_hostname.create_data_key(
'azure', key)
def test_03_gcp(self):
key = {'projectId': 'foo', 'location': 'bar', 'keyRing': 'baz',
'keyName': 'foo'}
# Missing client cert error.
with self.assertRaisesRegex(EncryptionError, self.cert_error):
self.client_encryption_no_client_cert.create_data_key('gcp', key)
# "HTTP status=404" here means that the TLS handshake succeeded.
with self.assertRaisesRegex(EncryptionError, 'HTTP status=404'):
self.client_encryption_with_tls.create_data_key('gcp', key)
# Expired cert error.
with self.assertRaisesRegex(
EncryptionError, 'expired|certificate verify failed'):
self.client_encryption_expired.create_data_key('gcp', key)
# Invalid cert hostname error.
with self.assertRaisesRegex(
EncryptionError, 'IP address mismatch|wronghost'):
self.client_encryption_invalid_hostname.create_data_key('gcp', key)
def test_04_kmip(self):
# Missing client cert error.
with self.assertRaisesRegex(EncryptionError, self.cert_error):
self.client_encryption_no_client_cert.create_data_key('kmip')
self.client_encryption_with_tls.create_data_key('kmip')
# Expired cert error.
with self.assertRaisesRegex(
EncryptionError, 'expired|certificate verify failed'):
self.client_encryption_expired.create_data_key('kmip')
# Invalid cert hostname error.
with self.assertRaisesRegex(
EncryptionError, 'IP address mismatch|wronghost'):
self.client_encryption_invalid_hostname.create_data_key('kmip')
if __name__ == "__main__":
unittest.main()