mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-2404 Update CSFLE spec tests for KMS providers 'azure' and 'gcp' (#509)

Browse Source
This commit is contained in:
Prashant Mital 2020-11-04 07:59:54 -08:00 committed by GitHub
parent e49c418264
commit 1e2a52fe8b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 11893 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3662
test/client-side-encryption/corpus/corpus-encrypted.json
View File

File diff suppressed because it is too large Load Diff

33
test/client-side-encryption/corpus/corpus-key-azure.json Normal file
View File

@ -0,0 +1,33 @@
{
"_id": {
"$binary": {
"base64": "AZUREAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1601573901680"
}
},
"updateDate": {
"$date": {
"$numberLong": "1601573901680"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "azure",
"keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
"keyName": "key-name-csfle"
},
"keyAltNames": ["azure"]
}

35
test/client-side-encryption/corpus/corpus-key-gcp.json Normal file
View File

@ -0,0 +1,35 @@
{
"_id": {
"$binary": {
"base64": "GCPAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "CiQAIgLj0WyktnB4dfYHo5SLZ41K4ASQrjJUaSzl5vvVH0G12G0SiQEAjlV8XPlbnHDEDFbdTO4QIe8ER2/172U1ouLazG0ysDtFFIlSvWX5ZnZUrRMmp/R2aJkzLXEt/zf8Mn4Lfm+itnjgo5R9K4pmPNvvPKNZX5C16lrPT+aA+rd+zXFSmlMg3i5jnxvTdLHhg3G7Q/Uv1ZIJskKt95bzLoe0tUVzRWMYXLIEcohnQg==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1601574333107"
}
},
"updateDate": {
"$date": {
"$numberLong": "1601574333107"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "gcp",
"projectId": "devprod-drivers",
"location": "global",
"keyRing": "key-ring-csfle",
"keyName": "key-name-csfle"
},
"keyAltNames": ["gcp"]
}

3254
test/client-side-encryption/corpus/corpus-schema.json
View File

File diff suppressed because it is too large Load Diff

4578
test/client-side-encryption/corpus/corpus.json
View File

File diff suppressed because it is too large Load Diff

12
test/client-side-encryption/custom/azure-dek.json
View File

@ -1,13 +1,13 @@
{
"_id": {
"$binary": {
"base64": "As3URE1jRcyHOPjaLWHOXA==",
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "df6fFLZqBsZSnQz2SnTYWNBtznIHktVSDMaidAdL7yVVgxBJQ0DyPZUR2HDQB4hdYym3w4C+VGqzcyTZNJOXn6nJzpGrGlIQMcjv93HE4sP2d245ShQCi1nTkLmMaXN63E2fzltOY3jW7ojf5Z4+r8kxmzyfymmSRgo0w8AF7lUWvFhnBYoE4tE322L31vtAK3Zj8pTPvw8/TcUdMSI9Y669IIzxbMy5yMPmdzpnb8nceUv6/CJoeiLhbt5GgaHqIAv7tHFOY8ZX8ztowMLa3GeAjd9clvzraDTqrfMFYco/kDKAW5iPQQ+Xuy1fP8tyFp0ZwaL/7Ed2sc819j8FTQ==",
"base64": "n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==",
"subType": "00"
}
},
@ -26,8 +26,8 @@
},
"masterKey": {
"provider": "azure",
"keyVaultEndpoint": "key-vault-kevinalbs.vault.azure.net",
"keyName": "test-key"
}
"keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
"keyName": "key-name-csfle"
},
"keyAltNames": ["altname", "azure_altname"]
}

5
test/client-side-encryption/custom/azure-gcp-schema.json
View File

@ -6,7 +6,7 @@
"encrypt": {
"keyId": [{
"$binary": {
"base64": "As3URE1jRcyHOPjaLWHOXA==",
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
}],
@ -18,7 +18,7 @@
"encrypt": {
"keyId": [{
"$binary": {
"base64": "osU8SLxJRHONbl8Oh5o+eg==",
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}],
@ -29,4 +29,3 @@
}
}
}

14
test/client-side-encryption/custom/gcp-dek.json
View File

@ -1,13 +1,13 @@
{
"_id": {
"$binary": {
"base64": "osU8SLxJRHONbl8Oh5o+eg==",
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "CiQAg4LDql74hjYPZ957Z7YpCrD6yTVVXKegflJDstQ/xngTyx0SiQEAkWNo/fjPj6jMNSvEop07/29Fu72QHFDRYM3e/KFHfnMQjKzfxb1yX1dC6MbO5FZG/UNBkXlJgPqbHNVuizea3QC24kV5iOiEb4nTM7+RW+8TfVb6QerWWe6MjC+kNpj4LMVcc1lFfVDeGgpJLyMLNGitrjR16qH8qQTNbGNy0toTL69JUmgS8Q==",
"base64": "CiQAIgLj0WyktnB4dfYHo5SLZ41K4ASQrjJUaSzl5vvVH0G12G0SiQEAjlV8XPlbnHDEDFbdTO4QIe8ER2/172U1ouLazG0ysDtFFIlSvWX5ZnZUrRMmp/R2aJkzLXEt/zf8Mn4Lfm+itnjgo5R9K4pmPNvvPKNZX5C16lrPT+aA+rd+zXFSmlMg3i5jnxvTdLHhg3G7Q/Uv1ZIJskKt95bzLoe0tUVzRWMYXLIEcohnQg==",
"subType": "00"
}
},
@ -26,10 +26,10 @@
},
"masterKey": {
"provider": "gcp",
"projectId": "csfle-poc",
"projectId": "devprod-drivers",
"location": "global",
"keyRing": "test",
"keyName": "quickstart"
}
"keyRing": "key-ring-csfle",
"keyName": "key-name-csfle"
},
"keyAltNames": ["altname", "gcp_altname"]
}

222
test/client-side-encryption/spec/azureKMS.json Normal file
View File

@ -0,0 +1,222 @@
{
"runOn": [
{
"minServerVersion": "4.1.10"
}
],
"database_name": "default",
"collection_name": "default",
"data": [],
"json_schema": {
"properties": {
"encrypted_string_aws": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_azure": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_gcp": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_local": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"
},
"key_vault_data": [
{
"_id": {
"$binary": {
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1601573901680"
}
},
"updateDate": {
"$date": {
"$numberLong": "1601573901680"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "azure",
"keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
"keyName": "key-name-csfle"
},
"keyAltNames": [
"altname",
"azure_altname"
]
}
],
"tests": [
{
"description": "Insert a document with auto encryption using Azure KMS provider",
"clientOptions": {
"autoEncryptOpts": {
"kmsProviders": {
"azure": {}
}
}
},
"operations": [
{
"name": "insertOne",
"arguments": {
"document": {
"_id": 1,
"encrypted_string_azure": "string0"
}
}
}
],
"expectations": [
{
"command_started_event": {
"command": {
"listCollections": 1,
"filter": {
"name": "default"
}
},
"command_name": "listCollections"
}
},
{
"command_started_event": {
"command": {
"listCollections": 1,
"filter": {
"name": "datakeys"
},
"$db": "keyvault"
},
"command_name": "listCollections"
}
},
{
"command_started_event": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault"
},
"command_name": "find"
}
},
{
"command_started_event": {
"command": {
"insert": "default",
"documents": [
{
"_id": 1,
"encrypted_string_azure": {
"$binary": {
"base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
"subType": "06"
}
}
}
],
"ordered": true
},
"command_name": "insert"
}
}
],
"outcome": {
"collection": {
"data": [
{
"_id": 1,
"encrypted_string_azure": {
"$binary": {
"base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
"subType": "06"
}
}
}
]
}
}
}
]
}

224
test/client-side-encryption/spec/gcpKMS.json Normal file
View File

@ -0,0 +1,224 @@
{
"runOn": [
{
"minServerVersion": "4.1.10"
}
],
"database_name": "default",
"collection_name": "default",
"data": [],
"json_schema": {
"properties": {
"encrypted_string_aws": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_azure": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_gcp": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"encrypted_string_local": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"
},
"key_vault_data": [
{
"_id": {
"$binary": {
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "CiQAIgLj0WyktnB4dfYHo5SLZ41K4ASQrjJUaSzl5vvVH0G12G0SiQEAjlV8XPlbnHDEDFbdTO4QIe8ER2/172U1ouLazG0ysDtFFIlSvWX5ZnZUrRMmp/R2aJkzLXEt/zf8Mn4Lfm+itnjgo5R9K4pmPNvvPKNZX5C16lrPT+aA+rd+zXFSmlMg3i5jnxvTdLHhg3G7Q/Uv1ZIJskKt95bzLoe0tUVzRWMYXLIEcohnQg==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1601574333107"
}
},
"updateDate": {
"$date": {
"$numberLong": "1601574333107"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "gcp",
"projectId": "devprod-drivers",
"location": "global",
"keyRing": "key-ring-csfle",
"keyName": "key-name-csfle"
},
"keyAltNames": [
"altname",
"gcp_altname"
]
}
],
"tests": [
{
"description": "Insert a document with auto encryption using GCP KMS provider",
"clientOptions": {
"autoEncryptOpts": {
"kmsProviders": {
"gcp": {}
}
}
},
"operations": [
{
"name": "insertOne",
"arguments": {
"document": {
"_id": 1,
"encrypted_string_gcp": "string0"
}
}
}
],
"expectations": [
{
"command_started_event": {
"command": {
"listCollections": 1,
"filter": {
"name": "default"
}
},
"command_name": "listCollections"
}
},
{
"command_started_event": {
"command": {
"listCollections": 1,
"filter": {
"name": "datakeys"
},
"$db": "keyvault"
},
"command_name": "listCollections"
}
},
{
"command_started_event": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault"
},
"command_name": "find"
}
},
{
"command_started_event": {
"command": {
"insert": "default",
"documents": [
{
"_id": 1,
"encrypted_string_gcp": {
"$binary": {
"base64": "ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==",
"subType": "06"
}
}
}
],
"ordered": true
},
"command_name": "insert"
}
}
],
"outcome": {
"collection": {
"data": [
{
"_id": 1,
"encrypted_string_gcp": {
"$binary": {
"base64": "ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==",
"subType": "06"
}
}
}
]
}
}
}
]
}

402
test/test_encryption.py
View File

@ -443,12 +443,20 @@ class TestExplicitSimple(EncryptionIntegrationTest):
# Spec tests
AWS_CREDS = {
'accessKeyId': os.environ.get('FLE_AWS_KEY', ''),
'secretAccessKey': os.environ.get('FLE_AWS_SECRET', '')
}
AZURE_CREDS = {
'tenantId': os.environ.get('FLE_AZURE_TENANTID', ''),
'clientId': os.environ.get('FLE_AZURE_CLIENTID', ''),
'clientSecret': os.environ.get('FLE_AZURE_CLIENTSECRET', '')}
GCP_CREDS = {
'email': os.environ.get('FLE_GCP_EMAIL', ''),
'privateKey': _unicode(os.environ.get('FLE_GCP_PRIVATEKEY', ''))}
class TestSpec(SpecRunner):
@ -466,6 +474,14 @@ class TestSpec(SpecRunner):
kms_providers['aws'] = AWS_CREDS
if not any(AWS_CREDS.values()):
self.skipTest('AWS environment credentials are not set')
if 'azure' in kms_providers:
kms_providers['azure'] = AZURE_CREDS
if not any(AZURE_CREDS.values()):
self.skipTest('Azure environment credentials are not set')
if 'gcp' in kms_providers:
kms_providers['gcp'] = GCP_CREDS
if not any(AZURE_CREDS.values()):
self.skipTest('GCP environment credentials are not set')
if 'key_vault_namespace' not in opts:
opts['key_vault_namespace'] = 'keyvault.datakeys'
opts = dict(opts)
@ -556,6 +572,10 @@ LOCAL_KEY_ID = Binary(
base64.b64decode(b'LOCALAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
AWS_KEY_ID = Binary(
base64.b64decode(b'AWSAAAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
AZURE_KEY_ID = Binary(
base64.b64decode(b'AZUREAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
GCP_KEY_ID = Binary(
base64.b64decode(b'GCPAAAAAAAAAAAAAAAAAAA=='), UUID_SUBTYPE)
def create_with_schema(coll, json_schema):
@ -578,120 +598,128 @@ def create_key_vault(vault, *data_keys):
class TestDataKeyDoubleEncryption(EncryptionIntegrationTest):
KMS_PROVIDERS = {'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS,
'local': {'key': LOCAL_MASTER_KEY}}
MASTER_KEYS = {
'aws': {
'region': 'us-east-1',
'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-'
'4bd9-9f25-e30687b580d0'},
'azure': {
'keyVaultEndpoint': 'key-vault-csfle.vault.azure.net',
'keyName': 'key-name-csfle'},
'gcp': {
'projectId': 'devprod-drivers',
'location': 'global',
'keyRing': 'key-ring-csfle',
'keyName': 'key-name-csfle'},
'local': None
}
@classmethod
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
@unittest.skipUnless(any([all(AWS_CREDS.values()),
all(AZURE_CREDS.values()),
all(GCP_CREDS.values())]),
'No environment credentials are set')
def setUpClass(cls):
super(TestDataKeyDoubleEncryption, cls).setUpClass()
@staticmethod
def kms_providers():
return {'aws': AWS_CREDS, 'local': {'key': LOCAL_MASTER_KEY}}
def test_data_key(self):
listener = OvertCommandListener()
client = rs_or_single_client(event_listeners=[listener])
self.addCleanup(client.close)
client.db.coll.drop()
vault = create_key_vault(client.keyvault.datakeys)
self.addCleanup(vault.drop)
cls.listener = OvertCommandListener()
cls.client = rs_or_single_client(event_listeners=[cls.listener])
cls.client.db.coll.drop()
cls.vault = create_key_vault(cls.client.keyvault.datakeys)
# Configure the encrypted field via the local schema_map option.
schemas = {
"db.coll": {
"bsonType": "object",
"properties": {
"encrypted_placeholder": {
"encrypt": {
"keyId": "/placeholder",
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
"db.coll": {
"bsonType": "object",
"properties": {
"encrypted_placeholder": {
"encrypt": {
"keyId": "/placeholder",
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
}
}
}
}
}
}
}
opts = AutoEncryptionOpts(
self.kms_providers(), 'keyvault.datakeys', schema_map=schemas)
client_encrypted = rs_or_single_client(
cls.KMS_PROVIDERS, 'keyvault.datakeys', schema_map=schemas)
cls.client_encrypted = rs_or_single_client(
auto_encryption_opts=opts, uuidRepresentation='standard')
self.addCleanup(client_encrypted.close)
cls.client_encryption = ClientEncryption(
cls.KMS_PROVIDERS, 'keyvault.datakeys', cls.client, OPTS)
client_encryption = ClientEncryption(
self.kms_providers(), 'keyvault.datakeys', client, OPTS)
self.addCleanup(client_encryption.close)
@classmethod
def tearDownClass(cls):
cls.client.close()
cls.vault.drop()
cls.client_encrypted.close()
cls.client_encryption.close()
# Local create data key.
listener.reset()
local_datakey_id = client_encryption.create_data_key(
'local', key_alt_names=['local_altname'])
self.assertBinaryUUID(local_datakey_id)
cmd = listener.results['started'][-1]
def setUp(self):
self.listener.reset()
def run_test(self, provider_name):
# Create data key.
master_key = self.MASTER_KEYS[provider_name]
datakey_id = self.client_encryption.create_data_key(
provider_name, master_key=master_key,
key_alt_names=['%s_altname' % (provider_name,)])
self.assertBinaryUUID(datakey_id)
cmd = self.listener.results['started'][-1]
self.assertEqual('insert', cmd.command_name)
self.assertEqual({'w': 'majority'}, cmd.command.get('writeConcern'))
docs = list(vault.find({'_id': local_datakey_id}))
docs = list(self.vault.find({'_id': datakey_id}))
self.assertEqual(len(docs), 1)
self.assertEqual(docs[0]['masterKey']['provider'], 'local')
self.assertEqual(docs[0]['masterKey']['provider'], provider_name)
# Local encrypt by key_id.
local_encrypted = client_encryption.encrypt(
'hello local',
# Encrypt by key_id.
encrypted = self.client_encryption.encrypt(
'hello %s' % (provider_name,),
Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_id=local_datakey_id)
self.assertEncrypted(local_encrypted)
client_encrypted.db.coll.insert_one(
{'_id': 'local', 'value': local_encrypted})
doc_decrypted = client_encrypted.db.coll.find_one({'_id': 'local'})
self.assertEqual(doc_decrypted['value'], 'hello local')
key_id=datakey_id)
self.assertEncrypted(encrypted)
self.client_encrypted.db.coll.insert_one(
{'_id': provider_name, 'value': encrypted})
doc_decrypted = self.client_encrypted.db.coll.find_one(
{'_id': provider_name})
self.assertEqual(doc_decrypted['value'], 'hello %s' % (provider_name,))
# Local encrypt by key_alt_name.
local_encrypted_altname = client_encryption.encrypt(
'hello local',
# Encrypt by key_alt_name.
encrypted_altname = self.client_encryption.encrypt(
'hello %s' % (provider_name,),
Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_alt_name='local_altname')
self.assertEqual(local_encrypted_altname, local_encrypted)
# AWS create data key.
listener.reset()
master_key = {
'region': 'us-east-1',
'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-'
'9f25-e30687b580d0'
}
aws_datakey_id = client_encryption.create_data_key(
'aws', master_key=master_key, key_alt_names=['aws_altname'])
self.assertBinaryUUID(aws_datakey_id)
cmd = listener.results['started'][-1]
self.assertEqual('insert', cmd.command_name)
self.assertEqual({'w': 'majority'}, cmd.command.get('writeConcern'))
docs = list(vault.find({'_id': aws_datakey_id}))
self.assertEqual(len(docs), 1)
self.assertEqual(docs[0]['masterKey']['provider'], 'aws')
# AWS encrypt by key_id.
aws_encrypted = client_encryption.encrypt(
'hello aws',
Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_id=aws_datakey_id)
self.assertEncrypted(aws_encrypted)
client_encrypted.db.coll.insert_one(
{'_id': 'aws', 'value': aws_encrypted})
doc_decrypted = client_encrypted.db.coll.find_one({'_id': 'aws'})
self.assertEqual(doc_decrypted['value'], 'hello aws')
# AWS encrypt by key_alt_name.
aws_encrypted_altname = client_encryption.encrypt(
'hello aws',
Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_alt_name='aws_altname')
self.assertEqual(aws_encrypted_altname, aws_encrypted)
key_alt_name='%s_altname' % (provider_name,))
self.assertEqual(encrypted_altname, encrypted)
# Explicitly encrypting an auto encrypted field.
msg = (r'Cannot encrypt element of type binData because schema '
r'requires that type is one of: \[ string \]')
with self.assertRaisesRegex(EncryptionError, msg):
client_encrypted.db.coll.insert_one(
{'encrypted_placeholder': local_encrypted})
self.client_encrypted.db.coll.insert_one(
{'encrypted_placeholder': encrypted})
def test_data_key_local(self):
self.run_test('local')
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_data_key_aws(self):
self.run_test('aws')
@unittest.skipUnless(all(AZURE_CREDS.values()),
'Azure environment credentials are not set')
def test_data_key_azure(self):
self.run_test('azure')
@unittest.skipUnless(all(GCP_CREDS.values()),
'GCP environment credentials are not set')
def test_data_key_gcp(self):
self.run_test('gcp')
class TestExternalKeyVault(EncryptionIntegrationTest):
@ -791,7 +819,10 @@ class TestCorpus(EncryptionIntegrationTest):
@staticmethod
def kms_providers():
return {'aws': AWS_CREDS, 'local': {'key': LOCAL_MASTER_KEY}}
return {'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS,
'local': {'key': LOCAL_MASTER_KEY}}
@staticmethod
def fix_up_schema(json_schema):
@ -827,7 +858,9 @@ class TestCorpus(EncryptionIntegrationTest):
vault = create_key_vault(
self.client.keyvault.datakeys,
json_data('corpus', 'corpus-key-local.json'),
json_data('corpus', 'corpus-key-aws.json'))
json_data('corpus', 'corpus-key-aws.json'),
json_data('corpus', 'corpus-key-azure.json'),
json_data('corpus', 'corpus-key-gcp.json'))
self.addCleanup(vault.drop)
client_encrypted = rs_or_single_client(
@ -843,7 +876,8 @@ class TestCorpus(EncryptionIntegrationTest):
corpus_copied = SON()
for key, value in corpus.items():
corpus_copied[key] = copy.deepcopy(value)
if key in ('_id', 'altname_aws', 'altname_local'):
if key in ('_id', 'altname_aws', 'altname_azure', 'altname_gcp',
'altname_local'):
continue
if value['method'] == 'auto':
continue
@ -851,12 +885,16 @@ class TestCorpus(EncryptionIntegrationTest):
identifier = value['identifier']
self.assertIn(identifier, ('id', 'altname'))
kms = value['kms']
self.assertIn(kms, ('local', 'aws'))
self.assertIn(kms, ('local', 'aws', 'azure', 'gcp'))
if identifier == 'id':
if kms == 'local':
kwargs = dict(key_id=LOCAL_KEY_ID)
else:
elif kms == 'aws':
kwargs = dict(key_id=AWS_KEY_ID)
elif kms == 'azure':
kwargs = dict(key_id=AZURE_KEY_ID)
else:
kwargs = dict(key_id=GCP_KEY_ID)
else:
kwargs = dict(key_alt_name=kms)
@ -888,7 +926,8 @@ class TestCorpus(EncryptionIntegrationTest):
'corpus', 'corpus-encrypted.json'), corpus)
corpus_encrypted_actual = coll.find_one()
for key, value in corpus_encrypted_actual.items():
if key in ('_id', 'altname_aws', 'altname_local'):
if key in ('_id', 'altname_aws', 'altname_azure',
'altname_gcp', 'altname_local'):
continue
if value['algo'] == 'det':
@ -1026,50 +1065,80 @@ class TestBsonSizeBatches(EncryptionIntegrationTest):
self.assertIn('object to insert too large', err['errmsg'])
class TestCustomEndpoint(EncryptionIntegrationTest):
"""Prose tests for creating data keys with a custom endpoint."""
@classmethod
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
@unittest.skipUnless(any([all(AWS_CREDS.values()),
all(AZURE_CREDS.values()),
all(GCP_CREDS.values())]),
'No environment credentials are set')
def setUpClass(cls):
super(TestCustomEndpoint, cls).setUpClass()
cls.client_encryption = ClientEncryption(
{'aws': AWS_CREDS}, 'keyvault.datakeys', client_context.client, OPTS)
def _test_create_data_key(self, master_key):
def setUp(self):
kms_providers = {'aws': AWS_CREDS,
'azure': AZURE_CREDS,
'gcp': GCP_CREDS}
self.client_encryption = ClientEncryption(
kms_providers=kms_providers,
key_vault_namespace='keyvault.datakeys',
key_vault_client=client_context.client,
codec_options=OPTS)
kms_providers_invalid = copy.deepcopy(kms_providers)
kms_providers_invalid['azure']['identityPlatformEndpoint'] = 'example.com:443'
kms_providers_invalid['gcp']['endpoint'] = 'example.com:443'
self.client_encryption_invalid = ClientEncryption(
kms_providers=kms_providers_invalid,
key_vault_namespace='keyvault.datakeys',
key_vault_client=client_context.client,
codec_options=OPTS)
def tearDown(self):
self.client_encryption.close()
self.client_encryption_invalid.close()
def run_test_expected_success(self, provider_name, master_key):
data_key_id = self.client_encryption.create_data_key(
'aws', master_key=master_key)
provider_name, master_key=master_key)
encrypted = self.client_encryption.encrypt(
'test', Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_id=data_key_id)
self.assertEqual('test', self.client_encryption.decrypt(encrypted))
def test_02_aws_region_key(self):
self._test_create_data_key({
"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
"89fcc2c4-08b0-4bd9-9f25-e30687b580d0")
})
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_01_aws_region_key(self):
self.run_test_expected_success(
'aws',
{"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
"89fcc2c4-08b0-4bd9-9f25-e30687b580d0")})
def test_03_aws_region_key_endpoint(self):
self._test_create_data_key({
"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
"89fcc2c4-08b0-4bd9-9f25-e30687b580d0"),
"endpoint": "kms.us-east-1.amazonaws.com"
})
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_02_aws_region_key_endpoint(self):
self.run_test_expected_success(
'aws',
{"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
"89fcc2c4-08b0-4bd9-9f25-e30687b580d0"),
"endpoint": "kms.us-east-1.amazonaws.com"})
def test_04_aws_region_key_endpoint_port(self):
self._test_create_data_key({
"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
"89fcc2c4-08b0-4bd9-9f25-e30687b580d0"),
"endpoint": "kms.us-east-1.amazonaws.com:443"
})
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_03_aws_region_key_endpoint_port(self):
self.run_test_expected_success(
'aws',
{"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
"89fcc2c4-08b0-4bd9-9f25-e30687b580d0"),
"endpoint": "kms.us-east-1.amazonaws.com:443"})
def test_05_endpoint_invalid_port(self):
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_04_aws_endpoint_invalid_port(self):
master_key = {
"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
@ -1081,7 +1150,9 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
'aws', master_key=master_key)
self.assertIsInstance(ctx.exception.cause, socket.error)
def test_05_endpoint_wrong_region(self):
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_05_aws_endpoint_wrong_region(self):
master_key = {
"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
@ -1096,7 +1167,9 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
self.client_encryption.create_data_key(
'aws', master_key=master_key)
def test_05_endpoint_invalid_host(self):
@unittest.skipUnless(all(AWS_CREDS.values()),
'AWS environment credentials are not set')
def test_06_aws_endpoint_invalid_host(self):
master_key = {
"region": "us-east-1",
"key": ("arn:aws:kms:us-east-1:579766882180:key/"
@ -1107,6 +1180,52 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
self.client_encryption.create_data_key(
'aws', master_key=master_key)
@unittest.skipUnless(all(AZURE_CREDS.values()),
'Azure environment credentials are not set')
def test_07_azure(self):
master_key = {'keyVaultEndpoint': 'key-vault-csfle.vault.azure.net',
'keyName': 'key-name-csfle'}
self.run_test_expected_success('azure', master_key)
# The full error should be something like:
# "Invalid JSON in KMS response. HTTP status=404. Error: Got parse error at '<', position 0: 'SPECIAL_EXPECTED'"
with self.assertRaisesRegex(EncryptionError, 'parse error'):
self.client_encryption_invalid.create_data_key(
'azure', master_key=master_key)
@unittest.skipUnless(all(GCP_CREDS.values()),
'GCP environment credentials are not set')
def test_08_gcp_valid_endpoint(self):
master_key = {
"projectId": "devprod-drivers",
"location": "global",
"keyRing": "key-ring-csfle",
"keyName": "key-name-csfle",
"endpoint": "cloudkms.googleapis.com:443"}
self.run_test_expected_success('gcp', master_key)
# The full error should be something like:
# "Invalid JSON in KMS response. HTTP status=404. Error: Got parse error at '<', position 0: 'SPECIAL_EXPECTED'"
with self.assertRaisesRegex(EncryptionError, 'parse error'):
self.client_encryption_invalid.create_data_key(
'gcp', master_key=master_key)
@unittest.skipUnless(all(GCP_CREDS.values()),
'GCP environment credentials are not set')
def test_09_gcp_invalid_endpoint(self):
master_key = {
"projectId": "devprod-drivers",
"location": "global",
"keyRing": "key-ring-csfle",
"keyName": "key-name-csfle",
"endpoint": "example.com:443"}
# The full error should be something like:
# "Invalid KMS response, no access_token returned. HTTP status=200"
with self.assertRaisesRegex(EncryptionError, "Invalid KMS response"):
self.client_encryption.create_data_key(
'gcp', master_key=master_key)
class AzureGCPEncryptionTestMixin(object):
DEK = None
@ -1129,12 +1248,12 @@ class AzureGCPEncryptionTestMixin(object):
self.addCleanup(client_encryption.close)
ciphertext = client_encryption.encrypt(
'test',
'string0',
algorithm=Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
key_id=Binary.from_uuid(self.DEK['_id'], STANDARD))
self.assertEqual(bytes(ciphertext), base64.b64decode(expectation))
self.assertEqual(client_encryption.decrypt(ciphertext), 'test')
self.assertEqual(client_encryption.decrypt(ciphertext), 'string0')
def _test_automatic(self, expectation_extjson, payload):
encrypted_db = "db"
@ -1172,16 +1291,10 @@ class AzureGCPEncryptionTestMixin(object):
self.assertEqual(output_doc[key], value)
AZURE_CREDS = {
'tenantId': os.environ.get('FLE_AZURE_TENANTID', ''),
'clientId': os.environ.get('FLE_AZURE_CLIENTID', ''),
'clientSecret': os.environ.get('FLE_AZURE_CLIENTSECRET', '')}
class TestAzureEncryption(AzureGCPEncryptionTestMixin,
EncryptionIntegrationTest):
@classmethod
@unittest.skipUnless(any(AZURE_CREDS.values()),
@unittest.skipUnless(all(AZURE_CREDS.values()),
'Azure environment credentials are not set')
def setUpClass(cls):
cls.KMS_PROVIDER_MAP = {'azure': AZURE_CREDS}
@ -1191,28 +1304,23 @@ class TestAzureEncryption(AzureGCPEncryptionTestMixin,
def test_explicit(self):
return self._test_explicit(
'AQLN1ERNY0XMhzj42i1hzlwC8/OSU9bHfaQRmmRF5l7d5ZpqJX13qF5zSyExo8N9c1b6uS/LoKrHNzcEMKNrkpi3jf2HiShTFRF0xi8AOD9yfw==')
'AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==')
def test_automatic(self):
expected_document_extjson = textwrap.dedent("""
{"secret_azure": {
"$binary": {
"base64": "AQLN1ERNY0XMhzj42i1hzlwC8/OSU9bHfaQRmmRF5l7d5ZpqJX13qF5zSyExo8N9c1b6uS/LoKrHNzcEMKNrkpi3jf2HiShTFRF0xi8AOD9yfw==",
"base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
"subType": "06"}
}}""")
return self._test_automatic(
expected_document_extjson, {"secret_azure": "test"})
GCP_CREDS = {
'email': os.environ.get('FLE_GCP_EMAIL', ''),
'privateKey': _unicode(os.environ.get('FLE_GCP_PRIVATEKEY', ''))}
expected_document_extjson, {"secret_azure": "string0"})
class TestGCPEncryption(AzureGCPEncryptionTestMixin,
EncryptionIntegrationTest):
@classmethod
@unittest.skipUnless(any(GCP_CREDS.values()),
@unittest.skipUnless(all(GCP_CREDS.values()),
'GCP environment credentials are not set')
def setUpClass(cls):
cls.KMS_PROVIDER_MAP = {'gcp': GCP_CREDS}
@ -1222,17 +1330,17 @@ class TestGCPEncryption(AzureGCPEncryptionTestMixin,
def test_explicit(self):
return self._test_explicit(
'AaLFPEi8SURzjW5fDoeaPnoCGcOFAmFOPpn5584VPJJ8iXIgml3YDxMRZD9IWv5otyoft8fBzL1LsDEp0lTeB32cV1gOj0IYeAKHhGIleuHZtA==')
'ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==')
def test_automatic(self):
expected_document_extjson = textwrap.dedent("""
{"secret_gcp": {
"$binary": {
"base64": "AaLFPEi8SURzjW5fDoeaPnoCGcOFAmFOPpn5584VPJJ8iXIgml3YDxMRZD9IWv5otyoft8fBzL1LsDEp0lTeB32cV1gOj0IYeAKHhGIleuHZtA==",
"base64": "ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==",
"subType": "06"}
}}""")
return self._test_automatic(
expected_document_extjson, {"secret_gcp": "test"})
expected_document_extjson, {"secret_gcp": "string0"})
if __name__ == "__main__":