diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 23445d75e..02c173d99 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,6 +375,10 @@ functions: export FLE_AZURE_CLIENTSECRET="${fle_azure_clientsecret}" export FLE_GCP_EMAIL="${fle_gcp_email}" export FLE_GCP_PRIVATEKEY="${fle_gcp_privatekey}" + # Needed for generating temporary aws credentials. + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION=us-east-1 EOT fi - command: shell.exec diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a43176943..e0e73efc3 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -134,6 +134,9 @@ if [ -n "$TEST_ENCRYPTION" ]; then python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" # PATH is updated by PREPARE_SHELL for access to mongocryptd. + # Get access to the AWS temporary credentials: + # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN + . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh fi if [ -z "$DATA_LAKE" ]; then diff --git a/.evergreen/test-encryption-requirements.txt b/.evergreen/test-encryption-requirements.txt index b5a752b6b..d02df867a 100644 --- a/.evergreen/test-encryption-requirements.txt +++ b/.evergreen/test-encryption-requirements.txt @@ -1,2 +1,4 @@ cffi>=1.12.0,<2 -cryptography>=2,<4 +cryptography>=2 +# boto3 is required by drivers-evergreen-tools/.evergreen/csfle/set-temp-creds.sh +boto3<2 diff --git a/pymongo/encryption.py b/pymongo/encryption.py index a54a9b13a..cb91b0afa 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -371,7 +371,8 @@ class ClientEncryption(object): - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings. These are the AWS access key ID and AWS secret access key used - to generate KMS messages. + to generate KMS messages. An optional "sessionToken" may be + included to support temporary AWS credentials. - `azure`: Map with "tenantId", "clientId", and "clientSecret" as strings. Additionally, "identityPlatformEndpoint" may also be specified as a string (defaults to 'login.microsoftonline.com'). diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index da3a0f191..9ad0f22ba 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -58,7 +58,8 @@ class AutoEncryptionOpts(object): - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings. These are the AWS access key ID and AWS secret access key used - to generate KMS messages. + to generate KMS messages. An optional "sessionToken" may be + included to support temporary AWS credentials. - `azure`: Map with "tenantId", "clientId", and "clientSecret" as strings. Additionally, "identityPlatformEndpoint" may also be specified as a string (defaults to 'login.microsoftonline.com'). diff --git a/test/client-side-encryption/spec/awsTemporary.json b/test/client-side-encryption/spec/awsTemporary.json new file mode 100644 index 000000000..80257c6c2 --- /dev/null +++ b/test/client-side-encryption/spec/awsTemporary.json @@ -0,0 +1,237 @@ +{ + "runOn": [ + { + "minServerVersion": "4.1.10" + } + ], + "database_name": "default", + "collection_name": "default", + "data": [], + "json_schema": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + }, + "key_vault_data": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ], + "tests": [ + { + "description": "Insert a document with auto encryption using the AWS provider with temporary credentials", + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "awsTemporary": {} + } + } + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + } + ], + "expectations": [ + { + "command_started_event": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "command_name": "listCollections" + } + }, + { + "command_started_event": { + "command": { + "listCollections": 1, + "filter": { + "name": "datakeys" + }, + "$db": "keyvault" + }, + "command_name": "listCollections" + } + }, + { + "command_started_event": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault" + }, + "command_name": "find" + } + }, + { + "command_started_event": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + }, + "command_name": "insert" + } + } + ], + "outcome": { + "collection": { + "data": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + } + }, + { + "description": "Insert with invalid temporary credentials", + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "awsTemporaryNoSessionToken": {} + } + } + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + }, + "result": { + "errorContains": "security token" + } + } + ] + } + ] +} diff --git a/test/test_encryption.py b/test/test_encryption.py index 938628e8c..cf516c26e 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -447,6 +447,17 @@ AWS_CREDS = { 'secretAccessKey': os.environ.get('FLE_AWS_SECRET', '') } +AWS_TEMP_CREDS = { + 'accessKeyId': os.environ.get('CSFLE_AWS_TEMP_ACCESS_KEY_ID', ''), + 'secretAccessKey': os.environ.get('CSFLE_AWS_TEMP_SECRET_ACCESS_KEY', ''), + 'sessionToken': os.environ.get('CSFLE_AWS_TEMP_SESSION_TOKEN', '') +} + +AWS_TEMP_NO_SESSION_CREDS = { + 'accessKeyId': os.environ.get('CSFLE_AWS_TEMP_ACCESS_KEY_ID', ''), + 'secretAccessKey': os.environ.get('CSFLE_AWS_TEMP_SECRET_ACCESS_KEY', '') +} + AZURE_CREDS = { 'tenantId': os.environ.get('FLE_AZURE_TENANTID', ''), 'clientId': os.environ.get('FLE_AZURE_CLIENTID', ''), @@ -473,6 +484,16 @@ class TestSpec(SpecRunner): kms_providers['aws'] = AWS_CREDS if not any(AWS_CREDS.values()): self.skipTest('AWS environment credentials are not set') + if 'awsTemporary' in kms_providers: + kms_providers['aws'] = AWS_TEMP_CREDS + del kms_providers['awsTemporary'] + if not any(AWS_TEMP_CREDS.values()): + self.skipTest('AWS Temp environment credentials are not set') + if 'awsTemporaryNoSessionToken' in kms_providers: + kms_providers['aws'] = AWS_TEMP_NO_SESSION_CREDS + del kms_providers['awsTemporaryNoSessionToken'] + if not any(AWS_TEMP_NO_SESSION_CREDS.values()): + self.skipTest('AWS Temp environment credentials are not set') if 'azure' in kms_providers: kms_providers['azure'] = AZURE_CREDS if not any(AZURE_CREDS.values()): @@ -700,17 +721,17 @@ class TestDataKeyDoubleEncryption(EncryptionIntegrationTest): def test_data_key_local(self): self.run_test('local') - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_data_key_aws(self): self.run_test('aws') - @unittest.skipUnless(all(AZURE_CREDS.values()), + @unittest.skipUnless(any(AZURE_CREDS.values()), 'Azure environment credentials are not set') def test_data_key_azure(self): self.run_test('azure') - @unittest.skipUnless(all(GCP_CREDS.values()), + @unittest.skipUnless(any(GCP_CREDS.values()), 'GCP environment credentials are not set') def test_data_key_gcp(self): self.run_test('gcp') @@ -806,7 +827,7 @@ class TestViews(EncryptionIntegrationTest): class TestCorpus(EncryptionIntegrationTest): @classmethod - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def setUpClass(cls): super(TestCorpus, cls).setUpClass() @@ -1101,7 +1122,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): key_id=data_key_id) self.assertEqual('test', self.client_encryption.decrypt(encrypted)) - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_01_aws_region_key(self): self.run_test_expected_success( @@ -1110,7 +1131,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): "key": ("arn:aws:kms:us-east-1:579766882180:key/" "89fcc2c4-08b0-4bd9-9f25-e30687b580d0")}) - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_02_aws_region_key_endpoint(self): self.run_test_expected_success( @@ -1120,7 +1141,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): "89fcc2c4-08b0-4bd9-9f25-e30687b580d0"), "endpoint": "kms.us-east-1.amazonaws.com"}) - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_03_aws_region_key_endpoint_port(self): self.run_test_expected_success( @@ -1130,7 +1151,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): "89fcc2c4-08b0-4bd9-9f25-e30687b580d0"), "endpoint": "kms.us-east-1.amazonaws.com:443"}) - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_04_aws_endpoint_invalid_port(self): master_key = { @@ -1144,7 +1165,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): 'aws', master_key=master_key) self.assertIsInstance(ctx.exception.cause, socket.error) - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_05_aws_endpoint_wrong_region(self): master_key = { @@ -1161,7 +1182,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): self.client_encryption.create_data_key( 'aws', master_key=master_key) - @unittest.skipUnless(all(AWS_CREDS.values()), + @unittest.skipUnless(any(AWS_CREDS.values()), 'AWS environment credentials are not set') def test_06_aws_endpoint_invalid_host(self): master_key = { @@ -1174,7 +1195,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): self.client_encryption.create_data_key( 'aws', master_key=master_key) - @unittest.skipUnless(all(AZURE_CREDS.values()), + @unittest.skipUnless(any(AZURE_CREDS.values()), 'Azure environment credentials are not set') def test_07_azure(self): master_key = {'keyVaultEndpoint': 'key-vault-csfle.vault.azure.net', @@ -1187,7 +1208,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): self.client_encryption_invalid.create_data_key( 'azure', master_key=master_key) - @unittest.skipUnless(all(GCP_CREDS.values()), + @unittest.skipUnless(any(GCP_CREDS.values()), 'GCP environment credentials are not set') def test_08_gcp_valid_endpoint(self): master_key = { @@ -1204,7 +1225,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): self.client_encryption_invalid.create_data_key( 'gcp', master_key=master_key) - @unittest.skipUnless(all(GCP_CREDS.values()), + @unittest.skipUnless(any(GCP_CREDS.values()), 'GCP environment credentials are not set') def test_09_gcp_invalid_endpoint(self): master_key = { @@ -1288,7 +1309,7 @@ class AzureGCPEncryptionTestMixin(object): class TestAzureEncryption(AzureGCPEncryptionTestMixin, EncryptionIntegrationTest): @classmethod - @unittest.skipUnless(all(AZURE_CREDS.values()), + @unittest.skipUnless(any(AZURE_CREDS.values()), 'Azure environment credentials are not set') def setUpClass(cls): cls.KMS_PROVIDER_MAP = {'azure': AZURE_CREDS} @@ -1314,7 +1335,7 @@ class TestAzureEncryption(AzureGCPEncryptionTestMixin, class TestGCPEncryption(AzureGCPEncryptionTestMixin, EncryptionIntegrationTest): @classmethod - @unittest.skipUnless(all(GCP_CREDS.values()), + @unittest.skipUnless(any(GCP_CREDS.values()), 'GCP environment credentials are not set') def setUpClass(cls): cls.KMS_PROVIDER_MAP = {'gcp': GCP_CREDS}