PYTHON-2539 Test AWS temporary credentials via "sessionToken" for CSFLE (#569)

2021-02-18 08:52:36 -08:00 · 2021-02-18 08:52:36 -08:00 · 99a4f28450
commit 99a4f28450
parent 95974617bd
7 changed files with 287 additions and 18 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -375,6 +375,10 @@ functions:
            export FLE_AZURE_CLIENTSECRET="${fle_azure_clientsecret}"
            export FLE_GCP_EMAIL="${fle_gcp_email}"
            export FLE_GCP_PRIVATEKEY="${fle_gcp_privatekey}"
+            # Needed for generating temporary aws credentials.
+            export AWS_ACCESS_KEY_ID="${fle_aws_key}"
+            export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}"
+            export AWS_DEFAULT_REGION=us-east-1
          EOT
          fi
    - command: shell.exec
--- a/.evergreen/run-tests.sh
+++ b/.evergreen/run-tests.sh
@ -134,6 +134,9 @@ if [ -n "$TEST_ENCRYPTION" ]; then
    python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
    # PATH is updated by PREPARE_SHELL for access to mongocryptd.

+    # Get access to the AWS temporary credentials:
+    # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
+    . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
 fi

 if [ -z "$DATA_LAKE" ]; then
--- a/.evergreen/test-encryption-requirements.txt
+++ b/.evergreen/test-encryption-requirements.txt
@ -1,2 +1,4 @@
 cffi>=1.12.0,<2
-cryptography>=2,<4
+cryptography>=2
+# boto3 is required by drivers-evergreen-tools/.evergreen/csfle/set-temp-creds.sh
+boto3<2
--- a/pymongo/encryption.py
+++ b/pymongo/encryption.py
@ -371,7 +371,8 @@ class ClientEncryption(object):

              - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
                These are the AWS access key ID and AWS secret access key used
-                to generate KMS messages.
+                to generate KMS messages. An optional "sessionToken" may be
+                included to support temporary AWS credentials.
              - `azure`: Map with "tenantId", "clientId", and "clientSecret" as
                strings. Additionally, "identityPlatformEndpoint" may also be
                specified as a string (defaults to 'login.microsoftonline.com').
--- a/pymongo/encryption_options.py
+++ b/pymongo/encryption_options.py
@ -58,7 +58,8 @@ class AutoEncryptionOpts(object):

              - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
                These are the AWS access key ID and AWS secret access key used
-                to generate KMS messages.
+                to generate KMS messages. An optional "sessionToken" may be
+                included to support temporary AWS credentials.
              - `azure`: Map with "tenantId", "clientId", and "clientSecret" as
                strings. Additionally, "identityPlatformEndpoint" may also be
                specified as a string (defaults to 'login.microsoftonline.com').
--- a/test/client-side-encryption/spec/awsTemporary.json
+++ b/test/client-side-encryption/spec/awsTemporary.json
@ -0,0 +1,237 @@
+{
+  "runOn": [
+    {
+      "minServerVersion": "4.1.10"
+    }
+  ],
+  "database_name": "default",
+  "collection_name": "default",
+  "data": [],
+  "json_schema": {
+    "properties": {
+      "encrypted_w_altname": {
+        "encrypt": {
+          "keyId": "/altname",
+          "bsonType": "string",
+          "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
+        }
+      },
+      "encrypted_string": {
+        "encrypt": {
+          "keyId": [
+            {
+              "$binary": {
+                "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+                "subType": "04"
+              }
+            }
+          ],
+          "bsonType": "string",
+          "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+        }
+      },
+      "random": {
+        "encrypt": {
+          "keyId": [
+            {
+              "$binary": {
+                "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+                "subType": "04"
+              }
+            }
+          ],
+          "bsonType": "string",
+          "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
+        }
+      },
+      "encrypted_string_equivalent": {
+        "encrypt": {
+          "keyId": [
+            {
+              "$binary": {
+                "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+                "subType": "04"
+              }
+            }
+          ],
+          "bsonType": "string",
+          "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+        }
+      }
+    },
+    "bsonType": "object"
+  },
+  "key_vault_data": [
+    {
+      "status": 1,
+      "_id": {
+        "$binary": {
+          "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+          "subType": "04"
+        }
+      },
+      "masterKey": {
+        "provider": "aws",
+        "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+        "region": "us-east-1"
+      },
+      "updateDate": {
+        "$date": {
+          "$numberLong": "1552949630483"
+        }
+      },
+      "keyMaterial": {
+        "$binary": {
+          "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO",
+          "subType": "00"
+        }
+      },
+      "creationDate": {
+        "$date": {
+          "$numberLong": "1552949630483"
+        }
+      },
+      "keyAltNames": [
+        "altname",
+        "another_altname"
+      ]
+    }
+  ],
+  "tests": [
+    {
+      "description": "Insert a document with auto encryption using the AWS provider with temporary credentials",
+      "clientOptions": {
+        "autoEncryptOpts": {
+          "kmsProviders": {
+            "awsTemporary": {}
+          }
+        }
+      },
+      "operations": [
+        {
+          "name": "insertOne",
+          "arguments": {
+            "document": {
+              "_id": 1,
+              "encrypted_string": "string0"
+            }
+          }
+        }
+      ],
+      "expectations": [
+        {
+          "command_started_event": {
+            "command": {
+              "listCollections": 1,
+              "filter": {
+                "name": "default"
+              }
+            },
+            "command_name": "listCollections"
+          }
+        },
+        {
+          "command_started_event": {
+            "command": {
+              "listCollections": 1,
+              "filter": {
+                "name": "datakeys"
+              },
+              "$db": "keyvault"
+            },
+            "command_name": "listCollections"
+          }
+        },
+        {
+          "command_started_event": {
+            "command": {
+              "find": "datakeys",
+              "filter": {
+                "$or": [
+                  {
+                    "_id": {
+                      "$in": [
+                        {
+                          "$binary": {
+                            "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+                            "subType": "04"
+                          }
+                        }
+                      ]
+                    }
+                  },
+                  {
+                    "keyAltNames": {
+                      "$in": []
+                    }
+                  }
+                ]
+              },
+              "$db": "keyvault"
+            },
+            "command_name": "find"
+          }
+        },
+        {
+          "command_started_event": {
+            "command": {
+              "insert": "default",
+              "documents": [
+                {
+                  "_id": 1,
+                  "encrypted_string": {
+                    "$binary": {
+                      "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==",
+                      "subType": "06"
+                    }
+                  }
+                }
+              ],
+              "ordered": true
+            },
+            "command_name": "insert"
+          }
+        }
+      ],
+      "outcome": {
+        "collection": {
+          "data": [
+            {
+              "_id": 1,
+              "encrypted_string": {
+                "$binary": {
+                  "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==",
+                  "subType": "06"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "description": "Insert with invalid temporary credentials",
+      "clientOptions": {
+        "autoEncryptOpts": {
+          "kmsProviders": {
+            "awsTemporaryNoSessionToken": {}
+          }
+        }
+      },
+      "operations": [
+        {
+          "name": "insertOne",
+          "arguments": {
+            "document": {
+              "_id": 1,
+              "encrypted_string": "string0"
+            }
+          },
+          "result": {
+            "errorContains": "security token"
+          }
+        }
+      ]
+    }
+  ]
+}
--- a/test/test_encryption.py
+++ b/test/test_encryption.py
@ -447,6 +447,17 @@ AWS_CREDS = {
    'secretAccessKey': os.environ.get('FLE_AWS_SECRET', '')
 }

+AWS_TEMP_CREDS = {
+    'accessKeyId': os.environ.get('CSFLE_AWS_TEMP_ACCESS_KEY_ID', ''),
+    'secretAccessKey': os.environ.get('CSFLE_AWS_TEMP_SECRET_ACCESS_KEY', ''),
+    'sessionToken': os.environ.get('CSFLE_AWS_TEMP_SESSION_TOKEN', '')
+}
+
+AWS_TEMP_NO_SESSION_CREDS = {
+    'accessKeyId': os.environ.get('CSFLE_AWS_TEMP_ACCESS_KEY_ID', ''),
+    'secretAccessKey': os.environ.get('CSFLE_AWS_TEMP_SECRET_ACCESS_KEY', '')
+}
+
 AZURE_CREDS = {
    'tenantId': os.environ.get('FLE_AZURE_TENANTID', ''),
    'clientId': os.environ.get('FLE_AZURE_CLIENTID', ''),
@ -473,6 +484,16 @@ class TestSpec(SpecRunner):
            kms_providers['aws'] = AWS_CREDS
            if not any(AWS_CREDS.values()):
                self.skipTest('AWS environment credentials are not set')
+        if 'awsTemporary' in kms_providers:
+            kms_providers['aws'] = AWS_TEMP_CREDS
+            del kms_providers['awsTemporary']
+            if not any(AWS_TEMP_CREDS.values()):
+                self.skipTest('AWS Temp environment credentials are not set')
+        if 'awsTemporaryNoSessionToken' in kms_providers:
+            kms_providers['aws'] = AWS_TEMP_NO_SESSION_CREDS
+            del kms_providers['awsTemporaryNoSessionToken']
+            if not any(AWS_TEMP_NO_SESSION_CREDS.values()):
+                self.skipTest('AWS Temp environment credentials are not set')
        if 'azure' in kms_providers:
            kms_providers['azure'] = AZURE_CREDS
            if not any(AZURE_CREDS.values()):
@ -700,17 +721,17 @@ class TestDataKeyDoubleEncryption(EncryptionIntegrationTest):
    def test_data_key_local(self):
        self.run_test('local')

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_data_key_aws(self):
        self.run_test('aws')

-    @unittest.skipUnless(all(AZURE_CREDS.values()),
+    @unittest.skipUnless(any(AZURE_CREDS.values()),
                         'Azure environment credentials are not set')
    def test_data_key_azure(self):
        self.run_test('azure')

-    @unittest.skipUnless(all(GCP_CREDS.values()),
+    @unittest.skipUnless(any(GCP_CREDS.values()),
                         'GCP environment credentials are not set')
    def test_data_key_gcp(self):
        self.run_test('gcp')
@ -806,7 +827,7 @@ class TestViews(EncryptionIntegrationTest):
 class TestCorpus(EncryptionIntegrationTest):

    @classmethod
-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def setUpClass(cls):
        super(TestCorpus, cls).setUpClass()
@ -1101,7 +1122,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
            key_id=data_key_id)
        self.assertEqual('test', self.client_encryption.decrypt(encrypted))

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_01_aws_region_key(self):
        self.run_test_expected_success(
@ -1110,7 +1131,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
             "key": ("arn:aws:kms:us-east-1:579766882180:key/"
                     "89fcc2c4-08b0-4bd9-9f25-e30687b580d0")})

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_02_aws_region_key_endpoint(self):
        self.run_test_expected_success(
@ -1120,7 +1141,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
                     "89fcc2c4-08b0-4bd9-9f25-e30687b580d0"),
             "endpoint": "kms.us-east-1.amazonaws.com"})

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_03_aws_region_key_endpoint_port(self):
        self.run_test_expected_success(
@ -1130,7 +1151,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
                     "89fcc2c4-08b0-4bd9-9f25-e30687b580d0"),
             "endpoint": "kms.us-east-1.amazonaws.com:443"})

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_04_aws_endpoint_invalid_port(self):
        master_key = {
@ -1144,7 +1165,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
                'aws', master_key=master_key)
        self.assertIsInstance(ctx.exception.cause, socket.error)

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_05_aws_endpoint_wrong_region(self):
        master_key = {
@ -1161,7 +1182,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
            self.client_encryption.create_data_key(
                'aws', master_key=master_key)

-    @unittest.skipUnless(all(AWS_CREDS.values()),
+    @unittest.skipUnless(any(AWS_CREDS.values()),
                         'AWS environment credentials are not set')
    def test_06_aws_endpoint_invalid_host(self):
        master_key = {
@ -1174,7 +1195,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
            self.client_encryption.create_data_key(
                'aws', master_key=master_key)

-    @unittest.skipUnless(all(AZURE_CREDS.values()),
+    @unittest.skipUnless(any(AZURE_CREDS.values()),
                         'Azure environment credentials are not set')
    def test_07_azure(self):
        master_key = {'keyVaultEndpoint': 'key-vault-csfle.vault.azure.net',
@ -1187,7 +1208,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
            self.client_encryption_invalid.create_data_key(
                'azure', master_key=master_key)

-    @unittest.skipUnless(all(GCP_CREDS.values()),
+    @unittest.skipUnless(any(GCP_CREDS.values()),
                         'GCP environment credentials are not set')
    def test_08_gcp_valid_endpoint(self):
        master_key = {
@ -1204,7 +1225,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest):
            self.client_encryption_invalid.create_data_key(
                'gcp', master_key=master_key)

-    @unittest.skipUnless(all(GCP_CREDS.values()),
+    @unittest.skipUnless(any(GCP_CREDS.values()),
                         'GCP environment credentials are not set')
    def test_09_gcp_invalid_endpoint(self):
        master_key = {
@ -1288,7 +1309,7 @@ class AzureGCPEncryptionTestMixin(object):
 class TestAzureEncryption(AzureGCPEncryptionTestMixin,
                          EncryptionIntegrationTest):
    @classmethod
-    @unittest.skipUnless(all(AZURE_CREDS.values()),
+    @unittest.skipUnless(any(AZURE_CREDS.values()),
                         'Azure environment credentials are not set')
    def setUpClass(cls):
        cls.KMS_PROVIDER_MAP = {'azure': AZURE_CREDS}
@ -1314,7 +1335,7 @@ class TestAzureEncryption(AzureGCPEncryptionTestMixin,
 class TestGCPEncryption(AzureGCPEncryptionTestMixin,
                        EncryptionIntegrationTest):
    @classmethod
-    @unittest.skipUnless(all(GCP_CREDS.values()),
+    @unittest.skipUnless(any(GCP_CREDS.values()),
                         'GCP environment credentials are not set')
    def setUpClass(cls):
        cls.KMS_PROVIDER_MAP = {'gcp': GCP_CREDS}