MOTOR-866 Key Management API (#175)

2022-09-10 16:42:33 -07:00 · 2022-09-10 16:42:33 -07:00 · 99e65f85c5
commit 99e65f85c5
parent 6e8e2ee0e1
8 changed files with 142 additions and 7 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -328,13 +328,75 @@ functions:
          sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh

  "run tox":
+    # If testing FLE, start the KMS mock servers, first create the virtualenv.
+    - command: shell.exec
+      params:
+        script: |
+          ${PREPARE_SHELL}
+          cd ${DRIVERS_TOOLS}/.evergreen/csfle
+          . ./activate_venv.sh
+    # Run in the background so the mock servers don't block the EVG task.
+    - command: shell.exec
+      params:
+        background: true
+        script: |
+          ${PREPARE_SHELL}
+          cd ${DRIVERS_TOOLS}/.evergreen/csfle
+          . ./activate_venv.sh
+          # The -u options forces the stdout and stderr streams to be unbuffered.
+          # TMPDIR is required to avoid "AF_UNIX path too long" errors.
+          TMPDIR="$(dirname $DRIVERS_TOOLS)" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem  --port 5698 &
+          python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
+          python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+          python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
+    # Wait up to 10 seconds for the KMIP server to start.
+    - command: shell.exec
+      params:
+        script: |
+          ${PREPARE_SHELL}
+          cd ${DRIVERS_TOOLS}/.evergreen/csfle
+          . ./activate_venv.sh
+          for i in $(seq 1 1 10); do
+             sleep 1
+             if python -u kms_kmip_client.py; then
+                echo 'KMS KMIP server started!'
+                exit 0
+             fi
+          done
+          echo 'Failed to start KMIP server!'
+          exit 1
+    - command: shell.exec
+      type: test
+      params:
+        silent: true
+        working_dir: "src"
+        script: |
+          cat <<EOT > fle_creds.sh
+          export FLE_AWS_KEY="${fle_aws_key}"
+          export FLE_AWS_SECRET="${fle_aws_secret}"
+          export FLE_AZURE_CLIENTID="${fle_azure_clientid}"
+          export FLE_AZURE_TENANTID="${fle_azure_tenantid}"
+          export FLE_AZURE_CLIENTSECRET="${fle_azure_clientsecret}"
+          export FLE_GCP_EMAIL="${fle_gcp_email}"
+          export FLE_GCP_PRIVATEKEY="${fle_gcp_privatekey}"
+          # Needed for generating temporary aws credentials.
+          export AWS_ACCESS_KEY_ID="${fle_aws_key}"
+          export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}"
+          export AWS_DEFAULT_REGION=us-east-1
+          EOT
    - command: shell.exec
      type: test
      params:
        working_dir: "src"
        script: |
          ${PREPARE_SHELL}
-
+          # Disable xtrace (just in case it was accidentally set).
+          set +x
+          . ./fle_creds.sh
+          rm -f ./fle_creds.sh
+          set -x
+          export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
+          export TEST_ENCRYPTION=1
          PYTHON_BINARY="${PYTHON_BINARY}" \
          TOX_BINARY="${TOX_BINARY}" \
          TOX_ENV="${TOX_ENV}" \
--- a/.gitignore
+++ b/.gitignore
@ -12,4 +12,5 @@ setup.cfg
 doc/_build/
 .idea/
 xunit-results
+xunit-synchro-results
 .eggs
--- a/motor/core.py
+++ b/motor/core.py
@ -1952,8 +1952,22 @@ class AgnosticClientEncryption(AgnosticBase):
    decrypt = AsyncCommand()
    close = AsyncCommand(doc=docstrings.close_doc)

+    # Key Management API
+    rewrap_many_data_key = AsyncCommand()
+    delete_key = AsyncCommand()
+    get_key = AsyncCommand()
+    add_key_alt_name = AsyncCommand()
+    get_key_by_alt_name = AsyncCommand()
+    remove_key_alt_name = AsyncCommand()
+
    def __init__(
-        self, kms_providers, key_vault_namespace, key_vault_client, codec_options, io_loop=None
+        self,
+        kms_providers,
+        key_vault_namespace,
+        key_vault_client,
+        codec_options,
+        io_loop=None,
+        kms_tls_options=None,
    ):
        """Explicit client-side field level encryption.

@ -1970,7 +1984,7 @@ class AgnosticClientEncryption(AgnosticBase):
            io_loop = None
        sync_client = key_vault_client.delegate
        delegate = self.__delegate_class__(
-            kms_providers, key_vault_namespace, sync_client, codec_options
+            kms_providers, key_vault_namespace, sync_client, codec_options, kms_tls_options
        )
        super().__init__(delegate)
        self._io_loop = io_loop
@ -1996,3 +2010,7 @@ class AgnosticClientEncryption(AgnosticBase):

    def __exit__(self, exc_type, exc_val, exc_tb):
        pass
+
+    async def get_keys(self):
+        cursor_class = create_class_with_framework(AgnosticCursor, self._framework, self.__module__)
+        return cursor_class(self.delegate.get_keys(), self)
--- a/synchro/init.py
+++ b/synchro/init.py
@ -768,9 +768,20 @@ class AutoEncryptionOpts(encryption_options.AutoEncryptionOpts):
 class ClientEncryption(Synchro):
    __delegate_class__ = motor.MotorClientEncryption

-    def __init__(self, kms_providers, key_vault_namespace, key_vault_client, codec_options):
+    def __init__(
+        self,
+        kms_providers,
+        key_vault_namespace,
+        key_vault_client,
+        codec_options,
+        kms_tls_options=None,
+    ):
        self.delegate = motor.MotorClientEncryption(
-            kms_providers, key_vault_namespace, key_vault_client.delegate, codec_options
+            kms_providers,
+            key_vault_namespace,
+            key_vault_client.delegate,
+            codec_options,
+            kms_tls_options=kms_tls_options,
        )

    def __enter__(self):
@ -778,3 +789,6 @@ class ClientEncryption(Synchro):

    def __exit__(self, *args):
        return self.synchronize(self.delegate.__aexit__)(*args)
+
+    def get_keys(self):
+        return Cursor(self.synchronize(self.delegate.get_keys)())
--- a/synchro/synchrotest.py
+++ b/synchro/synchrotest.py
@ -170,8 +170,18 @@ excluded_tests = [
    "*.test_md5",
    # Causes a deadlock.
    "TestFork.*",
-    # Also causes a deadlock
+    # Also causes a deadlock.
    "TestClientSimple.test_fork",
+    # These methods are picked up by nose despite not being a unittest.
+    "TestRewrapWithSeparateClientEncryption.run_test",
+    "TestCustomEndpoint.run_test_expected_success",
+    "TestDataKeyDoubleEncryption.run_test",
+    # Motor does not support CSOT.
+    "TestCsotGridfsFind.*",
+    # These tests are failing right now.
+    "TestUnifiedFindShutdownError.test_Concurrent_shutdown_error_on_find",
+    "TestUnifiedInsertShutdownError.test_Concurrent_shutdown_error_on_insert",
+    "TestUnifiedPoolClearedError.test_PoolClearedError_does_not_mark_server_unknown",
 ]


--- a/test/asyncio_tests/test_asyncio_cursor.py
+++ b/test/asyncio_tests/test_asyncio_cursor.py
@ -584,6 +584,18 @@ class TestAsyncIOCursor(AsyncIOMockServerTestCase):
            await contrast_cursor.close()
            self.assertTrue(contrast_cursor.closed)

+    @asyncio_test
+    async def test_generate_keys(self):
+        c = self.collection
+        KMS_PROVIDERS = {"local": {"key": b"\x00" * 96}}
+
+        async with motor_asyncio.AsyncIOMotorClientEncryption(
+            KMS_PROVIDERS, "keyvault.datakeys", c, bson.codec_options.CodecOptions()
+        ) as client_encryption:
+            self.assertIsInstance(
+                await client_encryption.get_keys(), motor_asyncio.AsyncIOMotorCursor
+            )
+

 class TestAsyncIOCursorMaxTimeMS(AsyncIOTestCase):
    def setUp(self):
--- a/test/tornado_tests/test_motor_cursor.py
+++ b/test/tornado_tests/test_motor_cursor.py
@ -516,6 +516,16 @@ class MotorCursorTest(MotorMockServerTest):
            lst = await method().batch_size(2).to_list(length=1)
            self.assertEqual([{"_id": 0}, {"_id": 1}], bson.decode_all(lst[0]))

+    @gen_test
+    async def test_generate_keys(self):
+        c = self.collection
+        KMS_PROVIDERS = {"local": {"key": b"\x00" * 96}}
+
+        async with motor.MotorClientEncryption(
+            KMS_PROVIDERS, "keyvault.datakeys", c, bson.codec_options.CodecOptions()
+        ) as client_encryption:
+            self.assertIsInstance(await client_encryption.get_keys(), motor.MotorCursor)
+

 class MotorCursorMaxTimeMSTest(MotorTest):
    def setUp(self):
--- a/tox.ini
+++ b/tox.ini
@ -43,6 +43,14 @@ passenv =
    DB_PASSWORD
    CERT_DIR
    ASYNC_TEST_TIMEOUT
+    FLE_AWS_KEY
+    FLE_AWS_SECRET
+    FLE_AZURE_CLIENTID
+    FLE_AZURE_TENANTID
+    FLE_AZURE_CLIENTSECRET
+    FLE_GCP_EMAIL
+    FLE_GCP_PRIVATEKEY
+

 basepython =
    py37,synchro37: {env:PYTHON_BINARY:python3.7}
@ -111,7 +119,7 @@ setenv =
    PYTHONPATH = {envtmpdir}/mongo-python-driver
 commands =
    git clone --depth 1 --branch master https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
-    pip install -e {envtmpdir}/mongo-python-driver
+    python3 -m pip install -e {envtmpdir}/mongo-python-driver
    python3 -m synchro.synchrotest --with-xunit --xunit-file=xunit-synchro-results -v -w {envtmpdir}/mongo-python-driver {posargs}

 [testenv:lint]