From b37b146ac88fc9647c3effd6e031dbf5cbee3cf5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 30 Jun 2022 12:35:29 -0500 Subject: [PATCH] PYTHON-3053 Key Management API (#958) --- pymongo/encryption.py | 189 +++++++++++++++- setup.py | 2 +- test/__init__.py | 22 ++ .../spec/unified/addKeyAltName.json | 6 +- ... createDataKey-kms_providers-invalid.json} | 17 +- .../{createKey.json => createDataKey.json} | 22 +- .../spec/unified/deleteKey.json | 6 +- .../spec/unified/getKey.json | 6 +- .../spec/unified/getKeyByAltName.json | 6 +- .../spec/unified/getKeys.json | 6 +- .../spec/unified/removeKeyAltName.json | 196 ++++++++++++----- .../spec/unified/rewrapManyDataKey.json | 204 +++++++++++++----- test/test_encryption.py | 92 +++++--- ...ntEncryptionOpts-additionalProperties.json | 30 +++ ...ncryptionOpts-keyVaultClient-required.json | 23 ++ ...entEncryptionOpts-keyVaultClient-type.json | 29 +++ ...yptionOpts-keyVaultNamespace-required.json | 28 +++ ...EncryptionOpts-keyVaultNamespace-type.json | 29 +++ ...pts-kmsProviders-additionalProperties.json | 29 +++ ...kmsProviders-aws-additionalProperties.json | 31 +++ ...tEncryptionOpts-kmsProviders-aws-type.json | 29 +++ ...sProviders-azure-additionalProperties.json | 31 +++ ...ncryptionOpts-kmsProviders-azure-type.json | 29 +++ ...kmsProviders-gcp-additionalProperties.json | 31 +++ ...tEncryptionOpts-kmsProviders-gcp-type.json | 29 +++ ...msProviders-kmip-additionalProperties.json | 31 +++ ...EncryptionOpts-kmsProviders-kmip-type.json | 29 +++ ...sProviders-local-additionalProperties.json | 31 +++ ...ncryptionOpts-kmsProviders-local-type.json | 29 +++ ...tEncryptionOpts-kmsProviders-required.json | 26 +++ ...lientEncryptionOpts-kmsProviders-type.json | 27 +++ ...cryptionOpts-tlsOptions_not_supported.json | 30 +++ ...clientEncryption-additionalProperties.json | 30 +++ ...ryption-clientEncryptionOpts-required.json | 17 ++ ...tEncryption-clientEncryptionOpts-type.json | 18 ++ .../entity-clientEncryption-id-required.json | 28 +++ .../entity-clientEncryption-id-type.json | 29 +++ .../invalid/runOnRequirement-csfle-type.json | 15 ++ ...Providers-missing_aws_kms_credentials.json | 36 ++++ ...oviders-missing_azure_kms_credentials.json | 36 ++++ ...Providers-missing_gcp_kms_credentials.json | 36 ++++ .../valid-fail/kmsProviders-no_kms.json | 32 +++ .../valid-fail/operation-unsupported.json | 22 ++ .../collectionData-createOptions.json | 3 +- ...kmsProviders-explicit_kms_credentials.json | 52 +++++ ...Providers-mixed_kms_credential_fields.json | 54 +++++ ...Providers-placeholder_kms_credentials.json | 70 ++++++ .../kmsProviders-unconfigured_kms.json | 39 ++++ test/unified_format.py | 103 ++++++++- 49 files changed, 1780 insertions(+), 165 deletions(-) rename test/client-side-encryption/spec/unified/{createKey-kms_providers-invalid.json => createDataKey-kms_providers-invalid.json} (86%) rename test/client-side-encryption/spec/unified/{createKey.json => createDataKey.json} (97%) create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-required.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-required.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-additionalProperties.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-required.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-type.json create mode 100644 test/unified-test-format/invalid/clientEncryptionOpts-tlsOptions_not_supported.json create mode 100644 test/unified-test-format/invalid/entity-clientEncryption-additionalProperties.json create mode 100644 test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-required.json create mode 100644 test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-type.json create mode 100644 test/unified-test-format/invalid/entity-clientEncryption-id-required.json create mode 100644 test/unified-test-format/invalid/entity-clientEncryption-id-type.json create mode 100644 test/unified-test-format/invalid/runOnRequirement-csfle-type.json create mode 100644 test/unified-test-format/valid-fail/kmsProviders-missing_aws_kms_credentials.json create mode 100644 test/unified-test-format/valid-fail/kmsProviders-missing_azure_kms_credentials.json create mode 100644 test/unified-test-format/valid-fail/kmsProviders-missing_gcp_kms_credentials.json create mode 100644 test/unified-test-format/valid-fail/kmsProviders-no_kms.json create mode 100644 test/unified-test-format/valid-fail/operation-unsupported.json create mode 100644 test/unified-test-format/valid-pass/kmsProviders-explicit_kms_credentials.json create mode 100644 test/unified-test-format/valid-pass/kmsProviders-mixed_kms_credential_fields.json create mode 100644 test/unified-test-format/valid-pass/kmsProviders-placeholder_kms_credentials.json create mode 100644 test/unified-test-format/valid-pass/kmsProviders-unconfigured_kms.json diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 096090e4a..b792a4487 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -17,7 +17,6 @@ import contextlib import enum import socket -import uuid import weakref from typing import Any, Mapping, Optional, Sequence @@ -40,6 +39,7 @@ from bson.errors import BSONError from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument, _inflate_bson from bson.son import SON from pymongo import _csot +from pymongo.cursor import Cursor from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import AutoEncryptionOpts from pymongo.errors import ( @@ -50,8 +50,10 @@ from pymongo.errors import ( ) from pymongo.mongo_client import MongoClient from pymongo.network import BLOCKING_IO_ERRORS +from pymongo.operations import UpdateOne from pymongo.pool import PoolOptions, _configured_socket from pymongo.read_concern import ReadConcern +from pymongo.results import BulkWriteResult, DeleteResult from pymongo.ssl_support import get_ssl_context from pymongo.uri_parser import parse_host from pymongo.write_concern import WriteConcern @@ -60,10 +62,11 @@ _HTTPS_PORT = 443 _KMS_CONNECT_TIMEOUT = 10 # TODO: CDRIVER-3262 will define this value. _MONGOCRYPTD_TIMEOUT_MS = 10000 + _DATA_KEY_OPTS: CodecOptions = CodecOptions(document_class=SON, uuid_representation=STANDARD) # Use RawBSONDocument codec options to avoid needlessly decoding # documents from the key vault. -_KEY_VAULT_OPTS = CodecOptions(document_class=RawBSONDocument, uuid_representation=STANDARD) +_KEY_VAULT_OPTS = CodecOptions(document_class=RawBSONDocument) @contextlib.contextmanager @@ -225,11 +228,11 @@ class _EncryptionIO(MongoCryptCallback): # type: ignore """ raw_doc = RawBSONDocument(data_key, _KEY_VAULT_OPTS) data_key_id = raw_doc.get("_id") - if not isinstance(data_key_id, uuid.UUID): - raise TypeError("data_key _id must be a UUID") + if not isinstance(data_key_id, Binary) or data_key_id.subtype != UUID_SUBTYPE: + raise TypeError("data_key _id must be Binary with a UUID subtype") self.key_vault_coll.insert_one(raw_doc) - return Binary(data_key_id.bytes, subtype=UUID_SUBTYPE) + return data_key_id def bson_encode(self, doc): """Encode a document to BSON. @@ -256,6 +259,30 @@ class _EncryptionIO(MongoCryptCallback): # type: ignore self.mongocryptd_client = None +class RewrapManyDataKeyResult(object): + def __init__(self, bulk_write_result: Optional[BulkWriteResult] = None) -> None: + """Result object returned by a ``rewrap_many_data_key`` operation. + + :Parameters: + - `bulk_write_result`: The result of the bulk write operation used to + update the key vault collection with one or more rewrapped data keys. + If ``rewrap_many_data_key()`` does not find any matching keys to + rewrap, no bulk write operation will be executed and this field will + be ``None``. + """ + self._bulk_write_result = bulk_write_result + + @property + def bulk_write_result(self) -> Optional[BulkWriteResult]: + """The result of the bulk write operation used to update the key vault + collection with one or more rewrapped data keys. If + ``rewrap_many_data_key()`` does not find any matching keys to rewrap, + no bulk write operation will be executed and this field will be + ``None``. + """ + return self._bulk_write_result + + class _Encrypter(object): """Encrypts and decrypts MongoDB commands. @@ -514,12 +541,15 @@ class ClientEncryption(object): self._encryption = ExplicitEncrypter( self._io_callbacks, MongoCryptOptions(kms_providers, None) ) + # Use the same key vault collection as the callback. + self._key_vault_coll = self._io_callbacks.key_vault_coll def create_data_key( self, kms_provider: str, master_key: Optional[Mapping[str, Any]] = None, key_alt_names: Optional[Sequence[str]] = None, + key_material: Optional[bytes] = None, ) -> Binary: """Create and insert a new data key into the key vault collection. @@ -580,16 +610,24 @@ class ClientEncryption(object): # reference the key with the alternate name client_encryption.encrypt("457-55-5462", keyAltName="name1", algorithm=Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Random) + - `key_material` (optional): Sets the custom key material to be used + by the data key for encryption and decryption. :Returns: The ``_id`` of the created data key document as a :class:`~bson.binary.Binary` with subtype :data:`~bson.binary.UUID_SUBTYPE`. + + .. versionchanged:: 4.2 + Added the `key_material` parameter. """ self._check_closed() with _wrap_encryption_errors(): return self._encryption.create_data_key( - kms_provider, master_key=master_key, key_alt_names=key_alt_names + kms_provider, + master_key=master_key, + key_alt_names=key_alt_names, + key_material=key_material, ) def encrypt( @@ -676,6 +714,145 @@ class ClientEncryption(object): decrypted_doc = self._encryption.decrypt(doc) return decode(decrypted_doc, codec_options=self._codec_options)["v"] + def get_key(self, id: Binary) -> Optional[RawBSONDocument]: + """Get a data key by id. + + :Parameters: + - `id` (Binary): The UUID of a key a which must be a + :class:`~bson.binary.Binary` with subtype 4 ( + :attr:`~bson.binary.UUID_SUBTYPE`). + + :Returns: + The key document. + """ + self._check_closed() + return self._key_vault_coll.find_one({"_id": id}) + + def get_keys(self) -> Cursor[RawBSONDocument]: + """Get all of the data keys. + + :Returns: + An instance of :class:`~pymongo.cursor.Cursor` over the data key + documents. + """ + self._check_closed() + return self._key_vault_coll.find({}) + + def delete_key(self, id: Binary) -> DeleteResult: + """Delete a key document in the key vault collection that has the given ``key_id``. + + :Parameters: + - `id` (Binary): The UUID of a key a which must be a + :class:`~bson.binary.Binary` with subtype 4 ( + :attr:`~bson.binary.UUID_SUBTYPE`). + + :Returns: + The delete result. + """ + self._check_closed() + return self._key_vault_coll.delete_one({"_id": id}) + + def add_key_alt_name(self, id: Binary, key_alt_name: str) -> Any: + """Add ``key_alt_name`` to the set of alternate names in the key document with UUID ``key_id``. + + :Parameters: + - ``id``: The UUID of a key a which must be a + :class:`~bson.binary.Binary` with subtype 4 ( + :attr:`~bson.binary.UUID_SUBTYPE`). + - ``key_alt_name``: The key alternate name to add. + + :Returns: + The previous version of the key document. + """ + self._check_closed() + update = {"$addToSet": {"keyAltNames": key_alt_name}} + return self._key_vault_coll.find_one_and_update({"_id": id}, update) + + def get_key_by_alt_name(self, key_alt_name: str) -> Optional[RawBSONDocument]: + """Get a key document in the key vault collection that has the given ``key_alt_name``. + + :Parameters: + - `key_alt_name`: (str): The key alternate name of the key to get. + + :Returns: + The key document. + """ + self._check_closed() + return self._key_vault_coll.find_one({"keyAltNames": key_alt_name}) + + def remove_key_alt_name(self, id: Binary, key_alt_name: str) -> Optional[RawBSONDocument]: + """Remove ``key_alt_name`` from the set of keyAltNames in the key document with UUID ``id``. + + Also removes the ``keyAltNames`` field from the key document if it would otherwise be empty. + + :Parameters: + - ``id``: The UUID of a key a which must be a + :class:`~bson.binary.Binary` with subtype 4 ( + :attr:`~bson.binary.UUID_SUBTYPE`). + - ``key_alt_name``: The key alternate name to remove. + + :Returns: + Returns the previous version of the key document. + """ + self._check_closed() + pipeline = [ + { + "$set": { + "keyAltNames": { + "$cond": [ + {"$eq": ["$keyAltNames", [key_alt_name]]}, + "$$REMOVE", + { + "$filter": { + "input": "$keyAltNames", + "cond": {"$ne": ["$$this", key_alt_name]}, + } + }, + ] + } + } + } + ] + return self._key_vault_coll.find_one_and_update({"_id": id}, pipeline) + + def rewrap_many_data_key( + self, + filter: Mapping[str, Any], + provider: Optional[str] = None, + master_key: Optional[Mapping[str, Any]] = None, + ) -> RewrapManyDataKeyResult: + """Decrypts and encrypts all matching data keys in the key vault with a possibly new `master_key` value. + + :Parameters: + - `filter`: A document used to filter the data keys. + - `provider`: The new KMS provider to use to encrypt the data keys, + or ``None`` to use the current KMS provider(s). + - ``master_key``: The master key fields corresponding to the new KMS + provider when ``provider`` is not ``None``. + + :Returns: + A :class:`RewrapManyDataKeyResult`. + """ + self._check_closed() + with _wrap_encryption_errors(): + raw_result = self._encryption.rewrap_many_data_key(filter, provider, master_key) + if raw_result is None: + return RewrapManyDataKeyResult() + + raw_doc = RawBSONDocument(raw_result, DEFAULT_RAW_BSON_OPTIONS) + replacements = [] + for key in raw_doc["v"]: + update_model = { + "$set": {"keyMaterial": key["keyMaterial"], "masterKey": key["masterKey"]}, + "$currentDate": {"updateDate": True}, + } + op = UpdateOne({"_id": key["_id"]}, update_model) + replacements.append(op) + if not replacements: + return RewrapManyDataKeyResult() + result = self._key_vault_coll.bulk_write(replacements) + return RewrapManyDataKeyResult(result) + def __enter__(self) -> "ClientEncryption": return self diff --git a/setup.py b/setup.py index c6b32b9fb..a61f56c3f 100755 --- a/setup.py +++ b/setup.py @@ -277,7 +277,7 @@ if sys.platform in ("win32", "darwin"): extras_require = { "encryption": [ - "pymongocrypt@git+ssh://git@github.com/mongodb/libmongocrypt.git@pymongocrypt-1.3.0b0#subdirectory=bindings/python" + "pymongocrypt@git+ssh://git@github.com/mongodb/libmongocrypt.git@161dbc8ae#subdirectory=bindings/python" ], "ocsp": pyopenssl_reqs, "snappy": ["python-snappy"], diff --git a/test/__init__.py b/test/__init__.py index 64c812c11..4ecc3c9e9 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -15,6 +15,7 @@ """Test suite for pymongo, bson, and gridfs. """ +import base64 import gc import os import socket @@ -116,6 +117,27 @@ elif TEST_SERVERLESS: COMPRESSORS = COMPRESSORS or "zlib" +# Shared KMS data. +LOCAL_MASTER_KEY = base64.b64decode( + b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" + b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" +) +AWS_CREDS = { + "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), +} +AZURE_CREDS = { + "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), + "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), + "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), +} +GCP_CREDS = { + "email": os.environ.get("FLE_GCP_EMAIL", ""), + "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), +} +KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} + + def is_server_resolvable(): """Returns True if 'server' is resolvable.""" socket_timeout = socket.getdefaulttimeout() diff --git a/test/client-side-encryption/spec/unified/addKeyAltName.json b/test/client-side-encryption/spec/unified/addKeyAltName.json index 7dc371143..8b6c174cb 100644 --- a/test/client-side-encryption/spec/unified/addKeyAltName.json +++ b/test/client-side-encryption/spec/unified/addKeyAltName.json @@ -22,7 +22,11 @@ "keyVaultClient": "client0", "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "local": {} + "local": { + "key": { + "$$placeholder": 1 + } + } } } } diff --git a/test/client-side-encryption/spec/unified/createKey-kms_providers-invalid.json b/test/client-side-encryption/spec/unified/createDataKey-kms_providers-invalid.json similarity index 86% rename from test/client-side-encryption/spec/unified/createKey-kms_providers-invalid.json rename to test/client-side-encryption/spec/unified/createDataKey-kms_providers-invalid.json index b2c8d83e0..16cf6ca70 100644 --- a/test/client-side-encryption/spec/unified/createKey-kms_providers-invalid.json +++ b/test/client-side-encryption/spec/unified/createDataKey-kms_providers-invalid.json @@ -1,5 +1,5 @@ { - "description": "createKey-provider-invalid", + "description": "createDataKey-provider-invalid", "schemaVersion": "1.8", "runOnRequirements": [ { @@ -24,7 +24,14 @@ "keyVaultClient": "client0", "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "aws": {} + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + } + } } } } @@ -35,7 +42,7 @@ "description": "create data key without required master key fields", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "aws", @@ -59,7 +66,7 @@ "description": "create data key with invalid master key field", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", @@ -85,7 +92,7 @@ "description": "create data key with invalid master key", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "aws", diff --git a/test/client-side-encryption/spec/unified/createKey.json b/test/client-side-encryption/spec/unified/createDataKey.json similarity index 97% rename from test/client-side-encryption/spec/unified/createKey.json rename to test/client-side-encryption/spec/unified/createDataKey.json index adb3fff20..110c726f9 100644 --- a/test/client-side-encryption/spec/unified/createKey.json +++ b/test/client-side-encryption/spec/unified/createDataKey.json @@ -1,5 +1,5 @@ { - "description": "createKey", + "description": "createDataKey", "schemaVersion": "1.8", "runOnRequirements": [ { @@ -90,7 +90,7 @@ "description": "create data key with AWS KMS provider", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "aws", @@ -153,7 +153,7 @@ "description": "create datakey with Azure KMS provider", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "azure", @@ -216,7 +216,7 @@ "description": "create datakey with GCP KMS provider", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "gcp", @@ -283,7 +283,7 @@ "description": "create datakey with KMIP KMS provider", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "kmip" @@ -341,7 +341,7 @@ "description": "create datakey with local KMS provider", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local" @@ -396,7 +396,7 @@ "description": "create datakey with no keyAltName", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", @@ -457,7 +457,7 @@ "description": "create datakey with single keyAltName", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", @@ -520,7 +520,7 @@ "description": "create datakey with multiple keyAltNames", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", @@ -619,7 +619,7 @@ "description": "create datakey with custom key material", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", @@ -682,7 +682,7 @@ "description": "create datakey with invalid custom key material (too short)", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", diff --git a/test/client-side-encryption/spec/unified/deleteKey.json b/test/client-side-encryption/spec/unified/deleteKey.json index a3b2f98a5..3a10fb082 100644 --- a/test/client-side-encryption/spec/unified/deleteKey.json +++ b/test/client-side-encryption/spec/unified/deleteKey.json @@ -22,7 +22,11 @@ "keyVaultClient": "client0", "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "local": {} + "local": { + "key": { + "$$placeholder": 1 + } + } } } } diff --git a/test/client-side-encryption/spec/unified/getKey.json b/test/client-side-encryption/spec/unified/getKey.json index f2f2c6811..6a7269b2c 100644 --- a/test/client-side-encryption/spec/unified/getKey.json +++ b/test/client-side-encryption/spec/unified/getKey.json @@ -22,7 +22,11 @@ "keyVaultClient": "client0", "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "local": {} + "local": { + "key": { + "$$placeholder": 1 + } + } } } } diff --git a/test/client-side-encryption/spec/unified/getKeyByAltName.json b/test/client-side-encryption/spec/unified/getKeyByAltName.json index 18ed2e194..f94459bbd 100644 --- a/test/client-side-encryption/spec/unified/getKeyByAltName.json +++ b/test/client-side-encryption/spec/unified/getKeyByAltName.json @@ -22,7 +22,11 @@ "keyVaultClient": "client0", "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "local": {} + "local": { + "key": { + "$$placeholder": 1 + } + } } } } diff --git a/test/client-side-encryption/spec/unified/getKeys.json b/test/client-side-encryption/spec/unified/getKeys.json index bd07af380..d94471235 100644 --- a/test/client-side-encryption/spec/unified/getKeys.json +++ b/test/client-side-encryption/spec/unified/getKeys.json @@ -87,7 +87,7 @@ "description": "getKeys with single key documents", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local", @@ -160,7 +160,7 @@ "description": "getKeys with many key documents", "operations": [ { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local" @@ -170,7 +170,7 @@ } }, { - "name": "createKey", + "name": "createDataKey", "object": "clientEncryption0", "arguments": { "kmsProvider": "local" diff --git a/test/client-side-encryption/spec/unified/removeKeyAltName.json b/test/client-side-encryption/spec/unified/removeKeyAltName.json index f94d9b02d..bef13c87d 100644 --- a/test/client-side-encryption/spec/unified/removeKeyAltName.json +++ b/test/client-side-encryption/spec/unified/removeKeyAltName.json @@ -22,7 +22,11 @@ "keyVaultClient": "client0", "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "local": {} + "local": { + "key": { + "$$placeholder": 1 + } + } } } } @@ -118,11 +122,36 @@ } } }, - "update": { - "$pull": { - "keyAltNames": "does_not_exist" + "update": [ + { + "$set": { + "keyAltNames": { + "$cond": [ + { + "$eq": [ + "$keyAltNames", + [ + "does_not_exist" + ] + ] + }, + "$$REMOVE", + { + "$filter": { + "input": "$keyAltNames", + "cond": { + "$ne": [ + "$$this", + "does_not_exist" + ] + } + } + } + ] + } + } } - }, + ], "writeConcern": { "w": "majority" } @@ -239,11 +268,36 @@ } } }, - "update": { - "$pull": { - "keyAltNames": "does_not_exist" + "update": [ + { + "$set": { + "keyAltNames": { + "$cond": [ + { + "$eq": [ + "$keyAltNames", + [ + "does_not_exist" + ] + ] + }, + "$$REMOVE", + { + "$filter": { + "input": "$keyAltNames", + "cond": { + "$ne": [ + "$$this", + "does_not_exist" + ] + } + } + } + ] + } + } } - }, + ], "writeConcern": { "w": "majority" } @@ -378,11 +432,36 @@ } } }, - "update": { - "$pull": { - "keyAltNames": "alternate_name" + "update": [ + { + "$set": { + "keyAltNames": { + "$cond": [ + { + "$eq": [ + "$keyAltNames", + [ + "alternate_name" + ] + ] + }, + "$$REMOVE", + { + "$filter": { + "input": "$keyAltNames", + "cond": { + "$ne": [ + "$$this", + "alternate_name" + ] + } + } + } + ] + } + } } - }, + ], "writeConcern": { "w": "majority" } @@ -501,11 +580,36 @@ } } }, - "update": { - "$pull": { - "keyAltNames": "alternate_name" + "update": [ + { + "$set": { + "keyAltNames": { + "$cond": [ + { + "$eq": [ + "$keyAltNames", + [ + "alternate_name" + ] + ] + }, + "$$REMOVE", + { + "$filter": { + "input": "$keyAltNames", + "cond": { + "$ne": [ + "$$this", + "alternate_name" + ] + } + } + } + ] + } + } } - }, + ], "writeConcern": { "w": "majority" } @@ -525,42 +629,36 @@ } } }, - "update": { - "$pull": { - "keyAltNames": "local_key" - } - }, - "writeConcern": { - "w": "majority" - } - } - } - }, - { - "commandStartedEvent": { - "databaseName": "keyvault", - "command": { - "update": "datakeys", - "updates": [ + "update": [ { - "q": { - "_id": { - "$binary": { - "base64": "bG9jYWxrZXlsb2NhbGtleQ==", - "subType": "04" - } - } - }, - "u": { - "$unset": { - "keyAltNames": true + "$set": { + "keyAltNames": { + "$cond": [ + { + "$eq": [ + "$keyAltNames", + [ + "local_key" + ] + ] + }, + "$$REMOVE", + { + "$filter": { + "input": "$keyAltNames", + "cond": { + "$ne": [ + "$$this", + "local_key" + ] + } + } + } + ] } } } - ], - "writeConcern": { - "w": "majority" - } + ] } } } diff --git a/test/client-side-encryption/spec/unified/rewrapManyDataKey.json b/test/client-side-encryption/spec/unified/rewrapManyDataKey.json index ed7568ca4..7e3abb127 100644 --- a/test/client-side-encryption/spec/unified/rewrapManyDataKey.json +++ b/test/client-side-encryption/spec/unified/rewrapManyDataKey.json @@ -266,7 +266,9 @@ } }, "expectResult": { - "bulkWriteResult": {} + "bulkWriteResult": { + "$$exists": false + } } } ], @@ -372,8 +374,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -396,8 +402,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -420,8 +430,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -444,8 +458,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } } ], "writeConcern": { @@ -538,8 +556,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -562,8 +584,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -586,8 +612,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -610,8 +640,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } } ], "writeConcern": { @@ -708,8 +742,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -734,8 +772,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -760,8 +802,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -786,8 +832,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } } ], "writeConcern": { @@ -877,8 +927,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -902,8 +956,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -927,8 +985,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -952,8 +1014,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } } ], "writeConcern": { @@ -1040,8 +1106,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1062,8 +1132,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1084,8 +1158,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1106,8 +1184,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } } ], "writeConcern": { @@ -1262,8 +1344,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1284,8 +1370,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1306,8 +1396,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1328,8 +1422,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } }, { "q": { @@ -1350,8 +1448,12 @@ "updateDate": true } }, - "upsert": false, - "multi": false + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } } ], "writeConcern": { diff --git a/test/test_encryption.py b/test/test_encryption.py index 458dd68f3..c3ba61d6e 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -31,14 +31,20 @@ from pymongo.collection import Collection sys.path[0:0] = [""] from test import ( + AWS_CREDS, + AZURE_CREDS, CA_PEM, CLIENT_PEM, + GCP_CREDS, + KMIP_CREDS, + LOCAL_MASTER_KEY, IntegrationTest, PyMongoTestCase, client_context, unittest, ) from test.test_bulk import BulkTestBase +from test.unified_format import generate_test_classes from test.utils import ( AllowListEventListener, OvertCommandListener, @@ -64,6 +70,7 @@ from pymongo.errors import ( AutoReconnect, BulkWriteError, ConfigurationError, + DuplicateKeyError, EncryptionError, InvalidOperation, OperationFailure, @@ -74,14 +81,13 @@ from pymongo.mongo_client import MongoClient from pymongo.operations import InsertOne, ReplaceOne, UpdateOne from pymongo.write_concern import WriteConcern +KMS_PROVIDERS = {"local": {"key": b"\x00" * 96}} + def get_client_opts(client): return client._MongoClient__options -KMS_PROVIDERS = {"local": {"key": b"\x00" * 96}} - - class TestAutoEncryptionOpts(PyMongoTestCase): @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") @unittest.skipUnless(os.environ.get("TEST_CRYPT_SHARED"), "crypt_shared lib is not installed") @@ -211,7 +217,7 @@ class EncryptionIntegrationTest(IntegrationTest): # Location of JSON test files. BASE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-side-encryption") -SPEC_PATH = os.path.join(BASE, "spec", "legacy") +SPEC_PATH = os.path.join(BASE, "spec") OPTS = CodecOptions() @@ -547,11 +553,6 @@ class TestExplicitSimple(EncryptionIntegrationTest): # Spec tests -AWS_CREDS = { - "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), -} - AWS_TEMP_CREDS = { "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), @@ -562,19 +563,6 @@ AWS_TEMP_NO_SESSION_CREDS = { "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), } - -AZURE_CREDS = { - "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), - "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), - "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), -} - -GCP_CREDS = { - "email": os.environ.get("FLE_GCP_EMAIL", ""), - "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), -} - -KMIP = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} KMS_TLS_OPTS = {"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}} @@ -611,7 +599,7 @@ class TestSpec(SpecRunner): if not any(AZURE_CREDS.values()): self.skipTest("GCP environment credentials are not set") if "kmip" in kms_providers: - kms_providers["kmip"] = KMIP + kms_providers["kmip"] = KMIP_CREDS opts["kms_tls_options"] = KMS_TLS_OPTS if "key_vault_namespace" not in opts: opts["key_vault_namespace"] = "keyvault.datakeys" @@ -685,21 +673,24 @@ def create_test(scenario_def, test, name): return run_scenario -test_creator = TestCreator(create_test, TestSpec, SPEC_PATH) +test_creator = TestCreator(create_test, TestSpec, os.path.join(SPEC_PATH, "legacy")) test_creator.create_tests() -# Prose Tests -LOCAL_MASTER_KEY = base64.b64decode( - b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" - b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" -) +if _HAVE_PYMONGOCRYPT: + globals().update( + generate_test_classes( + os.path.join(SPEC_PATH, "unified"), + module=__name__, + ) + ) +# Prose Tests ALL_KMS_PROVIDERS = { "aws": AWS_CREDS, "azure": AZURE_CREDS, "gcp": GCP_CREDS, - "kmip": KMIP, + "kmip": KMIP_CREDS, "local": {"key": LOCAL_MASTER_KEY}, } @@ -1232,7 +1223,12 @@ class TestCustomEndpoint(EncryptionIntegrationTest): super(TestCustomEndpoint, cls).setUpClass() def setUp(self): - kms_providers = {"aws": AWS_CREDS, "azure": AZURE_CREDS, "gcp": GCP_CREDS, "kmip": KMIP} + kms_providers = { + "aws": AWS_CREDS, + "azure": AZURE_CREDS, + "gcp": GCP_CREDS, + "kmip": KMIP_CREDS, + } self.client_encryption = ClientEncryption( kms_providers=kms_providers, key_vault_namespace="keyvault.datakeys", @@ -1409,7 +1405,7 @@ class TestCustomEndpoint(EncryptionIntegrationTest): self.client_encryption_invalid.create_data_key("kmip", key) def test_11_kmip_master_key_endpoint(self): - key = {"keyId": "1", "endpoint": KMIP["endpoint"]} + key = {"keyId": "1", "endpoint": KMIP_CREDS["endpoint"]} self.run_test_expected_success("kmip", key) # Override invalid endpoint: data_key_id = self.client_encryption_invalid.create_data_key("kmip", master_key=key) @@ -2066,6 +2062,38 @@ class TestKmsTLSOptions(EncryptionIntegrationTest): self.client_encryption_invalid_hostname.create_data_key("kmip") +# https://github.com/mongodb/specifications/blob/50e26fe/source/client-side-encryption/tests/README.rst#unique-index-on-keyaltnames +class TestUniqueIndexOnKeyAltNamesProse(EncryptionIntegrationTest): + def setUp(self): + self.client = client_context.client + self.client.keyvault.drop_collection("datakeys") + self.client.keyvault.datakeys.create_index( + "keyAltNames", unique=True, partialFilterExpression={"keyAltNames": {"$exists": True}} + ) + kms_providers_map = {"local": {"key": LOCAL_MASTER_KEY}} + self.client_encryption = ClientEncryption( + kms_providers_map, "keyvault.datakeys", self.client, CodecOptions() + ) + self.def_key_id = self.client_encryption.create_data_key("local", key_alt_names=["def"]) + + def test_01_create_key(self): + self.client_encryption.create_data_key("local", key_alt_names=["abc"]) + with self.assertRaisesRegex(EncryptionError, "E11000 duplicate key error collection"): + self.client_encryption.create_data_key("local", key_alt_names=["abc"]) + with self.assertRaisesRegex(EncryptionError, "E11000 duplicate key error collection"): + self.client_encryption.create_data_key("local", key_alt_names=["def"]) + + def test_02_add_key_alt_name(self): + key_id = self.client_encryption.create_data_key("local") + self.client_encryption.add_key_alt_name(key_id, "abc") + key_doc = self.client_encryption.add_key_alt_name(key_id, "abc") + assert key_doc["keyAltNames"] == ["abc"] + with self.assertRaisesRegex(DuplicateKeyError, "E11000 duplicate key error collection"): + self.client_encryption.add_key_alt_name(key_id, "def") + key_doc = self.client_encryption.add_key_alt_name(self.def_key_id, "def") + assert key_doc["keyAltNames"] == ["def"] + + # https://github.com/mongodb/specifications/blob/d4c9432/source/client-side-encryption/tests/README.rst#explicit-encryption class TestExplicitQueryableEncryption(EncryptionIntegrationTest): @client_context.require_no_standalone diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-additionalProperties.json new file mode 100644 index 000000000..26d14051a --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-additionalProperties.json @@ -0,0 +1,30 @@ +{ + "description": "clientEncryptionOpts-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + }, + "invalid": {} + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-required.json b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-required.json new file mode 100644 index 000000000..c43a2a912 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-required.json @@ -0,0 +1,23 @@ +{ + "description": "clientEncryptionOpts-keyVaultClient-required", + "schemaVersion": "1.8", + "createEntities": [ + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-type.json new file mode 100644 index 000000000..1be9167a4 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultClient-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-keyVaultClient-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": 0, + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-required.json b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-required.json new file mode 100644 index 000000000..3f54d89aa --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-required.json @@ -0,0 +1,28 @@ +{ + "description": "clientEncryptionOpts-keyVaultNamespace-required", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "kmsProviders": { + "aws": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-type.json new file mode 100644 index 000000000..53f2f5f08 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-keyVaultNamespace-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-keyVaultNamespace-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": 0, + "kmsProviders": { + "aws": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-additionalProperties.json new file mode 100644 index 000000000..cfd979e2b --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-additionalProperties.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "invalid": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-additionalProperties.json new file mode 100644 index 000000000..59b273487 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-aws-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "invalid": {} + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-type.json new file mode 100644 index 000000000..ffcc85bfc --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-aws-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-aws-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": 0 + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-additionalProperties.json new file mode 100644 index 000000000..1664b7909 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-azure-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "azure": { + "invalid": {} + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-type.json new file mode 100644 index 000000000..5bd50c807 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-azure-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "azure": 0 + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-additionalProperties.json new file mode 100644 index 000000000..120c088b0 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-gcp-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "gcp": { + "invalid": {} + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-type.json new file mode 100644 index 000000000..1dd1c8a2a --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-gcp-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "gcp": 0 + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-additionalProperties.json new file mode 100644 index 000000000..22ded2044 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-kmip-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "kmip": { + "invalid": {} + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-type.json new file mode 100644 index 000000000..9b9e74be3 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-kmip-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-kmip-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "kmip": 0 + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-additionalProperties.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-additionalProperties.json new file mode 100644 index 000000000..b93cfe00d --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-local-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "invalid": {} + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-type.json new file mode 100644 index 000000000..526ea2483 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-local-type.json @@ -0,0 +1,29 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-local-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": 0 + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-required.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-required.json new file mode 100644 index 000000000..b823a67ba --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-required.json @@ -0,0 +1,26 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-required", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys" + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-type.json new file mode 100644 index 000000000..e7a6190b6 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-type.json @@ -0,0 +1,27 @@ +{ + "description": "clientEncryptionOpts-kmsProviders-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": 0 + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-tlsOptions_not_supported.json b/test/unified-test-format/invalid/clientEncryptionOpts-tlsOptions_not_supported.json new file mode 100644 index 000000000..3b4972f23 --- /dev/null +++ b/test/unified-test-format/invalid/clientEncryptionOpts-tlsOptions_not_supported.json @@ -0,0 +1,30 @@ +{ + "description": "clientEncryptionOpts-tlsOptions_not_supported", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + }, + "tlsOptions": {} + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/entity-clientEncryption-additionalProperties.json b/test/unified-test-format/invalid/entity-clientEncryption-additionalProperties.json new file mode 100644 index 000000000..77c0a9143 --- /dev/null +++ b/test/unified-test-format/invalid/entity-clientEncryption-additionalProperties.json @@ -0,0 +1,30 @@ +{ + "description": "entity-clientEncryption-additionalProperties", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + } + }, + "invalid": {} + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-required.json b/test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-required.json new file mode 100644 index 000000000..88e852342 --- /dev/null +++ b/test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-required.json @@ -0,0 +1,17 @@ +{ + "description": "entity-clientEncryption-clientEncryptionOpts-required", + "schemaVersion": "1.8", + "createEntities": [ + { + "clientEncryption": { + "id": "clientEncryption0" + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-type.json b/test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-type.json new file mode 100644 index 000000000..77fb6a362 --- /dev/null +++ b/test/unified-test-format/invalid/entity-clientEncryption-clientEncryptionOpts-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-clientEncryption-clientEncryptionOpts-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": 0 + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/entity-clientEncryption-id-required.json b/test/unified-test-format/invalid/entity-clientEncryption-id-required.json new file mode 100644 index 000000000..464ba7159 --- /dev/null +++ b/test/unified-test-format/invalid/entity-clientEncryption-id-required.json @@ -0,0 +1,28 @@ +{ + "description": "entity-clientEncryption-id-required", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/entity-clientEncryption-id-type.json b/test/unified-test-format/invalid/entity-clientEncryption-id-type.json new file mode 100644 index 000000000..a7746657f --- /dev/null +++ b/test/unified-test-format/invalid/entity-clientEncryption-id-type.json @@ -0,0 +1,29 @@ +{ + "description": "entity-clientEncryption-id-type", + "schemaVersion": "1.8", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": 0, + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/runOnRequirement-csfle-type.json b/test/unified-test-format/invalid/runOnRequirement-csfle-type.json new file mode 100644 index 000000000..b48c850d1 --- /dev/null +++ b/test/unified-test-format/invalid/runOnRequirement-csfle-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-csfle-type", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": "foo" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/kmsProviders-missing_aws_kms_credentials.json b/test/unified-test-format/valid-fail/kmsProviders-missing_aws_kms_credentials.json new file mode 100644 index 000000000..e62de8003 --- /dev/null +++ b/test/unified-test-format/valid-fail/kmsProviders-missing_aws_kms_credentials.json @@ -0,0 +1,36 @@ +{ + "description": "kmsProviders-missing_aws_kms_credentials", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": "accessKeyId" + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/kmsProviders-missing_azure_kms_credentials.json b/test/unified-test-format/valid-fail/kmsProviders-missing_azure_kms_credentials.json new file mode 100644 index 000000000..8ef805d0f --- /dev/null +++ b/test/unified-test-format/valid-fail/kmsProviders-missing_azure_kms_credentials.json @@ -0,0 +1,36 @@ +{ + "description": "kmsProviders-missing_azure_kms_credentials", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "azure": { + "tenantId": "tenantId" + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/kmsProviders-missing_gcp_kms_credentials.json b/test/unified-test-format/valid-fail/kmsProviders-missing_gcp_kms_credentials.json new file mode 100644 index 000000000..c6da1ce58 --- /dev/null +++ b/test/unified-test-format/valid-fail/kmsProviders-missing_gcp_kms_credentials.json @@ -0,0 +1,36 @@ +{ + "description": "kmsProviders-missing_gcp_kms_credentials", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "gcp": { + "email": "email" + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/kmsProviders-no_kms.json b/test/unified-test-format/valid-fail/kmsProviders-no_kms.json new file mode 100644 index 000000000..57499b4ea --- /dev/null +++ b/test/unified-test-format/valid-fail/kmsProviders-no_kms.json @@ -0,0 +1,32 @@ +{ + "description": "clientEncryptionOpts-no_kms", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": {} + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/operation-unsupported.json b/test/unified-test-format/valid-fail/operation-unsupported.json new file mode 100644 index 000000000..d8ef5ab1c --- /dev/null +++ b/test/unified-test-format/valid-fail/operation-unsupported.json @@ -0,0 +1,22 @@ +{ + "description": "operation-unsupported", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "Unsupported operation", + "operations": [ + { + "name": "unsupportedOperation", + "object": "client0" + } + ] + } + ] +} diff --git a/test/unified-test-format/valid-pass/collectionData-createOptions.json b/test/unified-test-format/valid-pass/collectionData-createOptions.json index 07ab66baa..df3321a55 100644 --- a/test/unified-test-format/valid-pass/collectionData-createOptions.json +++ b/test/unified-test-format/valid-pass/collectionData-createOptions.json @@ -3,7 +3,8 @@ "schemaVersion": "1.9", "runOnRequirements": [ { - "minServerVersion": "3.6" + "minServerVersion": "3.6", + "serverless": "forbid" } ], "createEntities": [ diff --git a/test/unified-test-format/valid-pass/kmsProviders-explicit_kms_credentials.json b/test/unified-test-format/valid-pass/kmsProviders-explicit_kms_credentials.json new file mode 100644 index 000000000..7cc74939e --- /dev/null +++ b/test/unified-test-format/valid-pass/kmsProviders-explicit_kms_credentials.json @@ -0,0 +1,52 @@ +{ + "description": "kmsProviders-explicit_kms_credentials", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": "accessKeyId", + "secretAccessKey": "secretAccessKey" + }, + "azure": { + "tenantId": "tenantId", + "clientId": "clientId", + "clientSecret": "clientSecret" + }, + "gcp": { + "email": "email", + "privateKey": "cHJpdmF0ZUtleQo=" + }, + "kmip": { + "endpoint": "endpoint" + }, + "local": { + "key": "a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5a2V5" + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-pass/kmsProviders-mixed_kms_credential_fields.json b/test/unified-test-format/valid-pass/kmsProviders-mixed_kms_credential_fields.json new file mode 100644 index 000000000..363f2a457 --- /dev/null +++ b/test/unified-test-format/valid-pass/kmsProviders-mixed_kms_credential_fields.json @@ -0,0 +1,54 @@ +{ + "description": "kmsProviders-mixed_kms_credential_fields", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": "accessKeyId", + "secretAccessKey": { + "$$placeholder": 1 + } + }, + "azure": { + "tenantId": "tenantId", + "clientId": { + "$$placeholder": 1 + }, + "clientSecret": { + "$$placeholder": 1 + } + }, + "gcp": { + "email": "email", + "privateKey": { + "$$placeholder": 1 + } + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-pass/kmsProviders-placeholder_kms_credentials.json b/test/unified-test-format/valid-pass/kmsProviders-placeholder_kms_credentials.json new file mode 100644 index 000000000..3f7721f01 --- /dev/null +++ b/test/unified-test-format/valid-pass/kmsProviders-placeholder_kms_credentials.json @@ -0,0 +1,70 @@ +{ + "description": "kmsProviders-placeholder_kms_credentials", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + } + }, + "azure": { + "tenantId": { + "$$placeholder": 1 + }, + "clientId": { + "$$placeholder": 1 + }, + "clientSecret": { + "$$placeholder": 1 + } + }, + "gcp": { + "email": { + "$$placeholder": 1 + }, + "privateKey": { + "$$placeholder": 1 + } + }, + "kmip": { + "endpoint": { + "$$placeholder": 1 + } + }, + "local": { + "key": { + "$$placeholder": 1 + } + } + } + } + } + } + ], + "tests": [ + { + "description": "", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-pass/kmsProviders-unconfigured_kms.json b/test/unified-test-format/valid-pass/kmsProviders-unconfigured_kms.json new file mode 100644 index 000000000..12ca58094 --- /dev/null +++ b/test/unified-test-format/valid-pass/kmsProviders-unconfigured_kms.json @@ -0,0 +1,39 @@ +{ + "description": "kmsProviders-unconfigured_kms", + "schemaVersion": "1.8", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {}, + "azure": {}, + "gcp": {}, + "kmip": {}, + "local": {} + } + } + } + } + ], + "tests": [ + { + "description": "", + "skipReason": "DRIVERS-2280: waiting on driver support for on-demand credentials", + "operations": [] + } + ] +} diff --git a/test/unified_format.py b/test/unified_format.py index 001af4434..a7d8b533d 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -26,7 +26,18 @@ import sys import time import types from collections import abc -from test import IntegrationTest, client_context, unittest +from test import ( + AWS_CREDS, + AZURE_CREDS, + CA_PEM, + CLIENT_PEM, + GCP_CREDS, + KMIP_CREDS, + LOCAL_MASTER_KEY, + IntegrationTest, + client_context, + unittest, +) from test.utils import ( CMAPListener, camel_to_snake, @@ -45,6 +56,7 @@ from typing import Any import pymongo from bson import SON, Code, DBRef, Decimal128, Int64, MaxKey, MinKey, json_util from bson.binary import Binary +from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.objectid import ObjectId from bson.regex import RE_TYPE, Regex from gridfs import GridFSBucket @@ -53,10 +65,13 @@ from pymongo.change_stream import ChangeStream from pymongo.client_session import ClientSession, TransactionOptions, _TxnState from pymongo.collection import Collection from pymongo.database import Database +from pymongo.encryption import ClientEncryption +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.errors import ( BulkWriteError, ConfigurationError, ConnectionFailure, + EncryptionError, ExecutionTimeout, InvalidOperation, NetworkTimeout, @@ -93,6 +108,27 @@ JSON_OPTS = json_util.JSONOptions(tz_aware=False) IS_INTERRUPTED = False +KMS_TLS_OPTS = { + "kmip": { + "tlsCAFile": CA_PEM, + "tlsCertificateKeyFile": CLIENT_PEM, + } +} + + +# Build up a placeholder map. +PLACEHOLDER_MAP = dict() +for (provider_name, provider_data) in [ + ("local", {"key": LOCAL_MASTER_KEY}), + ("aws", AWS_CREDS), + ("azure", AZURE_CREDS), + ("gcp", GCP_CREDS), + ("kmip", KMIP_CREDS), +]: + for (key, value) in provider_data.items(): + placeholder = f"/clientEncryptionOpts/kmsProviders/{provider_name}/{key}" + PLACEHOLDER_MAP[placeholder] = value + def interrupt_loop(): global IS_INTERRUPTED @@ -169,6 +205,12 @@ def is_run_on_requirement_satisfied(requirement): else: auth_satisfied = not client_context.auth_enabled + csfle_satisfied = True + req_csfle = requirement.get("csfle") + if req_csfle is True: + min_version_satisfied = Version.from_string("4.2") <= server_version + csfle_satisfied = _HAVE_PYMONGOCRYPT and min_version_satisfied + return ( topology_satisfied and min_version_satisfied @@ -176,6 +218,7 @@ def is_run_on_requirement_satisfied(requirement): and serverless_satisfied and params_satisfied and auth_satisfied + and csfle_satisfied ) @@ -328,6 +371,19 @@ class EntityMapUtil(object): self._entities[key] = value + def _handle_placeholders(self, spec: dict, current: dict, path: str) -> Any: + if "$$placeholder" in current: + if path not in PLACEHOLDER_MAP: + raise ValueError(f"Could not find a placeholder value for {path}") + return PLACEHOLDER_MAP[path] + + for key in list(current): + value = current[key] + if isinstance(value, dict): + subpath = f"{path}/{key}" + current[key] = self._handle_placeholders(spec, value, subpath) + return current + def _create_entity(self, entity_spec, uri=None): if len(entity_spec) != 1: self.test.fail( @@ -335,6 +391,7 @@ class EntityMapUtil(object): ) entity_type, spec = next(iter(entity_spec.items())) + spec = self._handle_placeholders(spec, spec, "") if entity_type == "client": kwargs: dict = {} observe_events = spec.get("observeEvents", []) @@ -410,6 +467,19 @@ class EntityMapUtil(object): elif entity_type == "bucket": # TODO: implement the 'bucket' entity type self.test.skipTest("GridFS is not currently supported (PYTHON-2459)") + elif entity_type == "clientEncryption": + opts = camel_to_snake_args(spec["clientEncryptionOpts"].copy()) + if isinstance(opts["key_vault_client"], str): + opts["key_vault_client"] = self[opts["key_vault_client"]] + self[spec["id"]] = ClientEncryption( + opts["kms_providers"], + opts["key_vault_namespace"], + opts["key_vault_client"], + DEFAULT_CODEC_OPTIONS, + opts.get("kms_tls_options", KMS_TLS_OPTS), + ) + return + self.test.fail("Unable to create entity of unknown type %s" % (entity_type,)) def create_entities_from_spec(self, entity_spec, uri=None): @@ -872,7 +942,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): # Connection errors are considered client errors. if isinstance(exception, ConnectionFailure): self.assertNotIsInstance(exception, NotPrimaryError) - elif isinstance(exception, (InvalidOperation, ConfigurationError)): + elif isinstance(exception, (InvalidOperation, ConfigurationError, EncryptionError)): pass else: self.assertNotIsInstance(exception, PyMongoError) @@ -1033,6 +1103,33 @@ class UnifiedSpecTestMixinV1(IntegrationTest): self.__raise_if_unsupported("close", target, NonLazyCursor) return target.close() + def _clientEncryptionOperation_createDataKey(self, target, *args, **kwargs): + if "opts" in kwargs: + opts = kwargs.pop("opts") + kwargs["master_key"] = opts.get("masterKey") + kwargs["key_alt_names"] = opts.get("keyAltNames") + kwargs["key_material"] = opts.get("keyMaterial") + return target.create_data_key(*args, **kwargs) + + def _clientEncryptionOperation_getKeys(self, target, *args, **kwargs): + return list(target.get_keys(*args, **kwargs)) + + def _clientEncryptionOperation_deleteKey(self, target, *args, **kwargs): + result = target.delete_key(*args, **kwargs) + response = result.raw_result + response["deletedCount"] = result.deleted_count + return response + + def _clientEncryptionOperation_rewrapManyDataKey(self, target, *args, **kwargs): + if "opts" in kwargs: + opts = kwargs.pop("opts") + kwargs["provider"] = opts.get("provider") + kwargs["master_key"] = opts.get("masterKey") + data = target.rewrap_many_data_key(*args, **kwargs) + if data.bulk_write_result: + return dict(bulkWriteResult=parse_bulk_write_result(data.bulk_write_result)) + return dict() + def run_entity_operation(self, spec): target = self.entity_map[spec["object"]] client = target @@ -1075,6 +1172,8 @@ class UnifiedSpecTestMixinV1(IntegrationTest): client = target._client elif isinstance(target, GridFSBucket): raise NotImplementedError + elif isinstance(target, ClientEncryption): + method_name = "_clientEncryptionOperation_%s" % (opname,) else: method_name = "doesNotExist"