mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-4580 Add key_expiration_ms option for DEK cache lifetime (#2186)

Browse Source
This commit is contained in:
Shane Harvey 2025-03-10 17:11:38 -07:00 committed by GitHub
parent 61d435408e
commit 7ef18af49b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 549 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/changelog.rst
View File

@ -1,7 +1,23 @@
Changelog
=========
Changes in Version 4.11.2 (YYYY/MM/DD)
Changes in Version 4.12.0 (YYYY/MM/DD)
--------------------------------------
PyMongo 4.12 brings a number of changes including:
- Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to
:class:`~pymongo.encryption_options.AutoEncryptionOpts`.
Issues Resolved
...............
See the `PyMongo 4.12 release notes in JIRA`_ for the list of resolved issues
in this release.
.. _PyMongo 4.12 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=41916
Changes in Version 4.11.2 (2025/03/05)
--------------------------------------
Version 4.11.2 is a bug fix release.

26
pymongo/asynchronous/encryption.py
View File

@ -445,6 +445,7 @@ class _Encrypter:
bypass_encryption=opts._bypass_auto_encryption,
encrypted_fields_map=encrypted_fields_map,
bypass_query_analysis=opts._bypass_query_analysis,
key_expiration_ms=opts._key_expiration_ms,
),
)
self._closed = False
@ -547,11 +548,10 @@ class QueryType(str, enum.Enum):
def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions:
opts = MongoCryptOptions(**kwargs)
# Opt into range V2 encryption.
if hasattr(opts, "enable_range_v2"):
opts.enable_range_v2 = True
return opts
# For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms.
if kwargs.get("key_expiration_ms") is None:
kwargs.pop("key_expiration_ms", None)
return MongoCryptOptions(**kwargs)
class AsyncClientEncryption(Generic[_DocumentType]):
@ -564,6 +564,7 @@ class AsyncClientEncryption(Generic[_DocumentType]):
key_vault_client: AsyncMongoClient[_DocumentTypeArg],
codec_options: CodecOptions[_DocumentTypeArg],
kms_tls_options: Optional[Mapping[str, Any]] = None,
key_expiration_ms: Optional[int] = None,
) -> None:
"""Explicit client-side field level encryption.
@ -630,7 +631,12 @@ class AsyncClientEncryption(Generic[_DocumentType]):
Or to supply a client certificate::
kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}}
:param key_expiration_ms: The cache expiration time for data encryption keys.
Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000.
Set to 0 to disable key expiration.
.. versionchanged:: 4.12
Added the `key_expiration_ms` parameter.
.. versionchanged:: 4.0
Added the `kms_tls_options` parameter and the "kmip" KMS provider.
@ -666,14 +672,19 @@ class AsyncClientEncryption(Generic[_DocumentType]):
key_vault_coll = key_vault_client[db][coll]
opts = AutoEncryptionOpts(
kms_providers, key_vault_namespace, kms_tls_options=kms_tls_options
kms_providers,
key_vault_namespace,
kms_tls_options=kms_tls_options,
key_expiration_ms=key_expiration_ms,
)
self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO(
None, key_vault_coll, None, opts
)
self._encryption = AsyncExplicitEncrypter(
self._io_callbacks,
_create_mongocrypt_options(kms_providers=kms_providers, schema_map=None),
_create_mongocrypt_options(
kms_providers=kms_providers, schema_map=None, key_expiration_ms=key_expiration_ms
),
)
# Use the same key vault collection as the callback.
assert self._io_callbacks.key_vault_coll is not None
@ -700,6 +711,7 @@ class AsyncClientEncryption(Generic[_DocumentType]):
creation. :class:`~pymongo.errors.EncryptionError` will be
raised if the collection already exists.
:param database: the database to create the collection
:param name: the name of the collection to create
:param encrypted_fields: Document that describes the encrypted fields for
Queryable Encryption. The "keyId" may be set to ``None`` to auto-generate the data keys. For example:

10
pymongo/encryption_options.py
View File

@ -57,6 +57,7 @@ class AutoEncryptionOpts:
crypt_shared_lib_required: bool = False,
bypass_query_analysis: bool = False,
encrypted_fields_map: Optional[Mapping[str, Any]] = None,
key_expiration_ms: Optional[int] = None,
) -> None:
"""Options to configure automatic client-side field level encryption.
@ -191,9 +192,14 @@ class AutoEncryptionOpts:
]
}
}
:param key_expiration_ms: The cache expiration time for data encryption keys.
Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000.
Set to 0 to disable key expiration.
.. versionchanged:: 4.12
Added the `key_expiration_ms` parameter.
.. versionchanged:: 4.2
Added `encrypted_fields_map` `crypt_shared_lib_path`, `crypt_shared_lib_required`,
Added the `encrypted_fields_map`, `crypt_shared_lib_path`, `crypt_shared_lib_required`,
and `bypass_query_analysis` parameters.
.. versionchanged:: 4.0
@ -210,7 +216,6 @@ class AutoEncryptionOpts:
if encrypted_fields_map:
validate_is_mapping("encrypted_fields_map", encrypted_fields_map)
self._encrypted_fields_map = encrypted_fields_map
self._bypass_query_analysis = bypass_query_analysis
self._crypt_shared_lib_path = crypt_shared_lib_path
self._crypt_shared_lib_required = crypt_shared_lib_required
self._kms_providers = kms_providers
@ -233,6 +238,7 @@ class AutoEncryptionOpts:
# Maps KMS provider name to a SSLContext.
self._kms_ssl_contexts = _parse_kms_tls_options(kms_tls_options)
self._bypass_query_analysis = bypass_query_analysis
self._key_expiration_ms = key_expiration_ms
class RangeOpts:

26
pymongo/synchronous/encryption.py
View File

@ -442,6 +442,7 @@ class _Encrypter:
bypass_encryption=opts._bypass_auto_encryption,
encrypted_fields_map=encrypted_fields_map,
bypass_query_analysis=opts._bypass_query_analysis,
key_expiration_ms=opts._key_expiration_ms,
),
)
self._closed = False
@ -544,11 +545,10 @@ class QueryType(str, enum.Enum):
def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions:
opts = MongoCryptOptions(**kwargs)
# Opt into range V2 encryption.
if hasattr(opts, "enable_range_v2"):
opts.enable_range_v2 = True
return opts
# For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms.
if kwargs.get("key_expiration_ms") is None:
kwargs.pop("key_expiration_ms", None)
return MongoCryptOptions(**kwargs)
class ClientEncryption(Generic[_DocumentType]):
@ -561,6 +561,7 @@ class ClientEncryption(Generic[_DocumentType]):
key_vault_client: MongoClient[_DocumentTypeArg],
codec_options: CodecOptions[_DocumentTypeArg],
kms_tls_options: Optional[Mapping[str, Any]] = None,
key_expiration_ms: Optional[int] = None,
) -> None:
"""Explicit client-side field level encryption.
@ -627,7 +628,12 @@ class ClientEncryption(Generic[_DocumentType]):
Or to supply a client certificate::
kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}}
:param key_expiration_ms: The cache expiration time for data encryption keys.
Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000.
Set to 0 to disable key expiration.
.. versionchanged:: 4.12
Added the `key_expiration_ms` parameter.
.. versionchanged:: 4.0
Added the `kms_tls_options` parameter and the "kmip" KMS provider.
@ -659,14 +665,19 @@ class ClientEncryption(Generic[_DocumentType]):
key_vault_coll = key_vault_client[db][coll]
opts = AutoEncryptionOpts(
kms_providers, key_vault_namespace, kms_tls_options=kms_tls_options
kms_providers,
key_vault_namespace,
kms_tls_options=kms_tls_options,
key_expiration_ms=key_expiration_ms,
)
self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO(
None, key_vault_coll, None, opts
)
self._encryption = ExplicitEncrypter(
self._io_callbacks,
_create_mongocrypt_options(kms_providers=kms_providers, schema_map=None),
_create_mongocrypt_options(
kms_providers=kms_providers, schema_map=None, key_expiration_ms=key_expiration_ms
),
)
# Use the same key vault collection as the callback.
assert self._io_callbacks.key_vault_coll is not None
@ -693,6 +704,7 @@ class ClientEncryption(Generic[_DocumentType]):
creation. :class:`~pymongo.errors.EncryptionError` will be
raised if the collection already exists.
:param database: the database to create the collection
:param name: the name of the collection to create
:param encrypted_fields: Document that describes the encrypted fields for
Queryable Encryption. The "keyId" may be set to ``None`` to auto-generate the data keys. For example:

3
test/asynchronous/unified_format.py
View File

@ -378,6 +378,7 @@ class EntityMapUtil:
opts["key_vault_client"],
DEFAULT_CODEC_OPTIONS,
opts.get("kms_tls_options", kms_tls_options),
opts.get("key_expiration_ms"),
)
return
elif entity_type == "thread":
@ -439,7 +440,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest):
a class attribute ``TEST_SPEC``.
"""
SCHEMA_VERSION = Version.from_string("1.21")
SCHEMA_VERSION = Version.from_string("1.22")
RUN_ON_LOAD_BALANCER = True
RUN_ON_SERVERLESS = True
TEST_SPEC: Any

5
test/asynchronous/utils_spec_runner.py
View File

@ -18,6 +18,7 @@ from __future__ import annotations
import asyncio
import functools
import os
import time
import unittest
from asyncio import iscoroutinefunction
from collections import abc
@ -314,6 +315,10 @@ class AsyncSpecRunner(AsyncIntegrationTest):
coll = self.client[database][collection]
self.assertNotIn(index, [doc["name"] async for doc in await coll.list_indexes()])
async def wait(self, ms):
"""Run the "wait" test operation."""
await asyncio.sleep(ms / 1000.0)
def assertErrorLabelsContain(self, exc, expected_labels):
labels = [l for l in expected_labels if exc.has_error_label(l)]
self.assertEqual(labels, expected_labels)

3
test/client-side-encryption/spec/legacy/fle2v2-Rangev2-Compact.json
View File

@ -6,8 +6,7 @@
"replicaset",
"sharded",
"load-balanced"
],
"serverless": "forbid"
]
}
],
"database_name": "default",

270
test/client-side-encryption/spec/legacy/keyCache.json Normal file
View File

@ -0,0 +1,270 @@
{
"runOn": [
{
"minServerVersion": "4.1.10"
}
],
"database_name": "default",
"collection_name": "default",
"data": [],
"json_schema": {
"properties": {
"encrypted_w_altname": {
"encrypt": {
"keyId": "/altname",
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
}
},
"encrypted_string": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
},
"random": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
}
},
"encrypted_string_equivalent": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"
},
"key_vault_data": [
{
"status": 1,
"_id": {
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
},
"masterKey": {
"provider": "aws",
"key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
"region": "us-east-1"
},
"updateDate": {
"$date": {
"$numberLong": "1552949630483"
}
},
"keyMaterial": {
"$binary": {
"base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1552949630483"
}
},
"keyAltNames": [
"altname",
"another_altname"
]
}
],
"tests": [
{
"description": "Insert with deterministic encryption, then find it",
"clientOptions": {
"autoEncryptOpts": {
"kmsProviders": {
"aws": {}
},
"keyExpirationMS": 1
}
},
"operations": [
{
"name": "insertOne",
"arguments": {
"document": {
"_id": 1,
"encrypted_string": "string0"
}
}
},
{
"name": "wait",
"object": "testRunner",
"arguments": {
"ms": 50
}
},
{
"name": "find",
"arguments": {
"filter": {
"_id": 1
}
},
"result": [
{
"_id": 1,
"encrypted_string": "string0"
}
]
}
],
"expectations": [
{
"command_started_event": {
"command": {
"listCollections": 1,
"filter": {
"name": "default"
}
},
"command_name": "listCollections"
}
},
{
"command_started_event": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault",
"readConcern": {
"level": "majority"
}
},
"command_name": "find"
}
},
{
"command_started_event": {
"command": {
"insert": "default",
"documents": [
{
"_id": 1,
"encrypted_string": {
"$binary": {
"base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==",
"subType": "06"
}
}
}
],
"ordered": true
},
"command_name": "insert"
}
},
{
"command_started_event": {
"command": {
"find": "default",
"filter": {
"_id": 1
}
},
"command_name": "find"
}
},
{
"command_started_event": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault",
"readConcern": {
"level": "majority"
}
},
"command_name": "find"
}
}
],
"outcome": {
"collection": {
"data": [
{
"_id": 1,
"encrypted_string": {
"$binary": {
"base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==",
"subType": "06"
}
}
}
]
}
}
}
]
}

4
test/client-side-encryption/spec/legacy/timeoutMS.json
View File

@ -110,7 +110,7 @@
"listCollections"
],
"blockConnection": true,
"blockTimeMS": 600
"blockTimeMS": 60
}
},
"clientOptions": {
@ -119,7 +119,7 @@
"aws": {}
}
},
"timeoutMS": 500
"timeoutMS": 50
},
"operations": [
{

198
test/client-side-encryption/spec/unified/keyCache.json Normal file
View File

@ -0,0 +1,198 @@
{
"description": "keyCache-explicit",
"schemaVersion": "1.22",
"runOnRequirements": [
{
"csfle": true
}
],
"createEntities": [
{
"client": {
"id": "client0",
"observeEvents": [
"commandStartedEvent"
]
}
},
{
"clientEncryption": {
"id": "clientEncryption0",
"clientEncryptionOpts": {
"keyVaultClient": "client0",
"keyVaultNamespace": "keyvault.datakeys",
"kmsProviders": {
"local": {
"key": "OCTP9uKPPmvuqpHlqq83gPk4U6rUPxKVRRyVtrjFmVjdoa4Xzm1SzUbr7aIhNI42czkUBmrCtZKF31eaaJnxEBkqf0RFukA9Mo3NEHQWgAQ2cn9duOcRbaFUQo2z0/rB"
}
},
"keyExpirationMS": 1
}
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "keyvault"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "datakeys"
}
}
],
"initialData": [
{
"databaseName": "keyvault",
"collectionName": "datakeys",
"documents": [
{
"_id": {
"$binary": {
"base64": "a+YWzdygTAG62/cNUkqZiQ==",
"subType": "04"
}
},
"keyAltNames": [],
"keyMaterial": {
"$binary": {
"base64": "iocBkhO3YBokiJ+FtxDTS71/qKXQ7tSWhWbcnFTXBcMjarsepvALeJ5li+SdUd9ePuatjidxAdMo7vh1V2ZESLMkQWdpPJ9PaJjA67gKQKbbbB4Ik5F2uKjULvrMBnFNVRMup4JNUwWFQJpqbfMveXnUVcD06+pUpAkml/f+DSXrV3e5rxciiNVtz03dAG8wJrsKsFXWj6vTjFhsfknyBA==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1552949630483"
}
},
"updateDate": {
"$date": {
"$numberLong": "1552949630483"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "local"
}
}
]
}
],
"tests": [
{
"description": "decrypt, wait, and decrypt again",
"operations": [
{
"name": "decrypt",
"object": "clientEncryption0",
"arguments": {
"value": {
"$binary": {
"base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==",
"subType": "06"
}
}
},
"expectResult": "foobar"
},
{
"name": "wait",
"object": "testRunner",
"arguments": {
"ms": 50
}
},
{
"name": "decrypt",
"object": "clientEncryption0",
"arguments": {
"value": {
"$binary": {
"base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==",
"subType": "06"
}
}
},
"expectResult": "foobar"
}
],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "a+YWzdygTAG62/cNUkqZiQ==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault",
"readConcern": {
"level": "majority"
}
}
}
},
{
"commandStartedEvent": {
"command": {
"find": "datakeys",
"filter": {
"$or": [
{
"_id": {
"$in": [
{
"$binary": {
"base64": "a+YWzdygTAG62/cNUkqZiQ==",
"subType": "04"
}
}
]
}
},
{
"keyAltNames": {
"$in": []
}
}
]
},
"$db": "keyvault",
"readConcern": {
"level": "majority"
}
}
}
}
]
}
]
}
]
}

3
test/unified_format.py
View File

@ -377,6 +377,7 @@ class EntityMapUtil:
opts["key_vault_client"],
DEFAULT_CODEC_OPTIONS,
opts.get("kms_tls_options", kms_tls_options),
opts.get("key_expiration_ms"),
)
return
elif entity_type == "thread":
@ -438,7 +439,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
a class attribute ``TEST_SPEC``.
"""
SCHEMA_VERSION = Version.from_string("1.21")
SCHEMA_VERSION = Version.from_string("1.22")
RUN_ON_LOAD_BALANCER = True
RUN_ON_SERVERLESS = True
TEST_SPEC: Any

5
test/utils_spec_runner.py
View File

@ -18,6 +18,7 @@ from __future__ import annotations
import asyncio
import functools
import os
import time
import unittest
from asyncio import iscoroutinefunction
from collections import abc
@ -314,6 +315,10 @@ class SpecRunner(IntegrationTest):
coll = self.client[database][collection]
self.assertNotIn(index, [doc["name"] for doc in coll.list_indexes()])
def wait(self, ms):
"""Run the "wait" test operation."""
time.sleep(ms / 1000.0)
def assertErrorLabelsContain(self, exc, expected_labels):
labels = [l for l in expected_labels if exc.has_error_label(l)]
self.assertEqual(labels, expected_labels)

2
uv.lock generated
View File

@ -1133,7 +1133,7 @@ wheels = [
[[package]]
name = "pymongocrypt"
version = "1.13.0.dev0"
source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#90476d5db7737bab2ce1c198df5671a12dbaae1a" }
source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1e96c283162aa7789cf01f99f211e0ace8e6d49f" }
dependencies = [
{ name = "cffi" },
{ name = "cryptography" },