mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:mongodb/mongo-python-driver

Browse Source
This commit is contained in:
Steven Silvester 2025-03-22 11:32:33 -05:00
commit 63c37399df
No known key found for this signature in database
GPG Key ID: B1BF5EC3A8B32F91
31 changed files with 1020 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
.evergreen/config.yml
View File

@ -42,7 +42,7 @@ functions:
# Make an evergreen expansion file with dynamic values
- command: subprocess.exec
params:
include_expansions_in_env: ["is_patch", "project", "version_id", "skip_web_identity_auth_test", "skip_ECS_auth_test"]
include_expansions_in_env: ["is_patch", "project", "version_id"]
binary: bash
working_dir: "src"
args:
@ -213,16 +213,14 @@ functions:
params:
file: ${DRIVERS_TOOLS}/mo-expansion.yml
"run doctests":
- command: subprocess.exec
type: test
params:
include_expansions_in_env: [ "PYTHON_BINARY" ]
working_dir: "src"
binary: bash
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-doctests.sh
"run just script":
- command: subprocess.exec
type: test
params:
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
binary: bash
working_dir: "src"
args: [.evergreen/just.sh, "${JUSTFILE_TARGET}"]
"run tests":
- command: subprocess.exec
@ -248,7 +246,6 @@ functions:
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/cleanup.sh
"teardown system":
@ -268,27 +265,7 @@ functions:
- command: ec2.assume_role
params:
role_arn: ${aws_test_secrets_role}
"setup atlas":
- command: subprocess.exec
params:
binary: bash
include_expansions_in_env: ["task_id", "execution"]
env:
MONGODB_VERSION: "7.0"
LAMBDA_STACK_NAME: dbx-python-lambda
args:
- ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
- command: expansions.update
params:
file: atlas-expansion.yml
"teardown atlas":
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
duration_seconds: 3600
"attach benchmark test results":
- command: attach.results
@ -314,31 +291,6 @@ post:
- func: "upload test results"
- func: "cleanup"
task_groups:
- name: test_aws_lambda_task_group
setup_group:
- func: fetch source
- func: setup system
- func: setup atlas
teardown_task:
- func: teardown atlas
setup_group_can_fail_task: true
setup_group_timeout_secs: 1800
tasks:
- test-aws-lambda-deployed
- name: test_atlas_task_group_search_indexes
setup_group:
- func: fetch source
- func: setup system
- func: setup atlas
teardown_task:
- func: teardown atlas
setup_group_can_fail_task: true
setup_group_timeout_secs: 1800
tasks:
- test-search-index-helpers
tasks:
# Wildcard task. Do you need to find out what tools are available and where?
# Throw it here, and execute this task on all buildvariants
@ -350,63 +302,7 @@ tasks:
params:
args:
- src/.evergreen/scripts/run-getdata.sh
# Standard test tasks {{{
- name: "mockupdb"
tags: ["mockupdb"]
commands:
- func: "run tests"
vars:
TEST_NAME: mockupdb
- name: "doctests"
tags: ["doctests"]
commands:
- func: "run server"
- func: "run doctests"
- name: "test-search-index-helpers"
commands:
- func: "run server"
vars:
VERSION: "6.0"
TOPOLOGY: "replica_set"
- func: "run tests"
vars:
TEST_NAME: index_management
AUTH: "auth"
- name: "no-server"
tags: ["no-server"]
commands:
- func: "run tests"
- name: "free-threading"
tags: ["free-threading"]
commands:
- func: "run server"
vars:
VERSION: "8.0"
TOPOLOGY: "replica_set"
- func: "run tests"
- name: "test-aws-lambda-deployed"
commands:
- command: ec2.assume_role
params:
role_arn: ${LAMBDA_AWS_ROLE_ARN}
duration_seconds: 3600
- command: subprocess.exec
params:
working_dir: src
binary: bash
add_expansions_to_env: true
args:
- .evergreen/run-deployed-lambda-aws-tests.sh
env:
TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/test/lambda
# }}}
- name: "coverage-report"
tags: ["coverage"]
depends_on:
@ -456,12 +352,6 @@ tasks:
- ${github_commit}
buildvariants:
- name: "no-server"
display_name: "No server"
run_on:
- rhel84-small
tasks:
- name: "no-server"
- name: "Coverage Report"
display_name: "Coverage Report"
@ -482,12 +372,6 @@ buildvariants:
batchtime: 10080 # 7 days
- name: test-azurekms-fail
- name: rhel8-test-lambda
display_name: FaaS Lambda
run_on: rhel87-small
tasks:
- name: test_aws_lambda_task_group
- name: rhel8-import-time
display_name: Import Time
run_on: rhel87-small

54
.evergreen/generated_configs/tasks.yml
View File

@ -23,6 +23,15 @@ tasks:
TEST_NAME: data_lake
tags: [atlas_data_lake]
# Aws lambda tests
- name: test-aws-lambda-deployed
commands:
- func: assume ec2 role
- func: run tests
vars:
TEST_NAME: aws_lambda
tags: [aws_lambda]
# Aws tests
- name: test-auth-aws-4.4-regular
commands:
@ -704,6 +713,15 @@ tasks:
AWS_ROLE_SESSION_NAME: test
tags: [auth-aws, auth-aws-web-identity]
# Doctest tests
- name: test-doctests
commands:
- func: run server
- func: run just script
vars:
JUSTFILE_TARGET: docs-test
tags: [doctests]
# Enterprise auth tests
- name: test-enterprise-auth
commands:
@ -718,6 +736,16 @@ tasks:
AUTH: auth
tags: [enterprise_auth]
# Free threading tests
- name: test-free-threading
commands:
- func: run server
vars:
VERSION: "8.0"
TOPOLOGY: replica_set
- func: run tests
tags: [free-threading]
# Kms tests
- name: test-gcpkms
commands:
@ -790,6 +818,14 @@ tasks:
TEST_NAME: load_balancer
tags: [load-balancer, noauth, nossl]
# Mockupdb tests
- name: test-mockupdb
commands:
- func: run tests
vars:
TEST_NAME: mockupdb
tags: [mockupdb]
# Mod wsgi tests
- name: mod-wsgi-standalone
commands:
@ -832,6 +868,12 @@ tasks:
SUB_TEST_NAME: embedded
tags: [mod_wsgi]
# No server tests
- name: test-no-server
commands:
- func: run tests
tags: [no-server]
# Ocsp tests
- name: test-ocsp-ecdsa-valid-cert-server-does-not-staple
commands:
@ -1220,6 +1262,18 @@ tasks:
- func: send dashboard data
tags: [perf]
# Search index tests
- name: test-search-index-helpers
commands:
- func: assume ec2 role
- func: run server
vars:
TEST_NAME: search_index
- func: run tests
vars:
TEST_NAME: search_index
tags: [search_index]
# Server tests
- name: test-4.0-standalone-auth-ssl-sync
commands:

22
.evergreen/generated_configs/variants.yml
View File

@ -132,6 +132,14 @@ buildvariants:
expansions:
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
# Aws lambda tests
- name: faas-lambda
tasks:
- name: .aws_lambda
display_name: FaaS Lambda
run_on:
- rhel87-small
# Compression tests
- name: compression-snappy-rhel8-python3.9-no-c
tasks:
@ -234,7 +242,7 @@ buildvariants:
# Doctests tests
- name: doctests-rhel8-python3.9
tasks:
- name: doctests
- name: .doctests
display_name: Doctests RHEL8 Python3.9
run_on:
- rhel87-small
@ -664,7 +672,7 @@ buildvariants:
# Mockupdb tests
- name: mockupdb-rhel8-python3.9
tasks:
- name: mockupdb
- name: .mockupdb
display_name: MockupDB RHEL8 Python3.9
run_on:
- rhel87-small
@ -738,6 +746,14 @@ buildvariants:
NO_EXT: "1"
PYTHON_BINARY: /opt/python/3.13/bin/python3
# No server tests
- name: no-server
tasks:
- name: .no-server
display_name: No server
run_on:
- rhel87-small
# Ocsp tests
- name: ocsp-rhel8-v4.4-python3.9
tasks:
@ -986,7 +1002,7 @@ buildvariants:
# Search index tests
- name: search-index-helpers-rhel8-python3.9
tasks:
- name: test_atlas_task_group_search_indexes
- name: .search_index
display_name: Search Index Helpers RHEL8 Python3.9
run_on:
- rhel87-small

10
.evergreen/run-deployed-lambda-aws-tests.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
export PATH="/opt/python/3.9/bin:${PATH}"
python --version
pushd ./test/lambda
. build.sh
popd
. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh

5
.evergreen/run-tests.sh
View File

@ -34,7 +34,10 @@ fi
uv sync ${UV_ARGS} --reinstall
uv pip list
# Ensure we go back to base environment after the test.
trap "uv sync" EXIT HUP
# Start the test runner.
uv run .evergreen/scripts/run_tests.py "$@"
uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@"
popd

8
.evergreen/scripts/cleanup.sh
View File

@ -1,4 +1,12 @@
#!/bin/bash
HERE=$(dirname ${BASH_SOURCE:-$0})
# Try to source the env file.
if [ -f $HERE/env.sh ]; then
echo "Sourcing env file"
source $HERE/env.sh
fi
rm -rf "${DRIVERS_TOOLS}" || true
rm -f ./secrets-export.sh || true

2
.evergreen/scripts/configure-env.sh
View File

@ -60,8 +60,6 @@ export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
export MONGODB_BINARIES="$MONGODB_BINARIES"
export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
export CARGO_HOME="$CARGO_HOME"
export UV_TOOL_DIR="$UV_TOOL_DIR"

68
.evergreen/scripts/generate_config.py
View File

@ -672,7 +672,7 @@ def create_search_index_variants():
python = CPYTHONS[0]
return [
create_variant(
["test_atlas_task_group_search_indexes"],
[".search_index"],
get_display_name("Search Index Helpers", host, python=python),
python=python,
host=host,
@ -685,7 +685,7 @@ def create_mockupdb_variants():
python = CPYTHONS[0]
return [
create_variant(
["mockupdb"],
[".mockupdb"],
get_display_name("MockupDB", host, python=python),
python=python,
host=host,
@ -698,7 +698,7 @@ def create_doctests_variants():
python = CPYTHONS[0]
return [
create_variant(
["doctests"],
[".doctests"],
get_display_name("Doctests", host, python=python),
python=python,
host=host,
@ -748,6 +748,11 @@ def create_aws_auth_variants():
return variants
def create_no_server_variants():
host = HOSTS["rhel8"]
return [create_variant([".no-server"], "No server", host=host)]
def create_alternative_hosts_variants():
batchtime = BATCHTIME_WEEK
variants = []
@ -779,6 +784,11 @@ def create_alternative_hosts_variants():
return variants
def create_aws_lambda_variants():
host = HOSTS["rhel8"]
return [create_variant([".aws_lambda"], display_name="FaaS Lambda", host=host)]
##############
# Tasks
##############
@ -927,6 +937,27 @@ def _create_ocsp_task(algo, variant, server_type, base_task_name):
return EvgTask(name=task_name, tags=tags, commands=commands)
def create_aws_lambda_tasks():
assume_func = FunctionCall(func="assume ec2 role")
vars = dict(TEST_NAME="aws_lambda")
test_func = FunctionCall(func="run tests", vars=vars)
task_name = "test-aws-lambda-deployed"
tags = ["aws_lambda"]
commands = [assume_func, test_func]
return [EvgTask(name=task_name, tags=tags, commands=commands)]
def create_search_index_tasks():
assume_func = FunctionCall(func="assume ec2 role")
server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index"))
vars = dict(TEST_NAME="search_index")
test_func = FunctionCall(func="run tests", vars=vars)
task_name = "test-search-index-helpers"
tags = ["search_index"]
commands = [assume_func, server_func, test_func]
return [EvgTask(name=task_name, tags=tags, commands=commands)]
def create_atlas_connect_tasks():
vars = dict(TEST_NAME="atlas_connect")
assume_func = FunctionCall(func="assume ec2 role")
@ -1014,6 +1045,37 @@ def create_ocsp_tasks():
return tasks
def create_mockupdb_tasks():
test_func = FunctionCall(func="run tests", vars=dict(TEST_NAME="mockupdb"))
task_name = "test-mockupdb"
tags = ["mockupdb"]
return [EvgTask(name=task_name, tags=tags, commands=[test_func])]
def create_doctest_tasks():
server_func = FunctionCall(func="run server")
test_func = FunctionCall(func="run just script", vars=dict(JUSTFILE_TARGET="docs-test"))
task_name = "test-doctests"
tags = ["doctests"]
return [EvgTask(name=task_name, tags=tags, commands=[server_func, test_func])]
def create_no_server_tasks():
test_func = FunctionCall(func="run tests")
task_name = "test-no-server"
tags = ["no-server"]
return [EvgTask(name=task_name, tags=tags, commands=[test_func])]
def create_free_threading_tasks():
vars = dict(VERSION="8.0", TOPOLOGY="replica_set")
server_func = FunctionCall(func="run server", vars=vars)
test_func = FunctionCall(func="run tests")
task_name = "test-free-threading"
tags = ["free-threading"]
return [EvgTask(name=task_name, tags=tags, commands=[server_func, test_func])]
def create_serverless_tasks():
vars = dict(TEST_NAME="serverless", AUTH="auth", SSL="ssl")
test_func = FunctionCall(func="run tests", vars=vars)

21
.evergreen/scripts/run-with-env.sh
View File

@ -1,21 +0,0 @@
#!/bin/bash -eu
# Example use: bash run-with-env.sh run-tests.sh {args...}
# Parameter expansion to get just the current directory's name
if [ "${PWD##*/}" == "src" ]; then
. .evergreen/scripts/env.sh
if [ -f ".evergreen/scripts/test-env.sh" ]; then
. .evergreen/scripts/test-env.sh
fi
else
. src/.evergreen/scripts/env.sh
if [ -f "src/.evergreen/scripts/test-env.sh" ]; then
. src/.evergreen/scripts/test-env.sh
fi
fi
set -eu
# shellcheck source=/dev/null
. "$@"

20
.evergreen/scripts/run_server.py
View File

@ -22,14 +22,6 @@ def start_server():
if "VERSION" in os.environ:
os.environ["MONGODB_VERSION"] = os.environ["VERSION"]
if opts.auth:
extra_opts.append("--auth")
if opts.verbose:
extra_opts.append("-v")
elif opts.quiet:
extra_opts.append("-q")
if test_name == "auth_aws":
set_env("AUTH_AWS")
@ -51,6 +43,10 @@ def start_server():
if not found:
raise ValueError("Please provide an orchestration file")
elif test_name == "search_index":
os.environ["TOPOLOGY"] = "replica_set"
os.environ["MONGODB_VERSION"] = "7.0"
if not os.environ.get("TEST_CRYPT_SHARED"):
set_env("SKIP_CRYPT_SHARED")
@ -62,6 +58,14 @@ def start_server():
set_env("TLS_PEM_KEY_FILE", certs / "server.pem")
set_env("TLS_CA_FILE", certs / "ca.pem")
if opts.auth:
extra_opts.append("--auth")
if opts.verbose:
extra_opts.append("-v")
elif opts.quiet:
extra_opts.append("-q")
cmd = ["bash", f"{DRIVERS_TOOLS}/.evergreen/run-orchestration.sh", *extra_opts]
run_command(cmd, cwd=DRIVERS_TOOLS)

44
.evergreen/scripts/run_tests.py
View File

@ -4,8 +4,11 @@ import json
import logging
import os
import platform
import shutil
import sys
from datetime import datetime
from pathlib import Path
from shutil import which
import pytest
from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
@ -81,6 +84,42 @@ def handle_pymongocrypt() -> None:
LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})")
def handle_aws_lambda() -> None:
env = os.environ.copy()
target_dir = ROOT / "test/lambda"
env["TEST_LAMBDA_DIRECTORY"] = str(target_dir)
env.setdefault("AWS_REGION", "us-east-1")
dirs = ["pymongo", "gridfs", "bson"]
# Store the original .so files.
before_sos = []
for dname in dirs:
before_sos.extend(f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so"))
# Build the c extensions.
docker = which("docker") or which("podman")
if not docker:
raise ValueError("Could not find docker!")
image = "quay.io/pypa/manylinux2014_x86_64:latest"
run_command(
f'{docker} run --rm -v "{ROOT}:/src" --platform linux/amd64 {image} /src/test/lambda/build_internal.sh'
)
for dname in dirs:
target = ROOT / "test/lambda/mongodb" / dname
shutil.rmtree(target, ignore_errors=True)
shutil.copytree(ROOT / dname, target)
# Remove the original so files from the lambda directory.
for so_path in before_sos:
(ROOT / "test/lambda/mongodb" / so_path).unlink()
# Remove the new so files from the ROOT directory.
for dname in dirs:
so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")]
for so_path in list(so_paths):
if so_path not in before_sos:
Path(so_path).unlink()
script_name = "run-deployed-lambda-aws-tests.sh"
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/aws_lambda/{script_name}", env=env)
def run() -> None:
# Handle green framework first so they can patch modules.
if GREEN_FRAMEWORK:
@ -129,6 +168,11 @@ def run() -> None:
test_oidc_send_to_remote(SUB_TEST_NAME)
return
# Run deployed aws lambda tests.
if TEST_NAME == "aws_lambda":
handle_aws_lambda()
return
if os.environ.get("DEBUG_LOG"):
TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG} -o log_cli=1".split())

27
.evergreen/scripts/setup_tests.py
View File

@ -128,8 +128,7 @@ def handle_test_env() -> None:
TEST_ARGS = ""
# Start compiling the args we'll pass to uv.
# Run in an isolated environment so as not to pollute the base venv.
UV_ARGS = ["--isolated --extra test"]
UV_ARGS = ["--extra test --no-group dev"]
test_title = test_name
if sub_test_name:
@ -175,6 +174,28 @@ def handle_test_env() -> None:
if not config:
AUTH = "noauth"
if test_name in ["aws_lambda", "search_index"]:
env = os.environ.copy()
env["MONGODB_VERSION"] = "7.0"
env["LAMBDA_STACK_NAME"] = "dbx-python-lambda"
write_env("LAMBDA_STACK_NAME", env["LAMBDA_STACK_NAME"])
run_command(
f"bash {DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh",
env=env,
cwd=DRIVERS_TOOLS,
)
if test_name == "search_index":
AUTH = "auth"
if test_name == "aws_lambda":
UV_ARGS.append("--with pip")
# Store AWS creds if they were given.
if "AWS_ACCESS_KEY_ID" in os.environ:
for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]:
if key in os.environ:
write_env(key, os.environ[key])
if test_name == "data_lake":
# Stop any running mongo-orchestration which might be using the port.
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh")
@ -197,7 +218,7 @@ def handle_test_env() -> None:
elif test_name == "auth_oidc":
DB_USER = config["OIDC_ADMIN_USER"]
DB_PASSWORD = config["OIDC_ADMIN_PWD"]
elif test_name == "index_management":
elif test_name == "search_index":
config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh")
DB_USER = config["DRIVERS_ATLAS_LAMBDA_USER"]
DB_PASSWORD = config["DRIVERS_ATLAS_LAMBDA_PASSWORD"]

4
.evergreen/scripts/teardown_tests.py
View File

@ -40,6 +40,10 @@ elif TEST_NAME == "ocsp":
elif TEST_NAME == "serverless":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/serverless/teardown.sh")
# Tear down atlas cluster if applicable.
if TEST_NAME in ["aws_lambda", "search_index"]:
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh")
# Tear down auth_aws if applicable.
# We do not run web-identity hosts on macos, because the hosts lack permissions,
# so there is no reason to run the teardown, which would error with a 401.

7
.evergreen/scripts/utils.py
View File

@ -39,7 +39,7 @@ TEST_SUITE_MAP = {
"default_sync": "default",
"encryption": "encryption",
"enterprise_auth": "auth",
"index_management": "index_management",
"search_index": "search_index",
"kms": "kms",
"load_balancer": "load_balancer",
"mockupdb": "mockupdb",
@ -52,7 +52,7 @@ TEST_SUITE_MAP = {
# Tests that require a sub test suite.
SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
EXTRA_TESTS = ["mod_wsgi"]
EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"]
def get_test_options(
@ -153,7 +153,8 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None:
LOGGER.info("Running command '%s'... done.", cmd)
def create_archive() -> None:
def create_archive() -> str:
run_command("git add .", cwd=ROOT)
run_command('git commit -m "add files"', check=False, cwd=ROOT)
run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT)
return TMP_DRIVER_FILE

3
.gitignore vendored
View File

@ -18,6 +18,7 @@ mongocryptd.pid
.idea/
.vscode/
.nova/
.temp/
venv/
secrets-export.sh
libmongocrypt.tar.gz
@ -32,10 +33,10 @@ results.json
# Lambda temp files
test/lambda/.aws-sam
test/lambda/env.json
test/lambda/mongodb/pymongo/*
test/lambda/mongodb/gridfs/*
test/lambda/mongodb/bson/*
test/lambda/*.json
# test results and logs
xunit-results/

35
CONTRIBUTING.md
View File

@ -248,6 +248,7 @@ the pages will re-render and the browser will automatically refresh.
- Run the tests with `just run-tests`.
The supported types are [`default`, `azure`, `gcp`, `eks`, `aks`, and `gke`].
For the `eks` test, you will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
### KMS tests
@ -275,6 +276,40 @@ Note: these tests can only be run from an Evergreen host.
- Run `just setup-tests atlas_connect`.
- Run `just run-tests`.
### Search Index tests
- Run `just run-server search_index`.
- Run `just setup-tests search_index`.
- Run `just run-tests`.
### MockupDB tests
- Run `just setup-tests mockupdb`.
- Run `just run-tests`.
### Doc tests
The doc tests require a running server.
- Run `just run-server`.
- Run `just docs-test`.
### Free-threaded Python Tests
In the evergreen builds, the tests are configured to use the free-threaded python from the toolchain.
Locally you can run:
- Run `just run-server`.
- Run `just setup-tests`.
- Run `UV_PYTHON=3.13t just run-tests`.
### AWS Lambda tests
You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
- Run `just setup-tests aws_lambda`.
- Run `just run-tests`.
### mod_wsgi tests
Note: these tests can only be run from an Evergreen Linux host that has the Python toolchain.

1
doc/changelog.rst
View File

@ -8,6 +8,7 @@ PyMongo 4.12 brings a number of changes including:
- Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to
:class:`~pymongo.encryption_options.AutoEncryptionOpts`.
- Support for $lookup in CSFLE and QE supported on MongoDB 8.1+.
Issues Resolved
...............

10
pymongo/asynchronous/encryption.py
View File

@ -242,7 +242,7 @@ class _EncryptionIO(AsyncMongoCryptCallback): # type: ignore[misc]
)
raise exc from final_err
async def collection_info(self, database: str, filter: bytes) -> Optional[bytes]:
async def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]]:
"""Get the collection info for a namespace.
The returned collection info is passed to libmongocrypt which reads
@ -251,14 +251,12 @@ class _EncryptionIO(AsyncMongoCryptCallback): # type: ignore[misc]
:param database: The database on which to run listCollections.
:param filter: The filter to pass to listCollections.
:return: The first document from the listCollections command response as BSON.
:return: All documents from the listCollections command response as BSON.
"""
async with await self.client_ref()[database].list_collections(
filter=RawBSONDocument(filter)
) as cursor:
async for doc in cursor:
return _dict_to_bson(doc, False, _DATA_KEY_OPTS)
return None
return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) async for doc in cursor]
def spawn(self) -> None:
"""Spawn mongocryptd.
@ -551,7 +549,7 @@ def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions:
# For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms.
if kwargs.get("key_expiration_ms") is None:
kwargs.pop("key_expiration_ms", None)
return MongoCryptOptions(**kwargs)
return MongoCryptOptions(**kwargs, enable_multiple_collinfo=True)
class AsyncClientEncryption(Generic[_DocumentType]):

10
pymongo/synchronous/encryption.py
View File

@ -241,7 +241,7 @@ class _EncryptionIO(MongoCryptCallback): # type: ignore[misc]
)
raise exc from final_err
def collection_info(self, database: str, filter: bytes) -> Optional[bytes]:
def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]]:
"""Get the collection info for a namespace.
The returned collection info is passed to libmongocrypt which reads
@ -250,12 +250,10 @@ class _EncryptionIO(MongoCryptCallback): # type: ignore[misc]
:param database: The database on which to run listCollections.
:param filter: The filter to pass to listCollections.
:return: The first document from the listCollections command response as BSON.
:return: All documents from the listCollections command response as BSON.
"""
with self.client_ref()[database].list_collections(filter=RawBSONDocument(filter)) as cursor:
for doc in cursor:
return _dict_to_bson(doc, False, _DATA_KEY_OPTS)
return None
return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) for doc in cursor]
def spawn(self) -> None:
"""Spawn mongocryptd.
@ -548,7 +546,7 @@ def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions:
# For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms.
if kwargs.get("key_expiration_ms") is None:
kwargs.pop("key_expiration_ms", None)
return MongoCryptOptions(**kwargs)
return MongoCryptOptions(**kwargs, enable_multiple_collinfo=True)
class ClientEncryption(Generic[_DocumentType]):

2
pyproject.toml
View File

@ -128,7 +128,7 @@ markers = [
"atlas_connect: tests that rely on an atlas connection",
"data_lake: tests that rely on atlas data lake",
"perf: benchmark tests",
"index_management: index management tests",
"search_index: search index helper tests",
"kms: client-side field-level encryption tests using kms",
"encryption: encryption tests",
"load_balancer: load balancer tests",

307
test/asynchronous/test_encryption.py
View File

@ -73,7 +73,7 @@ from test.utils_shared import (
is_greenthread_patched,
)
from bson import DatetimeMS, Decimal128, encode, json_util
from bson import BSON, DatetimeMS, Decimal128, encode, json_util
from bson.binary import UUID_SUBTYPE, Binary, UuidRepresentation
from bson.codec_options import CodecOptions
from bson.errors import BSONError
@ -94,6 +94,7 @@ from pymongo.errors import (
EncryptionError,
InvalidOperation,
OperationFailure,
PyMongoError,
ServerSelectionTimeoutError,
WriteError,
)
@ -2419,6 +2420,310 @@ class TestExplicitQueryableEncryption(AsyncEncryptionIntegrationTest):
self.assertEqual(decrypted, val)
# https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup
class TestLookupProse(AsyncEncryptionIntegrationTest):
@async_client_context.require_no_standalone
@async_client_context.require_version_min(7, 0, -1)
async def asyncSetUp(self):
await super().asyncSetUp()
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
await encrypted_client.drop_database("db")
key_doc = json_data("etc", "data", "lookup", "key-doc.json")
await create_key_vault(encrypted_client.db.keyvault, key_doc)
self.addAsyncCleanup(async_client_context.client.drop_database, "db")
await encrypted_client.db.create_collection(
"csfle",
validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")},
)
await encrypted_client.db.create_collection(
"csfle2",
validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")},
)
await encrypted_client.db.create_collection(
"qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json")
)
await encrypted_client.db.create_collection(
"qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json")
)
await encrypted_client.db.create_collection("no_schema")
await encrypted_client.db.create_collection("no_schema2")
unencrypted_client = await self.async_rs_or_single_client()
await encrypted_client.db.csfle.insert_one({"csfle": "csfle"})
doc = await unencrypted_client.db.csfle.find_one()
self.assertTrue(isinstance(doc["csfle"], Binary))
await encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"})
doc = await unencrypted_client.db.csfle2.find_one()
self.assertTrue(isinstance(doc["csfle2"], Binary))
await encrypted_client.db.qe.insert_one({"qe": "qe"})
doc = await unencrypted_client.db.qe.find_one()
self.assertTrue(isinstance(doc["qe"], Binary))
await encrypted_client.db.qe2.insert_one({"qe2": "qe2"})
doc = await unencrypted_client.db.qe2.find_one()
self.assertTrue(isinstance(doc["qe2"], Binary))
await encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"})
await encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"})
await encrypted_client.close()
await unencrypted_client.close()
@async_client_context.require_version_min(8, 1, -1)
async def test_1_csfle_joins_no_schema(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "csfle"}},
{
"$lookup": {
"from": "no_schema",
"as": "matched",
"pipeline": [
{"$match": {"no_schema": "no_schema"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_2_qe_joins_no_schema(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.qe.aggregate(
[
{"$match": {"qe": "qe"}},
{
"$lookup": {
"from": "no_schema",
"as": "matched",
"pipeline": [
{"$match": {"no_schema": "no_schema"}},
{"$project": {"_id": 0, "__safeContent__": 0}},
],
}
},
{"$project": {"_id": 0, "__safeContent__": 0}},
]
)
)
self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_3_no_schema_joins_csfle(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.no_schema.aggregate(
[
{"$match": {"no_schema": "no_schema"}},
{
"$lookup": {
"from": "csfle",
"as": "matched",
"pipeline": [{"$match": {"csfle": "csfle"}}, {"$project": {"_id": 0}}],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_4_no_schema_joins_qe(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.no_schema.aggregate(
[
{"$match": {"no_schema": "no_schema"}},
{
"$lookup": {
"from": "qe",
"as": "matched",
"pipeline": [
{"$match": {"qe": "qe"}},
{"$project": {"_id": 0, "__safeContent__": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_5_csfle_joins_csfle2(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "csfle"}},
{
"$lookup": {
"from": "csfle2",
"as": "matched",
"pipeline": [
{"$match": {"csfle2": "csfle2"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_6_qe_joins_qe2(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.qe.aggregate(
[
{"$match": {"qe": "qe"}},
{
"$lookup": {
"from": "qe2",
"as": "matched",
"pipeline": [
{"$match": {"qe2": "qe2"}},
{"$project": {"_id": 0, "__safeContent__": 0}},
],
}
},
{"$project": {"_id": 0, "__safeContent__": 0}},
]
)
)
self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_7_no_schema_joins_no_schema2(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = await anext(
await encrypted_client.db.no_schema.aggregate(
[
{"$match": {"no_schema": "no_schema"}},
{
"$lookup": {
"from": "no_schema2",
"as": "matched",
"pipeline": [
{"$match": {"no_schema2": "no_schema2"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]})
@async_client_context.require_version_min(8, 1, -1)
async def test_8_csfle_joins_qe(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
with self.assertRaises(PyMongoError) as exc:
_ = await anext(
await encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "qe"}},
{
"$lookup": {
"from": "qe",
"as": "matched",
"pipeline": [{"$match": {"qe": "qe"}}, {"$project": {"_id": 0}}],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertTrue("not supported" in str(exc))
@async_client_context.require_version_max(8, 1, -1)
async def test_9_error(self):
encrypted_client = await self.async_rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
with self.assertRaises(PyMongoError) as exc:
_ = await anext(
await encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "csfle"}},
{
"$lookup": {
"from": "no_schema",
"as": "matched",
"pipeline": [
{"$match": {"no_schema": "no_schema"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertTrue("Upgrade" in str(exc))
# https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap
class TestRewrapWithSeparateClientEncryption(AsyncEncryptionIntegrationTest):
MASTER_KEYS: Mapping[str, Mapping[str, Any]] = {

2
test/asynchronous/test_index_management.py
View File

@ -38,7 +38,7 @@ from pymongo.write_concern import WriteConcern
_IS_SYNC = False
pytestmark = pytest.mark.index_management
pytestmark = pytest.mark.search_index
# Location of JSON test specifications.
if _IS_SYNC:

30
test/client-side-encryption/etc/data/lookup/key-doc.json Normal file
View File

@ -0,0 +1,30 @@
{
"_id": {
"$binary": {
"base64": "EjRWeBI0mHYSNBI0VniQEg==",
"subType": "04"
}
},
"keyMaterial": {
"$binary": {
"base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==",
"subType": "00"
}
},
"creationDate": {
"$date": {
"$numberLong": "1648914851981"
}
},
"updateDate": {
"$date": {
"$numberLong": "1648914851981"
}
},
"status": {
"$numberInt": "0"
},
"masterKey": {
"provider": "local"
}
}

19
test/client-side-encryption/etc/data/lookup/schema-csfle.json Normal file
View File

@ -0,0 +1,19 @@
{
"properties": {
"csfle": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "EjRWeBI0mHYSNBI0VniQEg==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"
}

19
test/client-side-encryption/etc/data/lookup/schema-csfle2.json Normal file
View File

@ -0,0 +1,19 @@
{
"properties": {
"csfle2": {
"encrypt": {
"keyId": [
{
"$binary": {
"base64": "EjRWeBI0mHYSNBI0VniQEg==",
"subType": "04"
}
}
],
"bsonType": "string",
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
"bsonType": "object"
}

20
test/client-side-encryption/etc/data/lookup/schema-qe.json Normal file
View File

@ -0,0 +1,20 @@
{
"escCollection": "enxcol_.qe.esc",
"ecocCollection": "enxcol_.qe.ecoc",
"fields": [
{
"keyId": {
"$binary": {
"base64": "EjRWeBI0mHYSNBI0VniQEg==",
"subType": "04"
}
},
"path": "qe",
"bsonType": "string",
"queries": {
"queryType": "equality",
"contention": 0
}
}
]
}

20
test/client-side-encryption/etc/data/lookup/schema-qe2.json Normal file
View File

@ -0,0 +1,20 @@
{
"escCollection": "enxcol_.qe2.esc",
"ecocCollection": "enxcol_.qe2.ecoc",
"fields": [
{
"keyId": {
"$binary": {
"base64": "EjRWeBI0mHYSNBI0VniQEg==",
"subType": "04"
}
},
"path": "qe2",
"bsonType": "string",
"queries": {
"queryType": "equality",
"contention": 0
}
}
]
}

28
test/lambda/build.sh
View File

@ -1,28 +0,0 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
set -o xtrace
rm -rf mongodb/pymongo
rm -rf mongodb/gridfs
rm -rf mongodb/bson
pushd ../..
rm -f pymongo/*.so
rm -f bson/*.so
image="quay.io/pypa/manylinux2014_x86_64:latest"
DOCKER=$(command -v docker) || true
if [ -z "$DOCKER" ]; then
PODMAN=$(command -v podman) || true
if [ -z "$PODMAN" ]; then
echo "docker or podman are required!"
exit 1
fi
DOCKER=podman
fi
$DOCKER run --rm -v "`pwd`:/src" $image /src/test/lambda/build_internal.sh
cp -r pymongo ./test/lambda/mongodb/pymongo
cp -r bson ./test/lambda/mongodb/bson
cp -r gridfs ./test/lambda/mongodb/gridfs
popd

307
test/test_encryption.py
View File

@ -73,7 +73,7 @@ from test.utils_shared import (
)
from test.utils_spec_runner import SpecRunner
from bson import DatetimeMS, Decimal128, encode, json_util
from bson import BSON, DatetimeMS, Decimal128, encode, json_util
from bson.binary import UUID_SUBTYPE, Binary, UuidRepresentation
from bson.codec_options import CodecOptions
from bson.errors import BSONError
@ -91,6 +91,7 @@ from pymongo.errors import (
EncryptionError,
InvalidOperation,
OperationFailure,
PyMongoError,
ServerSelectionTimeoutError,
WriteError,
)
@ -2403,6 +2404,310 @@ class TestExplicitQueryableEncryption(EncryptionIntegrationTest):
self.assertEqual(decrypted, val)
# https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup
class TestLookupProse(EncryptionIntegrationTest):
@client_context.require_no_standalone
@client_context.require_version_min(7, 0, -1)
def setUp(self):
super().setUp()
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
encrypted_client.drop_database("db")
key_doc = json_data("etc", "data", "lookup", "key-doc.json")
create_key_vault(encrypted_client.db.keyvault, key_doc)
self.addCleanup(client_context.client.drop_database, "db")
encrypted_client.db.create_collection(
"csfle",
validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")},
)
encrypted_client.db.create_collection(
"csfle2",
validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")},
)
encrypted_client.db.create_collection(
"qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json")
)
encrypted_client.db.create_collection(
"qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json")
)
encrypted_client.db.create_collection("no_schema")
encrypted_client.db.create_collection("no_schema2")
unencrypted_client = self.rs_or_single_client()
encrypted_client.db.csfle.insert_one({"csfle": "csfle"})
doc = unencrypted_client.db.csfle.find_one()
self.assertTrue(isinstance(doc["csfle"], Binary))
encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"})
doc = unencrypted_client.db.csfle2.find_one()
self.assertTrue(isinstance(doc["csfle2"], Binary))
encrypted_client.db.qe.insert_one({"qe": "qe"})
doc = unencrypted_client.db.qe.find_one()
self.assertTrue(isinstance(doc["qe"], Binary))
encrypted_client.db.qe2.insert_one({"qe2": "qe2"})
doc = unencrypted_client.db.qe2.find_one()
self.assertTrue(isinstance(doc["qe2"], Binary))
encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"})
encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"})
encrypted_client.close()
unencrypted_client.close()
@client_context.require_version_min(8, 1, -1)
def test_1_csfle_joins_no_schema(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "csfle"}},
{
"$lookup": {
"from": "no_schema",
"as": "matched",
"pipeline": [
{"$match": {"no_schema": "no_schema"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]})
@client_context.require_version_min(8, 1, -1)
def test_2_qe_joins_no_schema(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.qe.aggregate(
[
{"$match": {"qe": "qe"}},
{
"$lookup": {
"from": "no_schema",
"as": "matched",
"pipeline": [
{"$match": {"no_schema": "no_schema"}},
{"$project": {"_id": 0, "__safeContent__": 0}},
],
}
},
{"$project": {"_id": 0, "__safeContent__": 0}},
]
)
)
self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]})
@client_context.require_version_min(8, 1, -1)
def test_3_no_schema_joins_csfle(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.no_schema.aggregate(
[
{"$match": {"no_schema": "no_schema"}},
{
"$lookup": {
"from": "csfle",
"as": "matched",
"pipeline": [{"$match": {"csfle": "csfle"}}, {"$project": {"_id": 0}}],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]})
@client_context.require_version_min(8, 1, -1)
def test_4_no_schema_joins_qe(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.no_schema.aggregate(
[
{"$match": {"no_schema": "no_schema"}},
{
"$lookup": {
"from": "qe",
"as": "matched",
"pipeline": [
{"$match": {"qe": "qe"}},
{"$project": {"_id": 0, "__safeContent__": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]})
@client_context.require_version_min(8, 1, -1)
def test_5_csfle_joins_csfle2(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "csfle"}},
{
"$lookup": {
"from": "csfle2",
"as": "matched",
"pipeline": [
{"$match": {"csfle2": "csfle2"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]})
@client_context.require_version_min(8, 1, -1)
def test_6_qe_joins_qe2(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.qe.aggregate(
[
{"$match": {"qe": "qe"}},
{
"$lookup": {
"from": "qe2",
"as": "matched",
"pipeline": [
{"$match": {"qe2": "qe2"}},
{"$project": {"_id": 0, "__safeContent__": 0}},
],
}
},
{"$project": {"_id": 0, "__safeContent__": 0}},
]
)
)
self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]})
@client_context.require_version_min(8, 1, -1)
def test_7_no_schema_joins_no_schema2(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
doc = next(
encrypted_client.db.no_schema.aggregate(
[
{"$match": {"no_schema": "no_schema"}},
{
"$lookup": {
"from": "no_schema2",
"as": "matched",
"pipeline": [
{"$match": {"no_schema2": "no_schema2"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]})
@client_context.require_version_min(8, 1, -1)
def test_8_csfle_joins_qe(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
with self.assertRaises(PyMongoError) as exc:
_ = next(
encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "qe"}},
{
"$lookup": {
"from": "qe",
"as": "matched",
"pipeline": [{"$match": {"qe": "qe"}}, {"$project": {"_id": 0}}],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertTrue("not supported" in str(exc))
@client_context.require_version_max(8, 1, -1)
def test_9_error(self):
encrypted_client = self.rs_or_single_client(
auto_encryption_opts=AutoEncryptionOpts(
key_vault_namespace="db.keyvault",
kms_providers={"local": {"key": LOCAL_MASTER_KEY}},
)
)
with self.assertRaises(PyMongoError) as exc:
_ = next(
encrypted_client.db.csfle.aggregate(
[
{"$match": {"csfle": "csfle"}},
{
"$lookup": {
"from": "no_schema",
"as": "matched",
"pipeline": [
{"$match": {"no_schema": "no_schema"}},
{"$project": {"_id": 0}},
],
}
},
{"$project": {"_id": 0}},
]
)
)
self.assertTrue("Upgrade" in str(exc))
# https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap
class TestRewrapWithSeparateClientEncryption(EncryptionIntegrationTest):
MASTER_KEYS: Mapping[str, Mapping[str, Any]] = {

2
test/test_index_management.py
View File

@ -38,7 +38,7 @@ from pymongo.write_concern import WriteConcern
_IS_SYNC = True
pytestmark = pytest.mark.index_management
pytestmark = pytest.mark.search_index
# Location of JSON test specifications.
if _IS_SYNC:

4
uv.lock generated
View File

@ -1036,6 +1036,7 @@ snappy = [
{ name = "python-snappy" },
]
test = [
{ name = "pip" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
]
@ -1080,6 +1081,7 @@ requires-dist = [
{ name = "cryptography", marker = "extra == 'ocsp'", specifier = ">=2.5" },
{ name = "dnspython", specifier = ">=1.16.0,<3.0.0" },
{ name = "furo", marker = "extra == 'docs'", specifier = "==2024.8.6" },
{ name = "pip", marker = "extra == 'test'" },
{ name = "pykerberos", marker = "os_name != 'nt' and extra == 'gssapi'" },
{ name = "pymongo-auth-aws", marker = "extra == 'aws'", specifier = ">=1.1.0,<2.0.0" },
{ name = "pymongo-auth-aws", marker = "extra == 'encryption'", specifier = ">=1.1.0,<2.0.0" },
@ -1133,7 +1135,7 @@ wheels = [
[[package]]
name = "pymongocrypt"
version = "1.13.0.dev0"
source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1e96c283162aa7789cf01f99f211e0ace8e6d49f" }
source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1cad4ad1c4cd6c11c6a4710da2127dab6a374471" }
dependencies = [
{ name = "cffi" },
{ name = "cryptography" },