From 134f52ff23947c801b2a5b77bb60c2bea7e1b373 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 19 Mar 2025 15:58:12 -0500 Subject: [PATCH 1/3] PYTHON-5213 Convert AWS Lambda and Search Index tests to use new test scripts (#2205) --- .evergreen/config.yml | 80 +-------------------- .evergreen/generated_configs/tasks.yml | 21 ++++++ .evergreen/generated_configs/variants.yml | 10 ++- .evergreen/run-deployed-lambda-aws-tests.sh | 10 --- .evergreen/scripts/generate_config.py | 28 +++++++- .evergreen/scripts/run_server.py | 20 +++--- .evergreen/scripts/run_tests.py | 44 ++++++++++++ .evergreen/scripts/setup_tests.py | 24 ++++++- .evergreen/scripts/teardown_tests.py | 4 ++ .evergreen/scripts/utils.py | 7 +- .gitignore | 3 +- CONTRIBUTING.md | 14 ++++ pyproject.toml | 2 +- test/asynchronous/test_index_management.py | 2 +- test/lambda/build.sh | 28 -------- test/test_index_management.py | 2 +- 16 files changed, 164 insertions(+), 135 deletions(-) delete mode 100755 .evergreen/run-deployed-lambda-aws-tests.sh delete mode 100755 test/lambda/build.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 15bdac315..6b2b332a5 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -268,27 +268,7 @@ functions: - command: ec2.assume_role params: role_arn: ${aws_test_secrets_role} - - "setup atlas": - - command: subprocess.exec - params: - binary: bash - include_expansions_in_env: ["task_id", "execution"] - env: - MONGODB_VERSION: "7.0" - LAMBDA_STACK_NAME: dbx-python-lambda - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: atlas-expansion.yml - - "teardown atlas": - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + duration_seconds: 3600 "attach benchmark test results": - command: attach.results @@ -314,31 +294,6 @@ post: - func: "upload test results" - func: "cleanup" -task_groups: - - name: test_aws_lambda_task_group - setup_group: - - func: fetch source - - func: setup system - - func: setup atlas - teardown_task: - - func: teardown atlas - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - test-aws-lambda-deployed - - - name: test_atlas_task_group_search_indexes - setup_group: - - func: fetch source - - func: setup system - - func: setup atlas - teardown_task: - - func: teardown atlas - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - test-search-index-helpers - tasks: # Wildcard task. Do you need to find out what tools are available and where? # Throw it here, and execute this task on all buildvariants @@ -365,17 +320,6 @@ tasks: - func: "run server" - func: "run doctests" - - name: "test-search-index-helpers" - commands: - - func: "run server" - vars: - VERSION: "6.0" - TOPOLOGY: "replica_set" - - func: "run tests" - vars: - TEST_NAME: index_management - AUTH: "auth" - - name: "no-server" tags: ["no-server"] commands: @@ -390,22 +334,6 @@ tasks: TOPOLOGY: "replica_set" - func: "run tests" - - name: "test-aws-lambda-deployed" - commands: - - command: ec2.assume_role - params: - role_arn: ${LAMBDA_AWS_ROLE_ARN} - duration_seconds: 3600 - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - .evergreen/run-deployed-lambda-aws-tests.sh - env: - TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/test/lambda - # }}} - name: "coverage-report" tags: ["coverage"] @@ -482,12 +410,6 @@ buildvariants: batchtime: 10080 # 7 days - name: test-azurekms-fail -- name: rhel8-test-lambda - display_name: FaaS Lambda - run_on: rhel87-small - tasks: - - name: test_aws_lambda_task_group - - name: rhel8-import-time display_name: Import Time run_on: rhel87-small diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 7d03822a6..c692ec31d 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -23,6 +23,15 @@ tasks: TEST_NAME: data_lake tags: [atlas_data_lake] + # Aws lambda tests + - name: test-aws-lambda-deployed + commands: + - func: assume ec2 role + - func: run tests + vars: + TEST_NAME: aws_lambda + tags: [aws_lambda] + # Aws tests - name: test-auth-aws-4.4-regular commands: @@ -1220,6 +1229,18 @@ tasks: - func: send dashboard data tags: [perf] + # Search index tests + - name: test-search-index-helpers + commands: + - func: assume ec2 role + - func: run server + vars: + TEST_NAME: search_index + - func: run tests + vars: + TEST_NAME: search_index + tags: [search_index] + # Server tests - name: test-4.0-standalone-auth-ssl-sync commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 78f181fc6..361a0c9dd 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -132,6 +132,14 @@ buildvariants: expansions: PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 + # Aws lambda tests + - name: faas-lambda + tasks: + - name: .aws_lambda + display_name: FaaS Lambda + run_on: + - rhel87-small + # Compression tests - name: compression-snappy-rhel8-python3.9-no-c tasks: @@ -986,7 +994,7 @@ buildvariants: # Search index tests - name: search-index-helpers-rhel8-python3.9 tasks: - - name: test_atlas_task_group_search_indexes + - name: .search_index display_name: Search Index Helpers RHEL8 Python3.9 run_on: - rhel87-small diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh deleted file mode 100755 index aa16d6265..000000000 --- a/.evergreen/run-deployed-lambda-aws-tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail - -export PATH="/opt/python/3.9/bin:${PATH}" -python --version -pushd ./test/lambda - -. build.sh -popd -. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index ebb4b5d31..dad4f27bc 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -672,7 +672,7 @@ def create_search_index_variants(): python = CPYTHONS[0] return [ create_variant( - ["test_atlas_task_group_search_indexes"], + [".search_index"], get_display_name("Search Index Helpers", host, python=python), python=python, host=host, @@ -779,6 +779,11 @@ def create_alternative_hosts_variants(): return variants +def create_aws_lambda_variants(): + host = HOSTS["rhel8"] + return [create_variant([".aws_lambda"], display_name="FaaS Lambda", host=host)] + + ############## # Tasks ############## @@ -927,6 +932,27 @@ def _create_ocsp_task(algo, variant, server_type, base_task_name): return EvgTask(name=task_name, tags=tags, commands=commands) +def create_aws_lambda_tasks(): + assume_func = FunctionCall(func="assume ec2 role") + vars = dict(TEST_NAME="aws_lambda") + test_func = FunctionCall(func="run tests", vars=vars) + task_name = "test-aws-lambda-deployed" + tags = ["aws_lambda"] + commands = [assume_func, test_func] + return [EvgTask(name=task_name, tags=tags, commands=commands)] + + +def create_search_index_tasks(): + assume_func = FunctionCall(func="assume ec2 role") + server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index")) + vars = dict(TEST_NAME="search_index") + test_func = FunctionCall(func="run tests", vars=vars) + task_name = "test-search-index-helpers" + tags = ["search_index"] + commands = [assume_func, server_func, test_func] + return [EvgTask(name=task_name, tags=tags, commands=commands)] + + def create_atlas_connect_tasks(): vars = dict(TEST_NAME="atlas_connect") assume_func = FunctionCall(func="assume ec2 role") diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f43ada4bb..f85207daa 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -22,14 +22,6 @@ def start_server(): if "VERSION" in os.environ: os.environ["MONGODB_VERSION"] = os.environ["VERSION"] - if opts.auth: - extra_opts.append("--auth") - - if opts.verbose: - extra_opts.append("-v") - elif opts.quiet: - extra_opts.append("-q") - if test_name == "auth_aws": set_env("AUTH_AWS") @@ -51,6 +43,10 @@ def start_server(): if not found: raise ValueError("Please provide an orchestration file") + elif test_name == "search_index": + os.environ["TOPOLOGY"] = "replica_set" + os.environ["MONGODB_VERSION"] = "7.0" + if not os.environ.get("TEST_CRYPT_SHARED"): set_env("SKIP_CRYPT_SHARED") @@ -62,6 +58,14 @@ def start_server(): set_env("TLS_PEM_KEY_FILE", certs / "server.pem") set_env("TLS_CA_FILE", certs / "ca.pem") + if opts.auth: + extra_opts.append("--auth") + + if opts.verbose: + extra_opts.append("-v") + elif opts.quiet: + extra_opts.append("-q") + cmd = ["bash", f"{DRIVERS_TOOLS}/.evergreen/run-orchestration.sh", *extra_opts] run_command(cmd, cwd=DRIVERS_TOOLS) diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 2e23a366b..9f700d70e 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -4,8 +4,11 @@ import json import logging import os import platform +import shutil import sys from datetime import datetime +from pathlib import Path +from shutil import which import pytest from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command @@ -81,6 +84,42 @@ def handle_pymongocrypt() -> None: LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") +def handle_aws_lambda() -> None: + env = os.environ.copy() + target_dir = ROOT / "test/lambda" + env["TEST_LAMBDA_DIRECTORY"] = str(target_dir) + env.setdefault("AWS_REGION", "us-east-1") + dirs = ["pymongo", "gridfs", "bson"] + # Store the original .so files. + before_sos = [] + for dname in dirs: + before_sos.extend(f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")) + # Build the c extensions. + docker = which("docker") or which("podman") + if not docker: + raise ValueError("Could not find docker!") + image = "quay.io/pypa/manylinux2014_x86_64:latest" + run_command( + f'{docker} run --rm -v "{ROOT}:/src" --platform linux/amd64 {image} /src/test/lambda/build_internal.sh' + ) + for dname in dirs: + target = ROOT / "test/lambda/mongodb" / dname + shutil.rmtree(target, ignore_errors=True) + shutil.copytree(ROOT / dname, target) + # Remove the original so files from the lambda directory. + for so_path in before_sos: + (ROOT / "test/lambda/mongodb" / so_path).unlink() + # Remove the new so files from the ROOT directory. + for dname in dirs: + so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")] + for so_path in list(so_paths): + if so_path not in before_sos: + Path(so_path).unlink() + + script_name = "run-deployed-lambda-aws-tests.sh" + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/aws_lambda/{script_name}", env=env) + + def run() -> None: # Handle green framework first so they can patch modules. if GREEN_FRAMEWORK: @@ -129,6 +168,11 @@ def run() -> None: test_oidc_send_to_remote(SUB_TEST_NAME) return + # Run deployed aws lambda tests. + if TEST_NAME == "aws_lambda": + handle_aws_lambda() + return + if os.environ.get("DEBUG_LOG"): TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG} -o log_cli=1".split()) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 56011341a..ae7fde5ef 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -175,6 +175,28 @@ def handle_test_env() -> None: if not config: AUTH = "noauth" + if test_name in ["aws_lambda", "search_index"]: + env = os.environ.copy() + env["MONGODB_VERSION"] = "7.0" + env["LAMBDA_STACK_NAME"] = "dbx-python-lambda" + write_env("LAMBDA_STACK_NAME", env["LAMBDA_STACK_NAME"]) + run_command( + f"bash {DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh", + env=env, + cwd=DRIVERS_TOOLS, + ) + + if test_name == "search_index": + AUTH = "auth" + + if test_name == "aws_lambda": + UV_ARGS.append("--with pip") + # Store AWS creds if they were given. + if "AWS_ACCESS_KEY_ID" in os.environ: + for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]: + if key in os.environ: + write_env(key, os.environ[key]) + if test_name == "data_lake": # Stop any running mongo-orchestration which might be using the port. run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") @@ -197,7 +219,7 @@ def handle_test_env() -> None: elif test_name == "auth_oidc": DB_USER = config["OIDC_ADMIN_USER"] DB_PASSWORD = config["OIDC_ADMIN_PWD"] - elif test_name == "index_management": + elif test_name == "search_index": config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh") DB_USER = config["DRIVERS_ATLAS_LAMBDA_USER"] DB_PASSWORD = config["DRIVERS_ATLAS_LAMBDA_PASSWORD"] diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index d27cb8682..390e0a68e 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -40,6 +40,10 @@ elif TEST_NAME == "ocsp": elif TEST_NAME == "serverless": run_command(f"bash {DRIVERS_TOOLS}/.evergreen/serverless/teardown.sh") +# Tear down atlas cluster if applicable. +if TEST_NAME in ["aws_lambda", "search_index"]: + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh") + # Tear down auth_aws if applicable. # We do not run web-identity hosts on macos, because the hosts lack permissions, # so there is no reason to run the teardown, which would error with a 401. diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 535e392ea..0ff3b76a5 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -39,7 +39,7 @@ TEST_SUITE_MAP = { "default_sync": "default", "encryption": "encryption", "enterprise_auth": "auth", - "index_management": "index_management", + "search_index": "search_index", "kms": "kms", "load_balancer": "load_balancer", "mockupdb": "mockupdb", @@ -52,7 +52,7 @@ TEST_SUITE_MAP = { # Tests that require a sub test suite. SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"] -EXTRA_TESTS = ["mod_wsgi"] +EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"] def get_test_options( @@ -153,7 +153,8 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: LOGGER.info("Running command '%s'... done.", cmd) -def create_archive() -> None: +def create_archive() -> str: run_command("git add .", cwd=ROOT) run_command('git commit -m "add files"', check=False, cwd=ROOT) run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) + return TMP_DRIVER_FILE diff --git a/.gitignore b/.gitignore index 966059e69..a88a7556e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ mongocryptd.pid .idea/ .vscode/ .nova/ +.temp/ venv/ secrets-export.sh libmongocrypt.tar.gz @@ -32,10 +33,10 @@ results.json # Lambda temp files test/lambda/.aws-sam -test/lambda/env.json test/lambda/mongodb/pymongo/* test/lambda/mongodb/gridfs/* test/lambda/mongodb/bson/* +test/lambda/*.json # test results and logs xunit-results/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e466c6847..5f55e5668 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -248,6 +248,7 @@ the pages will re-render and the browser will automatically refresh. - Run the tests with `just run-tests`. The supported types are [`default`, `azure`, `gcp`, `eks`, `aks`, and `gke`]. +For the `eks` test, you will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets). ### KMS tests @@ -275,6 +276,19 @@ Note: these tests can only be run from an Evergreen host. - Run `just setup-tests atlas_connect`. - Run `just run-tests`. +### Search Index tests + +- Run `just run-server search_index`. +- Run `just setup-tests search_index`. +- Run `just run-tests`. + +### AWS Lambda tests + +You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets). + +- Run `just setup-tests aws_lambda`. +- Run `just run-tests`. + ### mod_wsgi tests Note: these tests can only be run from an Evergreen Linux host that has the Python toolchain. diff --git a/pyproject.toml b/pyproject.toml index 993b3e5ae..f8c25ed60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,7 +128,7 @@ markers = [ "atlas_connect: tests that rely on an atlas connection", "data_lake: tests that rely on atlas data lake", "perf: benchmark tests", - "index_management: index management tests", + "search_index: search index helper tests", "kms: client-side field-level encryption tests using kms", "encryption: encryption tests", "load_balancer: load balancer tests", diff --git a/test/asynchronous/test_index_management.py b/test/asynchronous/test_index_management.py index 4b218de13..890788fc5 100644 --- a/test/asynchronous/test_index_management.py +++ b/test/asynchronous/test_index_management.py @@ -38,7 +38,7 @@ from pymongo.write_concern import WriteConcern _IS_SYNC = False -pytestmark = pytest.mark.index_management +pytestmark = pytest.mark.search_index # Location of JSON test specifications. if _IS_SYNC: diff --git a/test/lambda/build.sh b/test/lambda/build.sh deleted file mode 100755 index c7cc24eab..000000000 --- a/test/lambda/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -set -o xtrace - -rm -rf mongodb/pymongo -rm -rf mongodb/gridfs -rm -rf mongodb/bson - -pushd ../.. -rm -f pymongo/*.so -rm -f bson/*.so -image="quay.io/pypa/manylinux2014_x86_64:latest" - -DOCKER=$(command -v docker) || true -if [ -z "$DOCKER" ]; then - PODMAN=$(command -v podman) || true - if [ -z "$PODMAN" ]; then - echo "docker or podman are required!" - exit 1 - fi - DOCKER=podman -fi - -$DOCKER run --rm -v "`pwd`:/src" $image /src/test/lambda/build_internal.sh -cp -r pymongo ./test/lambda/mongodb/pymongo -cp -r bson ./test/lambda/mongodb/bson -cp -r gridfs ./test/lambda/mongodb/gridfs -popd diff --git a/test/test_index_management.py b/test/test_index_management.py index 3a2b17cd3..dea8c0e2b 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -38,7 +38,7 @@ from pymongo.write_concern import WriteConcern _IS_SYNC = True -pytestmark = pytest.mark.index_management +pytestmark = pytest.mark.search_index # Location of JSON test specifications. if _IS_SYNC: From 737a1b73443223cc8c986826f3426d583f243b09 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 19 Mar 2025 20:53:35 -0500 Subject: [PATCH 2/3] PYTHON-5220 Convert remaining tests to use standard test setup (#2211) --- .evergreen/config.yml | 56 ++++------------------- .evergreen/generated_configs/tasks.yml | 33 +++++++++++++ .evergreen/generated_configs/variants.yml | 12 ++++- .evergreen/run-tests.sh | 5 +- .evergreen/scripts/cleanup.sh | 8 ++++ .evergreen/scripts/configure-env.sh | 2 - .evergreen/scripts/generate_config.py | 40 +++++++++++++++- .evergreen/scripts/run-with-env.sh | 21 --------- .evergreen/scripts/setup_tests.py | 3 +- CONTRIBUTING.md | 21 +++++++++ uv.lock | 2 + 11 files changed, 126 insertions(+), 77 deletions(-) delete mode 100755 .evergreen/scripts/run-with-env.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6b2b332a5..97845b86d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -42,7 +42,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "skip_web_identity_auth_test", "skip_ECS_auth_test"] + include_expansions_in_env: ["is_patch", "project", "version_id"] binary: bash working_dir: "src" args: @@ -213,16 +213,14 @@ functions: params: file: ${DRIVERS_TOOLS}/mo-expansion.yml - "run doctests": - - command: subprocess.exec - type: test - params: - include_expansions_in_env: [ "PYTHON_BINARY" ] - working_dir: "src" - binary: bash - args: - - .evergreen/scripts/run-with-env.sh - - .evergreen/scripts/run-doctests.sh + "run just script": + - command: subprocess.exec + type: test + params: + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + binary: bash + working_dir: "src" + args: [.evergreen/just.sh, "${JUSTFILE_TARGET}"] "run tests": - command: subprocess.exec @@ -248,7 +246,6 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/cleanup.sh "teardown system": @@ -305,36 +302,7 @@ tasks: params: args: - src/.evergreen/scripts/run-getdata.sh -# Standard test tasks {{{ - - name: "mockupdb" - tags: ["mockupdb"] - commands: - - func: "run tests" - vars: - TEST_NAME: mockupdb - - - name: "doctests" - tags: ["doctests"] - commands: - - func: "run server" - - func: "run doctests" - - - name: "no-server" - tags: ["no-server"] - commands: - - func: "run tests" - - - name: "free-threading" - tags: ["free-threading"] - commands: - - func: "run server" - vars: - VERSION: "8.0" - TOPOLOGY: "replica_set" - - func: "run tests" - -# }}} - name: "coverage-report" tags: ["coverage"] depends_on: @@ -384,12 +352,6 @@ tasks: - ${github_commit} buildvariants: -- name: "no-server" - display_name: "No server" - run_on: - - rhel84-small - tasks: - - name: "no-server" - name: "Coverage Report" display_name: "Coverage Report" diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index c692ec31d..efc784406 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -713,6 +713,15 @@ tasks: AWS_ROLE_SESSION_NAME: test tags: [auth-aws, auth-aws-web-identity] + # Doctest tests + - name: test-doctests + commands: + - func: run server + - func: run just script + vars: + JUSTFILE_TARGET: docs-test + tags: [doctests] + # Enterprise auth tests - name: test-enterprise-auth commands: @@ -727,6 +736,16 @@ tasks: AUTH: auth tags: [enterprise_auth] + # Free threading tests + - name: test-free-threading + commands: + - func: run server + vars: + VERSION: "8.0" + TOPOLOGY: replica_set + - func: run tests + tags: [free-threading] + # Kms tests - name: test-gcpkms commands: @@ -799,6 +818,14 @@ tasks: TEST_NAME: load_balancer tags: [load-balancer, noauth, nossl] + # Mockupdb tests + - name: test-mockupdb + commands: + - func: run tests + vars: + TEST_NAME: mockupdb + tags: [mockupdb] + # Mod wsgi tests - name: mod-wsgi-standalone commands: @@ -841,6 +868,12 @@ tasks: SUB_TEST_NAME: embedded tags: [mod_wsgi] + # No server tests + - name: test-no-server + commands: + - func: run tests + tags: [no-server] + # Ocsp tests - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 361a0c9dd..aa20fef89 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -242,7 +242,7 @@ buildvariants: # Doctests tests - name: doctests-rhel8-python3.9 tasks: - - name: doctests + - name: .doctests display_name: Doctests RHEL8 Python3.9 run_on: - rhel87-small @@ -672,7 +672,7 @@ buildvariants: # Mockupdb tests - name: mockupdb-rhel8-python3.9 tasks: - - name: mockupdb + - name: .mockupdb display_name: MockupDB RHEL8 Python3.9 run_on: - rhel87-small @@ -746,6 +746,14 @@ buildvariants: NO_EXT: "1" PYTHON_BINARY: /opt/python/3.13/bin/python3 + # No server tests + - name: no-server + tasks: + - name: .no-server + display_name: No server + run_on: + - rhel87-small + # Ocsp tests - name: ocsp-rhel8-v4.4-python3.9 tasks: diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 40336c6d2..6f53ced61 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -34,7 +34,10 @@ fi uv sync ${UV_ARGS} --reinstall uv pip list +# Ensure we go back to base environment after the test. +trap "uv sync" EXIT HUP + # Start the test runner. -uv run .evergreen/scripts/run_tests.py "$@" +uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@" popd diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index c58d2163d..6bb4b3ce5 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -1,4 +1,12 @@ #!/bin/bash +HERE=$(dirname ${BASH_SOURCE:-$0}) + +# Try to source the env file. +if [ -f $HERE/env.sh ]; then + echo "Sourcing env file" + source $HERE/env.sh +fi + rm -rf "${DRIVERS_TOOLS}" || true rm -f ./secrets-export.sh || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fa37b8fb0..9ec98bb5b 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -60,8 +60,6 @@ export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" -export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}" -export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export CARGO_HOME="$CARGO_HOME" export UV_TOOL_DIR="$UV_TOOL_DIR" diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index dad4f27bc..0a2496c66 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -685,7 +685,7 @@ def create_mockupdb_variants(): python = CPYTHONS[0] return [ create_variant( - ["mockupdb"], + [".mockupdb"], get_display_name("MockupDB", host, python=python), python=python, host=host, @@ -698,7 +698,7 @@ def create_doctests_variants(): python = CPYTHONS[0] return [ create_variant( - ["doctests"], + [".doctests"], get_display_name("Doctests", host, python=python), python=python, host=host, @@ -748,6 +748,11 @@ def create_aws_auth_variants(): return variants +def create_no_server_variants(): + host = HOSTS["rhel8"] + return [create_variant([".no-server"], "No server", host=host)] + + def create_alternative_hosts_variants(): batchtime = BATCHTIME_WEEK variants = [] @@ -1040,6 +1045,37 @@ def create_ocsp_tasks(): return tasks +def create_mockupdb_tasks(): + test_func = FunctionCall(func="run tests", vars=dict(TEST_NAME="mockupdb")) + task_name = "test-mockupdb" + tags = ["mockupdb"] + return [EvgTask(name=task_name, tags=tags, commands=[test_func])] + + +def create_doctest_tasks(): + server_func = FunctionCall(func="run server") + test_func = FunctionCall(func="run just script", vars=dict(JUSTFILE_TARGET="docs-test")) + task_name = "test-doctests" + tags = ["doctests"] + return [EvgTask(name=task_name, tags=tags, commands=[server_func, test_func])] + + +def create_no_server_tasks(): + test_func = FunctionCall(func="run tests") + task_name = "test-no-server" + tags = ["no-server"] + return [EvgTask(name=task_name, tags=tags, commands=[test_func])] + + +def create_free_threading_tasks(): + vars = dict(VERSION="8.0", TOPOLOGY="replica_set") + server_func = FunctionCall(func="run server", vars=vars) + test_func = FunctionCall(func="run tests") + task_name = "test-free-threading" + tags = ["free-threading"] + return [EvgTask(name=task_name, tags=tags, commands=[server_func, test_func])] + + def create_serverless_tasks(): vars = dict(TEST_NAME="serverless", AUTH="auth", SSL="ssl") test_func = FunctionCall(func="run tests", vars=vars) diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh deleted file mode 100755 index 2fd073605..000000000 --- a/.evergreen/scripts/run-with-env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -eu - -# Example use: bash run-with-env.sh run-tests.sh {args...} - -# Parameter expansion to get just the current directory's name -if [ "${PWD##*/}" == "src" ]; then - . .evergreen/scripts/env.sh - if [ -f ".evergreen/scripts/test-env.sh" ]; then - . .evergreen/scripts/test-env.sh - fi -else - . src/.evergreen/scripts/env.sh - if [ -f "src/.evergreen/scripts/test-env.sh" ]; then - . src/.evergreen/scripts/test-env.sh - fi -fi - -set -eu - -# shellcheck source=/dev/null -. "$@" diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index ae7fde5ef..17f9de1a7 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -128,8 +128,7 @@ def handle_test_env() -> None: TEST_ARGS = "" # Start compiling the args we'll pass to uv. - # Run in an isolated environment so as not to pollute the base venv. - UV_ARGS = ["--isolated --extra test"] + UV_ARGS = ["--extra test --no-group dev"] test_title = test_name if sub_test_name: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f55e5668..e75510171 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -282,6 +282,27 @@ Note: these tests can only be run from an Evergreen host. - Run `just setup-tests search_index`. - Run `just run-tests`. +### MockupDB tests + +- Run `just setup-tests mockupdb`. +- Run `just run-tests`. + +### Doc tests + +The doc tests require a running server. + +- Run `just run-server`. +- Run `just docs-test`. + +### Free-threaded Python Tests + +In the evergreen builds, the tests are configured to use the free-threaded python from the toolchain. +Locally you can run: + +- Run `just run-server`. +- Run `just setup-tests`. +- Run `UV_PYTHON=3.13t just run-tests`. + ### AWS Lambda tests You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets). diff --git a/uv.lock b/uv.lock index 8b5d592dc..7dae5c913 100644 --- a/uv.lock +++ b/uv.lock @@ -1036,6 +1036,7 @@ snappy = [ { name = "python-snappy" }, ] test = [ + { name = "pip" }, { name = "pytest" }, { name = "pytest-asyncio" }, ] @@ -1080,6 +1081,7 @@ requires-dist = [ { name = "cryptography", marker = "extra == 'ocsp'", specifier = ">=2.5" }, { name = "dnspython", specifier = ">=1.16.0,<3.0.0" }, { name = "furo", marker = "extra == 'docs'", specifier = "==2024.8.6" }, + { name = "pip", marker = "extra == 'test'" }, { name = "pykerberos", marker = "os_name != 'nt' and extra == 'gssapi'" }, { name = "pymongo-auth-aws", marker = "extra == 'aws'", specifier = ">=1.1.0,<2.0.0" }, { name = "pymongo-auth-aws", marker = "extra == 'encryption'", specifier = ">=1.1.0,<2.0.0" }, From 1145c9de543bb5748a61dbf338385ff2ff6330c6 Mon Sep 17 00:00:00 2001 From: Iris <58442094+sleepyStick@users.noreply.github.com> Date: Thu, 20 Mar 2025 13:55:52 -0700 Subject: [PATCH 3/3] PYTHON-5046 Support $lookup in CSFLE and QE (#2210) --- doc/changelog.rst | 1 + pymongo/asynchronous/encryption.py | 10 +- pymongo/synchronous/encryption.py | 10 +- test/asynchronous/test_encryption.py | 307 +++++++++++++++++- .../etc/data/lookup/key-doc.json | 30 ++ .../etc/data/lookup/schema-csfle.json | 19 ++ .../etc/data/lookup/schema-csfle2.json | 19 ++ .../etc/data/lookup/schema-qe.json | 20 ++ .../etc/data/lookup/schema-qe2.json | 20 ++ test/test_encryption.py | 307 +++++++++++++++++- uv.lock | 2 +- 11 files changed, 730 insertions(+), 15 deletions(-) create mode 100644 test/client-side-encryption/etc/data/lookup/key-doc.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-csfle.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-csfle2.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-qe.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-qe2.json diff --git a/doc/changelog.rst b/doc/changelog.rst index a54d22907..b172da6b8 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ PyMongo 4.12 brings a number of changes including: - Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to :class:`~pymongo.encryption_options.AutoEncryptionOpts`. +- Support for $lookup in CSFLE and QE supported on MongoDB 8.1+. Issues Resolved ............... diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index b18ed53f9..3582bec9a 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -242,7 +242,7 @@ class _EncryptionIO(AsyncMongoCryptCallback): # type: ignore[misc] ) raise exc from final_err - async def collection_info(self, database: str, filter: bytes) -> Optional[bytes]: + async def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]]: """Get the collection info for a namespace. The returned collection info is passed to libmongocrypt which reads @@ -251,14 +251,12 @@ class _EncryptionIO(AsyncMongoCryptCallback): # type: ignore[misc] :param database: The database on which to run listCollections. :param filter: The filter to pass to listCollections. - :return: The first document from the listCollections command response as BSON. + :return: All documents from the listCollections command response as BSON. """ async with await self.client_ref()[database].list_collections( filter=RawBSONDocument(filter) ) as cursor: - async for doc in cursor: - return _dict_to_bson(doc, False, _DATA_KEY_OPTS) - return None + return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) async for doc in cursor] def spawn(self) -> None: """Spawn mongocryptd. @@ -551,7 +549,7 @@ def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions: # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms. if kwargs.get("key_expiration_ms") is None: kwargs.pop("key_expiration_ms", None) - return MongoCryptOptions(**kwargs) + return MongoCryptOptions(**kwargs, enable_multiple_collinfo=True) class AsyncClientEncryption(Generic[_DocumentType]): diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 724131fa9..ebffc7d74 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -241,7 +241,7 @@ class _EncryptionIO(MongoCryptCallback): # type: ignore[misc] ) raise exc from final_err - def collection_info(self, database: str, filter: bytes) -> Optional[bytes]: + def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]]: """Get the collection info for a namespace. The returned collection info is passed to libmongocrypt which reads @@ -250,12 +250,10 @@ class _EncryptionIO(MongoCryptCallback): # type: ignore[misc] :param database: The database on which to run listCollections. :param filter: The filter to pass to listCollections. - :return: The first document from the listCollections command response as BSON. + :return: All documents from the listCollections command response as BSON. """ with self.client_ref()[database].list_collections(filter=RawBSONDocument(filter)) as cursor: - for doc in cursor: - return _dict_to_bson(doc, False, _DATA_KEY_OPTS) - return None + return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) for doc in cursor] def spawn(self) -> None: """Spawn mongocryptd. @@ -548,7 +546,7 @@ def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions: # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms. if kwargs.get("key_expiration_ms") is None: kwargs.pop("key_expiration_ms", None) - return MongoCryptOptions(**kwargs) + return MongoCryptOptions(**kwargs, enable_multiple_collinfo=True) class ClientEncryption(Generic[_DocumentType]): diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 728a6f913..3b9096ef6 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -73,7 +73,7 @@ from test.utils_shared import ( is_greenthread_patched, ) -from bson import DatetimeMS, Decimal128, encode, json_util +from bson import BSON, DatetimeMS, Decimal128, encode, json_util from bson.binary import UUID_SUBTYPE, Binary, UuidRepresentation from bson.codec_options import CodecOptions from bson.errors import BSONError @@ -94,6 +94,7 @@ from pymongo.errors import ( EncryptionError, InvalidOperation, OperationFailure, + PyMongoError, ServerSelectionTimeoutError, WriteError, ) @@ -2419,6 +2420,310 @@ class TestExplicitQueryableEncryption(AsyncEncryptionIntegrationTest): self.assertEqual(decrypted, val) +# https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup +class TestLookupProse(AsyncEncryptionIntegrationTest): + @async_client_context.require_no_standalone + @async_client_context.require_version_min(7, 0, -1) + async def asyncSetUp(self): + await super().asyncSetUp() + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + await encrypted_client.drop_database("db") + + key_doc = json_data("etc", "data", "lookup", "key-doc.json") + await create_key_vault(encrypted_client.db.keyvault, key_doc) + self.addAsyncCleanup(async_client_context.client.drop_database, "db") + + await encrypted_client.db.create_collection( + "csfle", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, + ) + await encrypted_client.db.create_collection( + "csfle2", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, + ) + await encrypted_client.db.create_collection( + "qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json") + ) + await encrypted_client.db.create_collection( + "qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json") + ) + await encrypted_client.db.create_collection("no_schema") + await encrypted_client.db.create_collection("no_schema2") + + unencrypted_client = await self.async_rs_or_single_client() + + await encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = await unencrypted_client.db.csfle.find_one() + self.assertTrue(isinstance(doc["csfle"], Binary)) + await encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = await unencrypted_client.db.csfle2.find_one() + self.assertTrue(isinstance(doc["csfle2"], Binary)) + await encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = await unencrypted_client.db.qe.find_one() + self.assertTrue(isinstance(doc["qe"], Binary)) + await encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = await unencrypted_client.db.qe2.find_one() + self.assertTrue(isinstance(doc["qe2"], Binary)) + await encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + await encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + + await encrypted_client.close() + await unencrypted_client.close() + + @async_client_context.require_version_min(8, 1, -1) + async def test_1_csfle_joins_no_schema(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_2_qe_joins_no_schema(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_3_no_schema_joins_csfle(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "csfle", + "as": "matched", + "pipeline": [{"$match": {"csfle": "csfle"}}, {"$project": {"_id": 0}}], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_4_no_schema_joins_qe(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "qe", + "as": "matched", + "pipeline": [ + {"$match": {"qe": "qe"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_5_csfle_joins_csfle2(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "csfle2", + "as": "matched", + "pipeline": [ + {"$match": {"csfle2": "csfle2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_6_qe_joins_qe2(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "qe2", + "as": "matched", + "pipeline": [ + {"$match": {"qe2": "qe2"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_7_no_schema_joins_no_schema2(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = await anext( + await encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "no_schema2", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema2": "no_schema2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) + + @async_client_context.require_version_min(8, 1, -1) + async def test_8_csfle_joins_qe(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + with self.assertRaises(PyMongoError) as exc: + _ = await anext( + await encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "qe"}}, + { + "$lookup": { + "from": "qe", + "as": "matched", + "pipeline": [{"$match": {"qe": "qe"}}, {"$project": {"_id": 0}}], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertTrue("not supported" in str(exc)) + + @async_client_context.require_version_max(8, 1, -1) + async def test_9_error(self): + encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + with self.assertRaises(PyMongoError) as exc: + _ = await anext( + await encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertTrue("Upgrade" in str(exc)) + + # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap class TestRewrapWithSeparateClientEncryption(AsyncEncryptionIntegrationTest): MASTER_KEYS: Mapping[str, Mapping[str, Any]] = { diff --git a/test/client-side-encryption/etc/data/lookup/key-doc.json b/test/client-side-encryption/etc/data/lookup/key-doc.json new file mode 100644 index 000000000..566b56c35 --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/key-doc.json @@ -0,0 +1,30 @@ +{ + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-csfle.json b/test/client-side-encryption/etc/data/lookup/schema-csfle.json new file mode 100644 index 000000000..29ac9ad5d --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-csfle.json @@ -0,0 +1,19 @@ +{ + "properties": { + "csfle": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-csfle2.json b/test/client-side-encryption/etc/data/lookup/schema-csfle2.json new file mode 100644 index 000000000..3f1c02781 --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-csfle2.json @@ -0,0 +1,19 @@ +{ + "properties": { + "csfle2": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-qe.json b/test/client-side-encryption/etc/data/lookup/schema-qe.json new file mode 100644 index 000000000..9428ea1b4 --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-qe.json @@ -0,0 +1,20 @@ +{ + "escCollection": "enxcol_.qe.esc", + "ecocCollection": "enxcol_.qe.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "qe", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": 0 + } + } + ] +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-qe2.json b/test/client-side-encryption/etc/data/lookup/schema-qe2.json new file mode 100644 index 000000000..77d5bd37c --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-qe2.json @@ -0,0 +1,20 @@ +{ + "escCollection": "enxcol_.qe2.esc", + "ecocCollection": "enxcol_.qe2.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "qe2", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": 0 + } + } + ] +} diff --git a/test/test_encryption.py b/test/test_encryption.py index 36c0ab0e2..6d669a538 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -73,7 +73,7 @@ from test.utils_shared import ( ) from test.utils_spec_runner import SpecRunner -from bson import DatetimeMS, Decimal128, encode, json_util +from bson import BSON, DatetimeMS, Decimal128, encode, json_util from bson.binary import UUID_SUBTYPE, Binary, UuidRepresentation from bson.codec_options import CodecOptions from bson.errors import BSONError @@ -91,6 +91,7 @@ from pymongo.errors import ( EncryptionError, InvalidOperation, OperationFailure, + PyMongoError, ServerSelectionTimeoutError, WriteError, ) @@ -2403,6 +2404,310 @@ class TestExplicitQueryableEncryption(EncryptionIntegrationTest): self.assertEqual(decrypted, val) +# https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup +class TestLookupProse(EncryptionIntegrationTest): + @client_context.require_no_standalone + @client_context.require_version_min(7, 0, -1) + def setUp(self): + super().setUp() + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + encrypted_client.drop_database("db") + + key_doc = json_data("etc", "data", "lookup", "key-doc.json") + create_key_vault(encrypted_client.db.keyvault, key_doc) + self.addCleanup(client_context.client.drop_database, "db") + + encrypted_client.db.create_collection( + "csfle", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, + ) + encrypted_client.db.create_collection( + "csfle2", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, + ) + encrypted_client.db.create_collection( + "qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json") + ) + encrypted_client.db.create_collection( + "qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json") + ) + encrypted_client.db.create_collection("no_schema") + encrypted_client.db.create_collection("no_schema2") + + unencrypted_client = self.rs_or_single_client() + + encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = unencrypted_client.db.csfle.find_one() + self.assertTrue(isinstance(doc["csfle"], Binary)) + encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = unencrypted_client.db.csfle2.find_one() + self.assertTrue(isinstance(doc["csfle2"], Binary)) + encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = unencrypted_client.db.qe.find_one() + self.assertTrue(isinstance(doc["qe"], Binary)) + encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = unencrypted_client.db.qe2.find_one() + self.assertTrue(isinstance(doc["qe2"], Binary)) + encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + + encrypted_client.close() + unencrypted_client.close() + + @client_context.require_version_min(8, 1, -1) + def test_1_csfle_joins_no_schema(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_2_qe_joins_no_schema(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_3_no_schema_joins_csfle(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "csfle", + "as": "matched", + "pipeline": [{"$match": {"csfle": "csfle"}}, {"$project": {"_id": 0}}], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_4_no_schema_joins_qe(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "qe", + "as": "matched", + "pipeline": [ + {"$match": {"qe": "qe"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_5_csfle_joins_csfle2(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "csfle2", + "as": "matched", + "pipeline": [ + {"$match": {"csfle2": "csfle2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_6_qe_joins_qe2(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "qe2", + "as": "matched", + "pipeline": [ + {"$match": {"qe2": "qe2"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_7_no_schema_joins_no_schema2(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "no_schema2", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema2": "no_schema2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) + + @client_context.require_version_min(8, 1, -1) + def test_8_csfle_joins_qe(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + with self.assertRaises(PyMongoError) as exc: + _ = next( + encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "qe"}}, + { + "$lookup": { + "from": "qe", + "as": "matched", + "pipeline": [{"$match": {"qe": "qe"}}, {"$project": {"_id": 0}}], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertTrue("not supported" in str(exc)) + + @client_context.require_version_max(8, 1, -1) + def test_9_error(self): + encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + with self.assertRaises(PyMongoError) as exc: + _ = next( + encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertTrue("Upgrade" in str(exc)) + + # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap class TestRewrapWithSeparateClientEncryption(EncryptionIntegrationTest): MASTER_KEYS: Mapping[str, Mapping[str, Any]] = { diff --git a/uv.lock b/uv.lock index 7dae5c913..758036fe5 100644 --- a/uv.lock +++ b/uv.lock @@ -1135,7 +1135,7 @@ wheels = [ [[package]] name = "pymongocrypt" version = "1.13.0.dev0" -source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1e96c283162aa7789cf01f99f211e0ace8e6d49f" } +source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1cad4ad1c4cd6c11c6a4710da2127dab6a374471" } dependencies = [ { name = "cffi" }, { name = "cryptography" },