mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:mongodb/mongo-python-driver

Browse Source
This commit is contained in:
Steven Silvester 2025-02-25 18:20:01 -06:00
commit bab98496ea
No known key found for this signature in database
GPG Key ID: B1BF5EC3A8B32F91
32 changed files with 351 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
.evergreen/config.yml
View File

@ -265,13 +265,13 @@ functions:
DISABLE_TEST_COMMANDS, GREEN_FRAMEWORK, NO_EXT, COMPRESSORS, MONGODB_API_VERSION]
binary: bash
working_dir: "src"
args: [.evergreen/just.sh, setup-test, "${TEST_NAME}", "${SUB_TEST_NAME}"]
args: [.evergreen/just.sh, setup-tests, "${TEST_NAME}", "${SUB_TEST_NAME}"]
- command: subprocess.exec
type: test
params:
working_dir: "src"
binary: bash
args: [.evergreen/just.sh, test-eg]
args: [.evergreen/just.sh, run-tests]
"run enterprise auth tests":
- command: subprocess.exec
@ -443,13 +443,12 @@ functions:
binary: bash
working_dir: "src"
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
- ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh
- command: subprocess.exec
params:
binary: bash
working_dir: "src"
args:
- ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh
binary: bash
working_dir: "src"
args: [.evergreen/just.sh, teardown-tests]
- command: subprocess.exec
params:
binary: bash
@ -562,51 +561,6 @@ task_groups:
tasks:
- ".serverless"
- name: testgcpkms_task_group
setup_group_can_fail_task: true
setup_group_timeout_secs: 1800 # 30 minutes
setup_group:
- func: fetch source
- func: setup system
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
teardown_task:
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh
- func: "upload test results"
tasks:
- testgcpkms-task
- name: testazurekms_task_group
setup_group:
- func: fetch source
- func: setup system
- command: subprocess.exec
params:
binary: bash
env:
AZUREKMS_VMNAME_PREFIX: "PYTHON_DRIVER"
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh
teardown_group:
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/delete-vm.sh
- func: "upload test results"
setup_group_can_fail_task: true
teardown_task_can_fail_task: true
setup_group_timeout_secs: 1800
tasks:
- testazurekms-task
- name: testazureoidc_task_group
setup_group:
- func: fetch source
@ -1395,60 +1349,6 @@ tasks:
commands:
- func: "download and merge coverage"
- name: "testgcpkms-task"
commands:
- command: subprocess.exec
type: test
params:
working_dir: "src"
binary: bash
include_expansions_in_env: ["DRIVERS_TOOLS"]
args:
- .evergreen/run-gcpkms-test.sh
- name: "testgcpkms-fail-task"
# testgcpkms-fail-task runs in a non-GCE environment.
# It is expected to fail to obtain GCE credentials.
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "latest"
TOPOLOGY: "server"
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["PYTHON_BINARY"]
working_dir: "src"
binary: "bash"
args:
- .evergreen/scripts/run-gcpkms-fail-test.sh
- name: testazurekms-task
commands:
- command: subprocess.exec
type: test
params:
binary: bash
working_dir: src
include_expansions_in_env: ["DRIVERS_TOOLS"]
args:
- .evergreen/run-azurekms-test.sh
- name: testazurekms-fail-task
commands:
- func: "bootstrap mongo-orchestration"
vars:
VERSION: "latest"
TOPOLOGY: "server"
- command: subprocess.exec
type: test
params:
binary: bash
working_dir: src
include_expansions_in_env: ["DRIVERS_TOOLS"]
args:
- .evergreen/run-azurekms-fail-test.sh
- name: "perf-6.0-standalone"
tags: ["perf"]
commands:
@ -1529,12 +1429,12 @@ buildvariants:
run_on:
- debian11-small
tasks:
- name: testgcpkms_task_group
- name: test-gcpkms
batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
- testgcpkms-fail-task
- name: testazurekms_task_group
- name: test-gcpkms-fail
- name: test-azurekms
batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
- testazurekms-fail-task
- name: test-azurekms-fail
- name: rhel8-test-lambda
display_name: FaaS Lambda

28
.evergreen/generated_configs/tasks.yml
View File

@ -1,4 +1,32 @@
tasks:
# Kms tests
- name: test-gcpkms
commands:
- func: run tests
vars:
TEST_NAME: kms
SUB_TEST_NAME: gcp
- name: test-gcpkms-fail
commands:
- func: bootstrap mongo-orchestration
- func: run tests
vars:
TEST_NAME: kms
SUB_TEST_NAME: gcp-fail
- name: test-azurekms
commands:
- func: run tests
vars:
TEST_NAME: kms
SUB_TEST_NAME: azure
- name: test-azurekms-fail
commands:
- func: bootstrap mongo-orchestration
- func: run tests
vars:
TEST_NAME: kms
SUB_TEST_NAME: azure-fail
# Load balancer tests
- name: test-load-balancer-auth-ssl
commands:

9
.evergreen/run-azurekms-fail-test.sh
View File

@ -1,9 +0,0 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
HERE=$(dirname ${BASH_SOURCE:-$0})
. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
bash $HERE/just.sh setup-test kms azure-fail
KEY_NAME="${AZUREKMS_KEYNAME}" \
KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
$HERE/just.sh test-eg
bash $HERE/scripts/teardown-tests.sh

27
.evergreen/run-azurekms-test.sh
View File

@ -1,27 +0,0 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
HERE=$(dirname ${BASH_SOURCE:-$0})
source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh
echo "Copying files ... begin"
export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
# Set up the remote files to test.
git add .
git commit -m "add files" || true
git archive -o /tmp/mongo-python-driver.tgz HEAD
# shellcheck disable=SC2088
AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
echo "Untarring file ... end"
echo "Running test ... begin"
AZUREKMS_CMD="bash .evergreen/just.sh setup-test kms azure" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
echo "Running test ... end"
bash $HERE/scripts/teardown-tests.sh

24
.evergreen/run-gcpkms-test.sh
View File

@ -1,24 +0,0 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
HERE=$(dirname ${BASH_SOURCE:-$0})
source ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/secrets-export.sh
echo "Copying files ... begin"
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
# Set up the remote files to test.
git add .
git commit -m "add files" || true
git archive -o /tmp/mongo-python-driver.tgz HEAD
GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Untarring file ... end"
echo "Running test ... begin"
GCPKMS_CMD="bash ./.evergreen/just.sh setup-test kms gcp" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Running test ... end"
bash $HERE/scripts/teardown-tests.sh

4
.evergreen/run-mongodb-aws-ecs-test.sh
View File

@ -30,5 +30,5 @@ export SET_XTRACE_ON=1
cd src
rm -rf .venv
rm -f .evergreen/scripts/test-env.sh || true
bash ./.evergreen/just.sh setup-test auth_aws ecs
bash .evergreen/just.sh test-eg
bash ./.evergreen/just.sh setup-tests auth_aws ecs
bash .evergreen/just.sh run-tests

4
.evergreen/run-mongodb-oidc-test.sh
View File

@ -29,5 +29,5 @@ else
exit 1
fi
COVERAGE=1 bash ./.evergreen/just.sh setup-test auth_oidc
bash ./.evergreen/just.sh test-eg "${@:1}"
COVERAGE=1 bash ./.evergreen/just.sh setup-tests auth_oidc
bash ./.evergreen/just.sh run-tests "${@:1}"

4
.evergreen/run-perf-tests.sh
View File

@ -15,5 +15,5 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
bash ./.evergreen/just.sh setup-test perf
bash ./.evergreen/just.sh test-eg
bash ./.evergreen/just.sh setup-tests perf
bash ./.evergreen/just.sh run-tests

2
.evergreen/run-tests.sh
View File

@ -20,7 +20,7 @@ if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then
echo "Sourcing test inputs"
. $SCRIPT_DIR/scripts/test-env.sh
else
echo "Missing test inputs, please run 'just setup-test'"
echo "Missing test inputs, please run 'just setup-tests'"
exit 1
fi

0
.evergreen/scripts/__init__.py Normal file
View File

8
.evergreen/scripts/bootstrap-mongo-orchestration.sh
View File

@ -34,10 +34,10 @@ if [ -z "${TEST_CRYPT_SHARED:-}" ]; then
export SKIP_CRYPT_SHARED=1
fi
MONGODB_VERSION=${VERSION} \
TOPOLOGY=${TOPOLOGY} \
AUTH=${AUTH:-noauth} \
SSL=${SSL:-nossl} \
MONGODB_VERSION=${VERSION:-} \
TOPOLOGY=${TOPOLOGY:-} \
AUTH=${AUTH:-} \
SSL=${SSL:-} \
STORAGE_ENGINE=${STORAGE_ENGINE:-} \
DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \
ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \

3
.evergreen/scripts/cleanup.sh
View File

@ -1,7 +1,4 @@
#!/bin/bash
if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
bash .evergreen/teardown-encryption.sh
fi
rm -rf "${DRIVERS_TOOLS}" || true
rm -f ./secrets-export.sh || true

19
.evergreen/scripts/generate_config.py
View File

@ -835,6 +835,25 @@ def create_load_balancer_tasks():
return tasks
def create_kms_tasks():
tasks = []
for kms_type in ["gcp", "azure"]:
for success in [True, False]:
name = f"test-{kms_type}kms"
sub_test_name = kms_type
if not success:
name += "-fail"
sub_test_name += "-fail"
commands = []
if not success:
commands.append(FunctionCall(func="bootstrap mongo-orchestration"))
test_vars = dict(TEST_NAME="kms", SUB_TEST_NAME=sub_test_name)
test_func = FunctionCall(func="run tests", vars=test_vars)
commands.append(test_func)
tasks.append(EvgTask(name=name, commands=commands))
return tasks
##################
# Generate Config
##################

130
.evergreen/scripts/kms_tester.py Normal file
View File

@ -0,0 +1,130 @@
from __future__ import annotations
import os
from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env
TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108
DIRS = dict(
gcp=f"{DRIVERS_TOOLS}/.evergreen/csfle/gcpkms",
azure=f"{DRIVERS_TOOLS}/.evergreen/csfle/azurekms",
)
def _setup_azure_vm(base_env: dict[str, str]) -> None:
LOGGER.info("Setting up Azure VM...")
azure_dir = DIRS["azure"]
env = base_env.copy()
env["AZUREKMS_SRC"] = TMP_DRIVER_FILE
env["AZUREKMS_DST"] = "~/"
run_command(f"{azure_dir}/copy-file.sh", env=env)
env = base_env.copy()
env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz"
run_command(f"{azure_dir}/run-command.sh", env=env)
env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote"
run_command(f"{azure_dir}/run-command.sh", env=env)
LOGGER.info("Setting up Azure VM... done.")
def _setup_gcp_vm(base_env: dict[str, str]) -> None:
LOGGER.info("Setting up GCP VM...")
gcp_dir = DIRS["gcp"]
env = base_env.copy()
env["GCPKMS_SRC"] = TMP_DRIVER_FILE
env["GCPKMS_DST"] = f"{env['GCPKMS_INSTANCENAME']}:"
run_command(f"{gcp_dir}/copy-file.sh", env=env)
env = base_env.copy()
env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz"
run_command(f"{gcp_dir}/run-command.sh", env=env)
env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote"
run_command(f"{gcp_dir}/run-command.sh", env=env)
LOGGER.info("Setting up GCP VM...")
def _create_archive() -> None:
run_command("git add .", cwd=ROOT)
run_command('git commit -m "add files"', check=False, cwd=ROOT)
run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT)
def _load_kms_config(sub_test_target: str) -> dict[str, str]:
target_dir = DIRS[sub_test_target]
config = read_env(f"{target_dir}/secrets-export.sh")
base_env = os.environ.copy()
for key, value in config.items():
base_env[key] = str(value)
return base_env
def setup_kms(sub_test_name: str) -> None:
if "-" in sub_test_name:
sub_test_target, sub_test_type = sub_test_name.split("-")
else:
sub_test_target = sub_test_name
sub_test_type = ""
assert sub_test_target in ["azure", "gcp"], sub_test_target
assert sub_test_type in ["", "remote", "fail"], sub_test_type
success = sub_test_type != "fail"
kms_dir = DIRS[sub_test_target]
if sub_test_target == "azure":
write_env("TEST_FLE_AZURE_AUTO")
else:
write_env("TEST_FLE_GCP_AUTO")
write_env("SUCCESS", success)
# For remote tests, there is no further work required.
if sub_test_type == "remote":
return
if sub_test_target == "azure":
run_command("./setup-secrets.sh", cwd=kms_dir)
if success:
_create_archive()
if sub_test_target == "azure":
os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER"
run_command("./setup.sh", cwd=kms_dir)
base_env = _load_kms_config(sub_test_target)
if sub_test_target == "azure":
_setup_azure_vm(base_env)
else:
_setup_gcp_vm(base_env)
if sub_test_target == "azure":
config = read_env(f"{kms_dir}/secrets-export.sh")
if success:
write_env("AZUREKMS_VMNAME", config["AZUREKMS_VMNAME"])
write_env("KEY_NAME", config["AZUREKMS_KEYNAME"])
write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"])
def test_kms_remote(sub_test_name: str) -> None:
env = _load_kms_config(sub_test_name)
if sub_test_name == "azure":
key_name = os.environ["KEY_NAME"]
key_vault_endpoint = os.environ["KEY_VAULT_ENDPOINT"]
env[
"AZUREKMS_CMD"
] = f'KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh run-tests'
else:
env["GCPKMS_CMD"] = "./.evergreen/just.sh run-tests"
cmd = f"{DIRS[sub_test_name]}/run-command.sh"
run_command(cmd, env=env)
def teardown_kms(sub_test_name: str) -> None:
run_command(f"{DIRS[sub_test_name]}/teardown.sh")
if __name__ == "__main__":
setup_kms()

4
.evergreen/scripts/run-atlas-tests.sh
View File

@ -4,5 +4,5 @@
set +x
set -o errexit
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test atlas
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-tests atlas
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh run-tests

4
.evergreen/scripts/run-enterprise-auth-tests.sh
View File

@ -5,5 +5,5 @@ set -eu
set +x
# Use the default python to bootstrap secrets.
bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test enterprise_auth
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-tests enterprise_auth
bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh run-tests

6
.evergreen/scripts/run-gcpkms-fail-test.sh
View File

@ -1,6 +0,0 @@
#!/bin/bash
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
. $HERE/env.sh
./.evergreen/just.sh setup-test kms gcp-fail
bash ./.evergreen/just.sh test-eg

4
.evergreen/scripts/run-mongodb-aws-test.sh
View File

@ -24,5 +24,5 @@ echo "Running MONGODB-AWS authentication tests for $1"
# Handle credentials and environment setup.
. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
bash ./.evergreen/just.sh setup-test auth_aws $1
bash ./.evergreen/just.sh test-eg
bash ./.evergreen/just.sh setup-tests auth_aws $1
bash ./.evergreen/just.sh run-tests

19
.evergreen/scripts/run_tests.py
View File

@ -1,27 +1,23 @@
from __future__ import annotations
import json
import logging
import os
import platform
import shutil
import sys
from datetime import datetime
from pathlib import Path
import pytest
from utils import LOGGER, ROOT
HERE = Path(__file__).absolute().parent
ROOT = HERE.parent.parent
AUTH = os.environ.get("AUTH", "noauth")
SSL = os.environ.get("SSL", "nossl")
UV_ARGS = os.environ.get("UV_ARGS", "")
TEST_PERF = os.environ.get("TEST_PERF")
GREEN_FRAMEWORK = os.environ.get("GREEN_FRAMEWORK")
TEST_ARGS = os.environ.get("TEST_ARGS", "").split()
LOGGER = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
TEST_NAME = os.environ.get("TEST_NAME")
SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME")
def handle_perf(start_time: datetime):
@ -103,7 +99,14 @@ def run() -> None:
if TEST_PERF:
start_time = datetime.now()
# Run the tests.
# Run remote kms tests.
if TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]:
from kms_tester import test_kms_remote
test_kms_remote(SUB_TEST_NAME)
return
# Run local tests.
pytest.main(TEST_ARGS)
# Handle perf test post actions.

2
.evergreen/scripts/setup-dev-env.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
set -eux
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"

82
.evergreen/scripts/setup_tests.py
View File

@ -2,29 +2,28 @@ from __future__ import annotations
import argparse
import base64
import dataclasses
import io
import logging
import os
import platform
import shlex
import shutil
import stat
import subprocess
import sys
import tarfile
from pathlib import Path
from typing import Any
from urllib import request
HERE = Path(__file__).absolute().parent
ROOT = HERE.parent.parent
ENV_FILE = HERE / "test-env.sh"
DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
PLATFORM = "windows" if os.name == "nt" else sys.platform.lower()
LOGGER = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
from utils import (
DRIVERS_TOOLS,
ENV_FILE,
HERE,
LOGGER,
PLATFORM,
ROOT,
Distro,
read_env,
run_command,
write_env,
)
# Passthrough environment variables.
PASS_THROUGH_ENV = ["GREEN_FRAMEWORK", "NO_EXT", "MONGODB_API_VERSION"]
@ -41,7 +40,7 @@ TEST_SUITE_MAP = {
"encryption": "encryption",
"enterprise_auth": "auth",
"index_management": "index_management",
"kms": "csfle",
"kms": "kms",
"load_balancer": "load_balancer",
"mockupdb": "mockupdb",
"pyopenssl": "",
@ -69,51 +68,23 @@ EXTRAS_MAP = {
GROUP_MAP = dict(mockupdb="mockupdb", perf="perf")
@dataclasses.dataclass
class Distro:
name: str
version_id: str
arch: str
def write_env(name: str, value: Any = "1") -> None:
with ENV_FILE.open("a", newline="\n") as fid:
# Remove any existing quote chars.
value = str(value).replace('"', "")
fid.write(f'export {name}="{value}"\n')
def is_set(var: str) -> bool:
value = os.environ.get(var, "")
return len(value.strip()) > 0
def run_command(cmd: str) -> None:
LOGGER.info("Running command %s...", cmd)
subprocess.check_call(shlex.split(cmd)) # noqa: S603
LOGGER.info("Running command %s... done.", cmd)
def read_env(path: Path | str) -> dict[str, Any]:
config = dict()
with Path(path).open() as fid:
for line in fid.readlines():
if "=" not in line:
continue
name, _, value = line.strip().partition("=")
if value.startswith(('"', "'")):
value = value[1:-1]
name = name.replace("export ", "")
config[name] = value
return config
def get_options():
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("test_name", choices=sorted(TEST_SUITE_MAP), nargs="?", default="default")
parser.add_argument("sub_test_name", nargs="?")
parser.add_argument(
"test_name",
choices=sorted(TEST_SUITE_MAP),
nargs="?",
default="default",
help="The name of the test suite to set up, typically the same name as a pytest marker.",
)
parser.add_argument("sub_test_name", nargs="?", help="The sub test name, for example 'azure'")
parser.add_argument(
"--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level"
)
@ -238,6 +209,7 @@ def handle_test_env() -> None:
# Set an environment variable for the test name and sub test name.
write_env(f"TEST_{test_name.upper()}")
write_env("TEST_NAME", test_name)
write_env("SUB_TEST_NAME", sub_test_name)
# Handle pass through env vars.
@ -388,15 +360,9 @@ def handle_test_env() -> None:
write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}")
if test_name == "kms":
if sub_test_name.startswith("azure"):
write_env("TEST_FLE_AZURE_AUTO")
else:
write_env("TEST_FLE_GCP_AUTO")
from kms_tester import setup_kms
write_env("SUCCESS", "fail" not in sub_test_name)
MONGODB_URI = os.environ.get("MONGODB_URI", "")
if "@" in MONGODB_URI:
raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!")
setup_kms(sub_test_name)
if test_name == "ocsp":
write_env("CA_FILE", os.environ["CA_FILE"])

34
.evergreen/scripts/teardown-tests.sh
View File

@ -2,28 +2,28 @@
set -eu
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")"
SCRIPT_DIR="$( cd -- "$SCRIPT_DIR" > /dev/null 2>&1 && pwd )"
ROOT_DIR="$(dirname $SCRIPT_DIR)"
# Remove temporary test files.
pushd $ROOT_DIR > /dev/null
rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid > /dev/null
popd > /dev/null
if [ ! -f $SCRIPT_DIR/test-env.sh ]; then
exit 0
fi
# Try to source the env file.
if [ -f $SCRIPT_DIR/env.sh ]; then
source $SCRIPT_DIR/env.sh
echo "Sourcing env inputs"
. $SCRIPT_DIR/env.sh
else
echo "Not sourcing env inputs"
fi
source $SCRIPT_DIR/test-env.sh
# Shut down csfle servers if applicable
if [ -n "${TEST_ENCRYPTION:-}" ]; then
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
# Handle test inputs.
if [ -f $SCRIPT_DIR/test-env.sh ]; then
echo "Sourcing test inputs"
. $SCRIPT_DIR/test-env.sh
else
echo "Missing test inputs, please run 'just setup-tests'"
fi
# Shut down load balancer if applicable.
if [ -n "${TEST_LOAD_BALANCER:-}" ]; then
bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop
fi
# Teardown the test runner.
uv run $SCRIPT_DIR/teardown_tests.py
popd > /dev/null

26
.evergreen/scripts/teardown_tests.py Normal file
View File

@ -0,0 +1,26 @@
from __future__ import annotations
import os
from utils import DRIVERS_TOOLS, LOGGER, run_command
TEST_NAME = os.environ.get("TEST_NAME", "unconfigured")
SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME")
LOGGER.info(f"Tearing down tests of type '{TEST_NAME}'...")
# Shut down csfle servers if applicable
if TEST_NAME == "encryption":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh")
# Shut down load balancer if applicable.
elif TEST_NAME == "load-balancer":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop")
# Tear down kms VM if applicable.
elif TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]:
from kms_tester import teardown_kms
teardown_kms(SUB_TEST_NAME)
LOGGER.info(f"Tearing down tests of type '{TEST_NAME}'... done.")

54
.evergreen/scripts/utils.py Normal file
View File

@ -0,0 +1,54 @@
from __future__ import annotations
import dataclasses
import logging
import os
import shlex
import subprocess
import sys
from pathlib import Path
from typing import Any
HERE = Path(__file__).absolute().parent
ROOT = HERE.parent.parent
DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
LOGGER = logging.getLogger("test")
logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
ENV_FILE = HERE / "test-env.sh"
PLATFORM = "windows" if os.name == "nt" else sys.platform.lower()
@dataclasses.dataclass
class Distro:
name: str
version_id: str
arch: str
def read_env(path: Path | str) -> dict[str, Any]:
config = dict()
with Path(path).open() as fid:
for line in fid.readlines():
if "=" not in line:
continue
name, _, value = line.strip().partition("=")
if value.startswith(('"', "'")):
value = value[1:-1]
name = name.replace("export ", "")
config[name] = value
return config
def write_env(name: str, value: Any = "1") -> None:
with ENV_FILE.open("a", newline="\n") as fid:
# Remove any existing quote chars.
value = str(value).replace('"', "")
fid.write(f'export {name}="{value}"\n')
def run_command(cmd: str, **kwargs: Any) -> None:
LOGGER.info("Running command %s...", cmd)
kwargs.setdefault("check", True)
subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603
LOGGER.info("Running command %s... done.", cmd)

10
CONTRIBUTING.md
View File

@ -212,17 +212,17 @@ the pages will re-render and the browser will automatically refresh.
- Start the servers using
`LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh`.
- Set up the test using:
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' just setup-test load-balancer`.
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' just setup-tests load-balancer`.
- Run the tests from the `pymongo` checkout directory using:
`just test-eg`.
`just run-tests`.
## Running Encryption Tests Locally
- Clone `drivers-evergreen-tools`:
`git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`.
- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`
- Run `AWS_PROFILE=<profile> just setup-test encryption` after setting up your AWS profile with `aws configure sso`.
- Run the tests with `just test-eg`.
- When done, run `just teardown-test` to clean up.
- Run `AWS_PROFILE=<profile> just setup-tests encryption` after setting up your AWS profile with `aws configure sso`.
- Run the tests with `just run-tests`.
- When done, run `just teardown-tests` to clean up.
## Re-sync Spec Tests

2
doc/changelog.rst
View File

@ -189,7 +189,7 @@ Issues Resolved
See the `PyMongo 4.9 release notes in JIRA`_ for the list of resolved issues
in this release.
.. _Is PyMongo Fork-Safe : https://www.mongodb.com/docs/languages/python/pymongo-driver/current/faq/#is-pymongo-fork-safe-
.. _Is PyMongo Fork-Safe: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/troubleshooting/#forking-a-process-causes-a-deadlock
.. _PyMongo 4.9 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=39940

6
justfile
View File

@ -61,13 +61,13 @@ test *args="-v --durations=5 --maxfail=10":
{{uv_run}} --extra test pytest {{args}}
[group('test')]
test-eg *args:
run-tests *args:
bash ./.evergreen/run-tests.sh {{args}}
[group('test')]
setup-test *args="":
setup-tests *args="":
bash .evergreen/scripts/setup-tests.sh {{args}}
[group('test')]
teardown-test:
teardown-tests:
bash .evergreen/scripts/teardown-tests.sh

4
pymongo/asynchronous/topology.py
View File

@ -236,9 +236,7 @@ class Topology:
warnings.warn( # type: ignore[call-overload] # noqa: B028
"AsyncMongoClient opened before fork. May not be entirely fork-safe, "
"proceed with caution. See PyMongo's documentation for details: "
"https://www.mongodb.com/docs/languages/"
"python/pymongo-driver/current/faq/"
"#is-pymongo-fork-safe-",
"https://www.mongodb.com/docs/languages/python/pymongo-driver/current/troubleshooting/#forking-a-process-causes-a-deadlock",
**kwargs,
)
async with self._lock:

4
pymongo/synchronous/topology.py
View File

@ -236,9 +236,7 @@ class Topology:
warnings.warn( # type: ignore[call-overload] # noqa: B028
"MongoClient opened before fork. May not be entirely fork-safe, "
"proceed with caution. See PyMongo's documentation for details: "
"https://www.mongodb.com/docs/languages/"
"python/pymongo-driver/current/faq/"
"#is-pymongo-fork-safe-",
"https://www.mongodb.com/docs/languages/python/pymongo-driver/current/troubleshooting/#forking-a-process-causes-a-deadlock",
**kwargs,
)
with self._lock:

2
pyproject.toml
View File

@ -129,7 +129,7 @@ markers = [
"data_lake: tests that rely on atlas data lake",
"perf: benchmark tests",
"index_management: index management tests",
"csfle: client-side field-level encryption tests",
"kms: client-side field-level encryption tests using kms",
"encryption: encryption tests",
"load_balancer: load balancer tests",
"mockupdb: tests that rely on mockupdb",

2
test/asynchronous/test_on_demand_csfle.py
View File

@ -34,7 +34,7 @@ from pymongo.asynchronous.encryption import (
_IS_SYNC = False
pytestmark = pytest.mark.csfle
pytestmark = pytest.mark.kms
class TestonDemandGCPCredentials(AsyncIntegrationTest):

2
test/test_on_demand_csfle.py
View File

@ -34,7 +34,7 @@ from pymongo.synchronous.encryption import (
_IS_SYNC = True
pytestmark = pytest.mark.csfle
pytestmark = pytest.mark.kms
class TestonDemandGCPCredentials(IntegrationTest):