Merge branch 'master' of github.com:mongodb/mongo-python-driver

This commit is contained in:
Steven Silvester 2025-04-07 15:16:58 -05:00
commit 51afaa8552
No known key found for this signature in database
GPG Key ID: B1BF5EC3A8B32F91
11 changed files with 274 additions and 158 deletions

View File

@ -25,6 +25,7 @@ timeout:
binary: ls -la
include:
- filename: .evergreen/generated_configs/functions.yml
- filename: .evergreen/generated_configs/tasks.yml
- filename: .evergreen/generated_configs/variants.yml
@ -52,147 +53,6 @@ functions:
params:
file: src/expansion.yml
"upload coverage" :
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: src/.coverage
optional: true
# Upload the coverage report for all tasks in a single build to the same directory.
remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
bucket: ${bucket_name}
permissions: public-read
content_type: text/html
display_name: "Raw Coverage Report"
"download and merge coverage" :
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: subprocess.exec
params:
silent: true
binary: bash
working_dir: "src"
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
args:
- .evergreen/scripts/download-and-merge-coverage.sh
- ${bucket_name}
- ${revision}
- ${version_id}
- command: subprocess.exec
params:
working_dir: "src"
binary: bash
args:
- .evergreen/combine-coverage.sh
# Upload the resulting html coverage report.
- command: subprocess.exec
params:
silent: true
binary: bash
working_dir: "src"
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
args:
- .evergreen/scripts/upload-coverage-report.sh
- ${bucket_name}
- ${revision}
- ${version_id}
# Attach the index.html with s3.put so it shows up in the Evergreen UI.
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: src/htmlcov/index.html
remote_file: coverage/${revision}/${version_id}/htmlcov/index.html
bucket: ${bucket_name}
permissions: public-read
content_type: text/html
display_name: "Coverage Report HTML"
"upload mo artifacts":
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: archive.targz_pack
params:
target: "mongo-coredumps.tgz"
source_dir: "./"
include:
- "./**.core"
- "./**.mdmp" # Windows: minidumps
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: mongo-coredumps.tgz
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz
bucket: ${bucket_name}
permissions: public-read
content_type: ${content_type|application/gzip}
display_name: Core Dumps - Execution
optional: true
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz
bucket: ${bucket_name}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "drivers-tools-logs.tar.gz"
"upload working dir":
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: archive.targz_pack
params:
target: "working-dir.tar.gz"
source_dir: ${PROJECT_DIRECTORY}/
include:
- "./**"
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: working-dir.tar.gz
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz
bucket: ${bucket_name}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "working-dir.tar.gz"
- command: archive.targz_pack
params:
target: "drivers-dir.tar.gz"
source_dir: ${DRIVERS_TOOLS}
include:
- "./**"
exclude_files:
# Windows cannot read the mongod *.lock files because they are locked.
- "*.lock"
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: drivers-dir.tar.gz
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz
bucket: ${bucket_name}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "drivers-dir.tar.gz"
"upload test results":
- command: attach.results
params:

View File

@ -0,0 +1,117 @@
functions:
# Download and merge coverage
download and merge coverage:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
type: setup
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/download-and-merge-coverage.sh
- ${bucket_name}
- ${revision}
- ${version_id}
working_dir: src
silent: true
include_expansions_in_env:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
type: test
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/combine-coverage.sh
working_dir: src
type: test
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/upload-coverage-report.sh
- ${bucket_name}
- ${revision}
- ${version_id}
working_dir: src
silent: true
include_expansions_in_env:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
type: test
- command: s3.put
params:
remote_file: coverage/${revision}/${version_id}/htmlcov/index.html
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: src/htmlcov/index.html
permissions: public-read
content_type: text/html
display_name: Coverage Report HTML
optional: "true"
type: setup
# Upload coverage
upload coverage:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
type: setup
- command: s3.put
params:
remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: src/.coverage
permissions: public-read
content_type: text/html
display_name: Raw Coverage Report
optional: "true"
type: setup
# Upload mo artifacts
upload mo artifacts:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
type: setup
- command: archive.targz_pack
params:
target: mongo-coredumps.tgz
source_dir: ./
include:
- ./**.core
- ./**.mdmp
- command: s3.put
params:
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: mongo-coredumps.tgz
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: Core Dumps - Execution
optional: "true"
type: setup
- command: s3.put
params:
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: drivers-tools-logs.tar.gz
optional: "true"
type: setup

View File

@ -9,7 +9,14 @@ from pathlib import Path
from typing import Any
from shrub.v3.evg_build_variant import BuildVariant
from shrub.v3.evg_command import EvgCommandType, FunctionCall, subprocess_exec
from shrub.v3.evg_command import (
EvgCommandType,
FunctionCall,
archive_targz_pack,
ec2_assume_role,
s3_put,
subprocess_exec,
)
from shrub.v3.evg_project import EvgProject
from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef
from shrub.v3.shrub_service import ShrubService
@ -233,6 +240,12 @@ def handle_c_ext(c_ext, expansions) -> None:
expansions["NO_EXT"] = "1"
def get_assume_role(**kwargs):
kwargs.setdefault("command_type", EvgCommandType.SETUP)
kwargs.setdefault("role_arn", "${assume_role_arn}")
return ec2_assume_role(**kwargs)
def get_subprocess_exec(**kwargs):
kwargs.setdefault("binary", "bash")
kwargs.setdefault("working_dir", "src")
@ -240,6 +253,18 @@ def get_subprocess_exec(**kwargs):
return subprocess_exec(**kwargs)
def get_s3_put(**kwargs):
kwargs["aws_key"] = "${AWS_ACCESS_KEY_ID}"
kwargs["aws_secret"] = "${AWS_SECRET_ACCESS_KEY}" # noqa:S105
kwargs["aws_session_token"] = "${AWS_SESSION_TOKEN}" # noqa:S105
kwargs["bucket"] = "${bucket_name}"
kwargs.setdefault("optional", "true")
kwargs.setdefault("permissions", "public-read")
kwargs.setdefault("content_type", "${content_type|application/x-gzip}")
kwargs.setdefault("command_type", EvgCommandType.SETUP)
return s3_put(**kwargs)
def generate_yaml(tasks=None, variants=None):
"""Generate the yaml for a given set of tasks and variants."""
project = EvgProject(tasks=tasks, buildvariants=variants)
@ -1193,6 +1218,79 @@ def create_serverless_tasks():
return [EvgTask(name=task_name, tags=tags, commands=[test_func])]
##############
# Functions
##############
def create_upload_coverage_func():
# Upload the coverage report for all tasks in a single build to the same directory.
remote_file = (
"coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}"
)
display_name = "Raw Coverage Report"
cmd = get_s3_put(
local_file="src/.coverage",
remote_file=remote_file,
display_name=display_name,
content_type="text/html",
)
return "upload coverage", [get_assume_role(), cmd]
def create_download_and_merge_coverage_func():
include_expansions = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
args = [
".evergreen/scripts/download-and-merge-coverage.sh",
"${bucket_name}",
"${revision}",
"${version_id}",
]
merge_cmd = get_subprocess_exec(
silent=True, include_expansions_in_env=include_expansions, args=args
)
combine_cmd = get_subprocess_exec(args=[".evergreen/combine-coverage.sh"])
# Upload the resulting html coverage report.
args = [
".evergreen/scripts/upload-coverage-report.sh",
"${bucket_name}",
"${revision}",
"${version_id}",
]
upload_cmd = get_subprocess_exec(
silent=True, include_expansions_in_env=include_expansions, args=args
)
display_name = "Coverage Report HTML"
remote_file = "coverage/${revision}/${version_id}/htmlcov/index.html"
put_cmd = get_s3_put(
local_file="src/htmlcov/index.html",
remote_file=remote_file,
display_name=display_name,
content_type="text/html",
)
cmds = [get_assume_role(), merge_cmd, combine_cmd, upload_cmd, put_cmd]
return "download and merge coverage", cmds
def create_upload_mo_artifacts_func():
include = ["./**.core", "./**.mdmp"] # Windows: minidumps
archive_cmd = archive_targz_pack(target="mongo-coredumps.tgz", source_dir="./", include=include)
display_name = "Core Dumps - Execution"
remote_file = "${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz"
s3_dumps = get_s3_put(
local_file="mongo-coredumps.tgz", remote_file=remote_file, display_name=display_name
)
display_name = "drivers-tools-logs.tar.gz"
remote_file = "${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz"
s3_logs = get_s3_put(
local_file="${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz",
remote_file=remote_file,
display_name=display_name,
)
cmds = [get_assume_role(), archive_cmd, s3_dumps, s3_logs]
return "upload mo artifacts", cmds
##################
# Generate Config
##################
@ -1258,5 +1356,40 @@ def write_tasks_to_file():
fid.write(f"{line}\n")
def write_functions_to_file():
mod = sys.modules[__name__]
here = Path(__file__).absolute().parent
target = here.parent / "generated_configs" / "functions.yml"
if target.exists():
target.unlink()
with target.open("w") as fid:
fid.write("functions:\n")
functions = dict()
for name, func in sorted(getmembers(mod, isfunction)):
if name.startswith("_") or not name.endswith("_func"):
continue
if not name.startswith("create_"):
raise ValueError("Function creators must start with create_")
title = name.replace("create_", "").replace("_func", "").replace("_", " ").capitalize()
func_name, cmds = func()
functions = dict()
functions[func_name] = cmds
project = EvgProject(functions=functions, tasks=None, buildvariants=None)
out = ShrubService.generate_yaml(project).splitlines()
with target.open("a") as fid:
fid.write(f" # {title}\n")
for line in out[1:]:
fid.write(f"{line}\n")
fid.write("\n")
# Remove extra trailing newline:
data = target.read_text().splitlines()
with target.open("w") as fid:
for line in data[:-1]:
fid.write(f"{line}\n")
write_variants_to_file()
write_tasks_to_file()
write_functions_to_file()

View File

@ -39,7 +39,7 @@ function _pip_install() {
# Ensure just is installed.
if ! command -v just 2>/dev/null; then
if ! command -v just >/dev/null 2>&1; then
# On most systems we can install directly.
_TARGET=""
if [ "Windows_NT" = "${OS:-}" ]; then
@ -54,7 +54,7 @@ if ! command -v just 2>/dev/null; then
fi
# Install uv.
if ! command -v uv 2>/dev/null; then
if ! command -v uv >/dev/null 2>&1; then
echo "Installing uv..."
# On most systems we can install directly.
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {

View File

@ -11,7 +11,7 @@ pushd $ROOT > /dev/null
if [ -f $HERE/env.sh ]; then
. $HERE/env.sh
fi
# PYTHON_BINARY may be defined in test-env.sh.
# PYTHON_BINARY or PYTHON_VERSION may be defined in test-env.sh.
if [ -f $HERE/test-env.sh ]; then
. $HERE/test-env.sh
fi
@ -21,7 +21,6 @@ bash $HERE/install-dependencies.sh
# Get the appropriate UV_PYTHON.
. $ROOT/.evergreen/utils.sh
set -x
if [ -z "${PYTHON_BINARY:-}" ]; then
if [ -n "${PYTHON_VERSION:-}" ]; then
@ -31,7 +30,6 @@ if [ -z "${PYTHON_BINARY:-}" ]; then
fi
fi
export UV_PYTHON=${PYTHON_BINARY}
echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
echo "Using python $UV_PYTHON"
# Add the default install path to the path if needed.

View File

@ -26,7 +26,14 @@ from utils import (
)
# Passthrough environment variables.
PASS_THROUGH_ENV = ["GREEN_FRAMEWORK", "NO_EXT", "MONGODB_API_VERSION", "DEBUG_LOG"]
PASS_THROUGH_ENV = [
"GREEN_FRAMEWORK",
"NO_EXT",
"MONGODB_API_VERSION",
"DEBUG_LOG",
"PYTHON_BINARY",
"PYTHON_VERSION",
]
# Map the test name to test extra.
EXTRAS_MAP = {
@ -164,8 +171,8 @@ def handle_test_env() -> None:
# Handle pass through env vars.
for var in PASS_THROUGH_ENV:
if is_set(var) or getattr(opts, var.lower()):
write_env(var, os.environ[var])
if is_set(var) or getattr(opts, var.lower(), ""):
write_env(var, os.environ.get(var, getattr(opts, var.lower(), "")))
if extra := EXTRAS_MAP.get(test_name, ""):
UV_ARGS.append(f"--extra {extra}")

View File

@ -121,4 +121,4 @@ repos:
entry: .evergreen/scripts/generate-config.sh
language: python
require_serial: true
additional_dependencies: ["shrub.py>=3.8.0", "pyyaml>=6.0.2"]
additional_dependencies: ["shrub.py>=3.9.0", "pyyaml>=6.0.2"]

View File

@ -354,7 +354,7 @@ class Topology:
operationId=operation_id,
topologyDescription=self.description,
clientId=self.description._topology_settings._topology_id,
remainingTimeMS=int(end_time - time.monotonic()),
remainingTimeMS=int(1000 * (end_time - time.monotonic())),
)
logged_waiting = True

View File

@ -354,7 +354,7 @@ class Topology:
operationId=operation_id,
topologyDescription=self.description,
clientId=self.description._topology_settings._topology_id,
remainingTimeMS=int(end_time - time.monotonic()),
remainingTimeMS=int(1000 * (end_time - time.monotonic())),
)
logged_waiting = True

View File

@ -1,3 +1,3 @@
pymongo-auth-aws>=1.1.0,<2.0.0
pymongocrypt>=1.12.0,<2.0.0
pymongocrypt>=1.13.0,<2.0.0
certifi;os.name=='nt' or sys_platform=='darwin'

9
uv.lock generated
View File

@ -1,4 +1,5 @@
version = 1
revision = 1
requires-python = ">=3.9"
resolution-markers = [
"python_full_version >= '3.10'",
@ -997,7 +998,6 @@ sdist = { url = "https://files.pythonhosted.org/packages/07/e9/ae44ea7d7605df9e5
[[package]]
name = "pymongo"
version = "4.12.0.dev0"
source = { editable = "." }
dependencies = [
{ name = "dnspython" },
@ -1086,7 +1086,7 @@ requires-dist = [
{ name = "pykerberos", marker = "os_name != 'nt' and extra == 'gssapi'" },
{ name = "pymongo-auth-aws", marker = "extra == 'aws'", specifier = ">=1.1.0,<2.0.0" },
{ name = "pymongo-auth-aws", marker = "extra == 'encryption'", specifier = ">=1.1.0,<2.0.0" },
{ name = "pymongocrypt", marker = "extra == 'encryption'", specifier = ">=1.12.0,<2.0.0" },
{ name = "pymongocrypt", marker = "extra == 'encryption'", specifier = ">=1.13.0,<2.0.0" },
{ name = "pyopenssl", marker = "extra == 'ocsp'", specifier = ">=17.2.0" },
{ name = "pytest", marker = "extra == 'test'", specifier = ">=8.2" },
{ name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.24.0" },
@ -1101,6 +1101,7 @@ requires-dist = [
{ name = "winkerberos", marker = "os_name == 'nt' and extra == 'gssapi'", specifier = ">=0.5.0" },
{ name = "zstandard", marker = "extra == 'zstd'" },
]
provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "test", "zstd"]
[package.metadata.requires-dev]
coverage = [
@ -1136,8 +1137,8 @@ wheels = [
[[package]]
name = "pymongocrypt"
version = "1.13.0.dev0"
source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1cad4ad1c4cd6c11c6a4710da2127dab6a374471" }
version = "1.14.0.dev0"
source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#af621673c46d3d8fd2a2fe9d5540e24a79d9357a" }
dependencies = [
{ name = "cffi" },
{ name = "cryptography" },