Compare commits

..

2 Commits

Author SHA1 Message Date
Steven Silvester
0fc861dd1c
use master branch 2024-06-02 19:02:27 -05:00
Steven Silvester
3cf6838ee9
use release-python 2024-06-02 19:00:43 -05:00
1305 changed files with 73719 additions and 258689 deletions

View File

@ -1,4 +0,0 @@
# do not notify until at least 100 builds have been uploaded from the CI pipeline
# you can also set after_n_builds on comments independently
comment:
after_n_builds: 100

30
.evergreen/build-mac.sh Executable file
View File

@ -0,0 +1,30 @@
#!/bin/bash -ex
# Get access to testinstall.
. .evergreen/utils.sh
# Create temp directory for validated files.
rm -rf validdist
mkdir -p validdist
mv dist/* validdist || true
VERSION=${VERSION:-3.10}
PYTHON=/Library/Frameworks/Python.framework/Versions/$VERSION/bin/python3
rm -rf build
createvirtualenv $PYTHON releasevenv
python -m pip install build
python -m build --wheel .
deactivate || true
rm -rf releasevenv
# Test that each wheel is installable.
for release in dist/*; do
testinstall $PYTHON $release
mv $release validdist/
done
mv validdist/* dist
rm -rf validdist
ls dist

View File

@ -0,0 +1,42 @@
#!/bin/bash -ex
cd /src
# Get access to testinstall.
. .evergreen/utils.sh
# Create temp directory for validated files.
rm -rf validdist
mkdir -p validdist
mv dist/* validdist || true
# Compile wheels
for PYTHON in /opt/python/*/bin/python; do
if [[ ! $PYTHON =~ (cp38|cp39|cp310|cp311|cp312) ]]; then
continue
fi
# https://github.com/pypa/manylinux/issues/49
rm -rf build
$PYTHON -m pip install build
$PYTHON -m build --wheel .
rm -rf build
# Audit wheels and write manylinux tag
for whl in dist/*.whl; do
# Skip already built manylinux wheels.
if [[ "$whl" != *"manylinux"* ]]; then
auditwheel repair $whl -w dist
rm $whl
fi
done
# Test that each wheel is installable.
# Test without virtualenv because it's not present on manylinux containers.
for release in dist/*; do
testinstall $PYTHON $release "without-virtualenv"
mv $release validdist/
done
done
mv validdist/* dist
rm -rf validdist
ls dist

50
.evergreen/build-manylinux.sh Executable file
View File

@ -0,0 +1,50 @@
#!/bin/bash -ex
docker version
# Set up qemu support using the method used in docker/setup-qemu-action
# https://github.com/docker/setup-qemu-action/blob/2b82ce82d56a2a04d2637cd93a637ae1b359c0a7/README.md?plain=1#L46
docker run --rm --privileged tonistiigi/binfmt:latest --install all
# manylinux1 2021-05-05-b64d921 and manylinux2014 2021-05-05-1ac6ef3 were
# the last releases to generate pip < 20.3 compatible wheels. After that
# auditwheel was upgraded to v4 which produces PEP 600 manylinux_x_y wheels
# which requires pip >= 20.3. We use the older docker image to support older
# pip versions.
BUILD_WITH_TAG="$1"
if [ -n "$BUILD_WITH_TAG" ]; then
images=(quay.io/pypa/manylinux1_x86_64:2021-05-05-b64d921 \
quay.io/pypa/manylinux1_i686:2021-05-05-b64d921 \
quay.io/pypa/manylinux2014_x86_64:2021-05-05-1ac6ef3 \
quay.io/pypa/manylinux2014_i686:2021-05-05-1ac6ef3 \
quay.io/pypa/manylinux2014_aarch64:2021-05-05-1ac6ef3 \
quay.io/pypa/manylinux2014_ppc64le:2021-05-05-1ac6ef3 \
quay.io/pypa/manylinux2014_s390x:2021-05-05-1ac6ef3)
else
images=(quay.io/pypa/manylinux1_x86_64 \
quay.io/pypa/manylinux1_i686 \
quay.io/pypa/manylinux2014_x86_64 \
quay.io/pypa/manylinux2014_i686 \
quay.io/pypa/manylinux2014_aarch64 \
quay.io/pypa/manylinux2014_ppc64le \
quay.io/pypa/manylinux2014_s390x)
fi
for image in "${images[@]}"; do
docker pull $image
docker run --rm -v "`pwd`:/src" $image /src/.evergreen/build-manylinux-internal.sh
done
ls dist
# Check for any unexpected files.
unexpected=$(find dist \! \( -iname dist -or \
-iname '*cp38*' -or \
-iname '*cp39*' -or \
-iname '*cp310*' -or \
-iname '*cp311*' -or \
-iname '*cp312*' \))
if [ -n "$unexpected" ]; then
echo "Unexpected files:" $unexpected
exit 1
fi

29
.evergreen/build-windows.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash -ex
# Get access to testinstall.
. .evergreen/utils.sh
# Create temp directory for validated files.
rm -rf validdist
mkdir -p validdist
mv dist/* validdist || true
for VERSION in 38 39 310 311 312; do
_pythons=("C:/Python/Python${VERSION}/python.exe" \
"C:/Python/32/Python${VERSION}/python.exe")
for PYTHON in "${_pythons[@]}"; do
rm -rf build
$PYTHON -m pip install build
$PYTHON -m build --wheel .
# Test that each wheel is installable.
for release in dist/*; do
testinstall $PYTHON $release
mv $release validdist/
done
done
done
mv validdist/* dist
rm -rf validdist
ls dist

22
.evergreen/combine-coverage.sh Executable file → Normal file
View File

@ -3,14 +3,22 @@
# Coverage combine merges (and removes) all the coverage files and
# generates a new .coverage file in the current directory.
set -eu
set -o xtrace # Write all commands first to stderr
set -o errexit # Exit the script with error if any of the commands fail
# Set up the virtual env.
. .evergreen/scripts/setup-dev-env.sh
uv sync --group coverage
source .venv/bin/activate
. .evergreen/utils.sh
if [ -z "$PYTHON_BINARY" ]; then
PYTHON_BINARY=$(find_python3)
fi
createvirtualenv "$PYTHON_BINARY" covenv
# Keep in sync with run-tests.sh
# coverage >=5 is needed for relative_files=true.
pip install -q "coverage>=5,<=7.5"
pip list
ls -la coverage/
coverage combine coverage/coverage.*
coverage html -d htmlcov
python -m coverage combine coverage/coverage.*
python -m coverage html -d htmlcov

File diff suppressed because it is too large Load Diff

View File

@ -1,343 +0,0 @@
functions:
# Assume ec2 role
assume ec2 role:
- command: ec2.assume_role
params:
role_arn: ${aws_test_secrets_role}
duration_seconds: 3600
# Attach benchmark test results
attach benchmark test results:
- command: attach.results
params:
file_location: src/report.json
# Cleanup
cleanup:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/cleanup.sh
working_dir: src
type: test
# Download and merge coverage
download and merge coverage:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
type: setup
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/download-and-merge-coverage.sh
- ${bucket_name}
- ${revision}
- ${version_id}
working_dir: src
silent: true
include_expansions_in_env:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
type: test
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/combine-coverage.sh
working_dir: src
type: test
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/upload-coverage-report.sh
- ${bucket_name}
- ${revision}
- ${version_id}
working_dir: src
silent: true
include_expansions_in_env:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
type: test
- command: s3.put
params:
remote_file: coverage/${revision}/${version_id}/htmlcov/index.html
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: src/htmlcov/index.html
permissions: public-read
content_type: text/html
display_name: Coverage Report HTML
optional: "true"
type: setup
# Fetch source
fetch source:
- command: git.get_project
params:
directory: src
# Run server
run server:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/just.sh
- run-server
- ${TEST_NAME}
working_dir: src
include_expansions_in_env:
- VERSION
- TOPOLOGY
- AUTH
- SSL
- ORCHESTRATION_FILE
- UV_PYTHON
- TOOLCHAIN_VERSION
- STORAGE_ENGINE
- REQUIRE_API_VERSION
- DRIVERS_TOOLS
- TEST_CRYPT_SHARED
- AUTH_AWS
- LOAD_BALANCER
- LOCAL_ATLAS
- NO_EXT
type: test
- command: expansions.update
params:
file: ${DRIVERS_TOOLS}/mo-expansion.yml
# Run tests
run tests:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/just.sh
- setup-tests
- ${TEST_NAME}
- ${SUB_TEST_NAME}
working_dir: src
include_expansions_in_env:
- AUTH
- SSL
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
- COVERAGE
- UV_PYTHON
- LIBMONGOCRYPT_URL
- MONGODB_URI
- TOOLCHAIN_VERSION
- DISABLE_TEST_COMMANDS
- GREEN_FRAMEWORK
- NO_EXT
- COMPRESSORS
- MONGODB_API_VERSION
- REQUIRE_API_VERSION
- DEBUG_LOG
- DISABLE_FLAKY
- ORCHESTRATION_FILE
- OCSP_SERVER_TYPE
- VERSION
- IS_WIN32
- REQUIRE_FIPS
- TEST_MIN_DEPS
type: test
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/just.sh
- run-tests
working_dir: src
type: test
# Send dashboard data
send dashboard data:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/perf-submission-setup.sh
working_dir: src
include_expansions_in_env:
- requester
- revision_order_id
- project_id
- version_id
- build_variant
- parsed_order_id
- task_name
- task_id
- execution
- is_mainline
type: test
- command: expansions.update
params:
file: src/expansion.yml
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/perf-submission.sh
working_dir: src
include_expansions_in_env:
- requester
- revision_order_id
- project_id
- version_id
- build_variant
- parsed_order_id
- task_name
- task_id
- execution
- is_mainline
type: test
# Setup system
setup system:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/setup-system.sh
working_dir: src
include_expansions_in_env:
- is_patch
- project
- version_id
type: test
- command: expansions.update
params:
file: src/expansion.yml
# Teardown system
teardown system:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/just.sh
- teardown-tests
working_dir: src
type: test
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/teardown.sh
working_dir: src
type: test
# Test numpy
test numpy:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/just.sh
- test-numpy
working_dir: src
include_expansions_in_env:
- TOOLCHAIN_VERSION
- COVERAGE
type: test
# Upload coverage codecov
upload codecov:
- command: subprocess.exec
params:
binary: bash
args:
- .evergreen/scripts/upload-codecov.sh
working_dir: src
include_expansions_in_env:
- CODECOV_TOKEN
- build_variant
- task_name
- github_commit
- github_pr_number
- github_pr_head_branch
- github_author
- requester
- branch_name
type: test
# Upload coverage
upload coverage:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
type: setup
- command: s3.put
params:
remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: src/.coverage
permissions: public-read
content_type: text/html
display_name: Raw Coverage Report
optional: "true"
type: setup
# Upload mo artifacts
upload mo artifacts:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
type: setup
- command: archive.targz_pack
params:
target: mongo-coredumps.tgz
source_dir: ./
include:
- ./**.core
- ./**.mdmp
- command: s3.put
params:
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: mongo-coredumps.tgz
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: Core Dumps - Execution
optional: "true"
type: setup
- command: s3.put
params:
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
bucket: ${bucket_name}
local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: drivers-tools-logs.tar.gz
optional: "true"
type: setup
# Upload test results
upload test results:
- command: attach.results
params:
file_location: ${DRIVERS_TOOLS}/results.json
- command: attach.xunit_results
params:
file: src/xunit-results/TEST-*.xml

File diff suppressed because it is too large Load Diff

View File

@ -1,680 +0,0 @@
buildvariants:
# Alternative hosts tests
- name: other-hosts-rhel9-fips-latest
tasks:
- name: .test-no-toolchain
display_name: Other hosts RHEL9-FIPS latest
run_on:
- rhel92-fips
batchtime: 1440
expansions:
VERSION: latest
NO_EXT: "1"
REQUIRE_FIPS: "1"
UV_PYTHON: /usr/bin/python3.11
tags: []
- name: other-hosts-rhel8-zseries-latest
tasks:
- name: .test-no-toolchain
display_name: Other hosts RHEL8-zseries latest
run_on:
- rhel8-zseries-small
batchtime: 1440
expansions:
VERSION: latest
NO_EXT: "1"
tags: []
- name: other-hosts-rhel8-power8-latest
tasks:
- name: .test-no-toolchain
display_name: Other hosts RHEL8-POWER8 latest
run_on:
- rhel8-power-small
batchtime: 1440
expansions:
VERSION: latest
NO_EXT: "1"
tags: []
- name: other-hosts-rhel8-arm64-latest
tasks:
- name: .test-no-toolchain
display_name: Other hosts RHEL8-arm64 latest
run_on:
- rhel82-arm64-small
batchtime: 1440
expansions:
VERSION: latest
NO_EXT: "1"
tags: []
- name: other-hosts-amazon2023-latest
tasks:
- name: .test-no-toolchain
display_name: Other hosts Amazon2023 latest
run_on:
- amazon2023-arm64-latest-large-m8g
batchtime: 1440
expansions:
VERSION: latest
NO_EXT: "1"
tags: [pr]
# Atlas connect tests
- name: atlas-connect-rhel8
tasks:
- name: .test-no-orchestration
display_name: Atlas connect RHEL8
run_on:
- rhel87-small
expansions:
TEST_NAME: atlas_connect
tags: [pr]
# Aws auth tests
- name: auth-aws-rhel8
tasks:
- name: .auth-aws
display_name: Auth AWS RHEL8
run_on:
- rhel87-small
tags: []
- name: auth-aws-win64
tasks:
- name: .auth-aws
display_name: Auth AWS Win64
run_on:
- windows-2022-latest-small
tags: []
- name: auth-aws-macos
tasks:
- name: .auth-aws !.auth-aws-web-identity !.auth-aws-ec2
display_name: Auth AWS macOS
run_on:
- macos-14
tags: [pr]
- name: auth-aws-ecs-macos
tasks:
- name: .auth-aws-ecs
display_name: Auth AWS ECS macOS
run_on:
- ubuntu2404-small
tags: [pr]
# Aws lambda tests
- name: faas-lambda
tasks:
- name: .aws_lambda
display_name: FaaS Lambda
run_on:
- rhel87-small
# Backport pr tests
- name: backport-pr
tasks:
- name: backport-pr
display_name: Backport PR
run_on:
- rhel87-small
# Compression tests
- name: compression-snappy-rhel8
tasks:
- name: .test-standard
display_name: Compression snappy RHEL8
run_on:
- rhel87-small
expansions:
COMPRESSOR: snappy
- name: compression-zlib-rhel8
tasks:
- name: .test-standard
display_name: Compression zlib RHEL8
run_on:
- rhel87-small
expansions:
COMPRESSOR: zlib
- name: compression-zstd-rhel8
tasks:
- name: .test-standard !.server-4.2
display_name: Compression zstd RHEL8
run_on:
- rhel87-small
expansions:
COMPRESSOR: zstd
- name: compression-zstd-ubuntu-22
tasks:
- name: .test-standard !.server-4.2 !.server-4.4 !.server-5.0 .python-3.14
- name: .test-standard !.server-4.2 !.server-4.4 !.server-5.0 .python-3.14t
display_name: Compression zstd Ubuntu-22
run_on:
- ubuntu2204-small
expansions:
COMPRESSOR: ztsd
# Coverage report tests
- name: coverage-report
tasks:
- name: coverage-report
display_name: Coverage Report
run_on:
- rhel87-small
# Disable test commands tests
- name: disable-test-commands-rhel8
tasks:
- name: .test-standard .server-latest
display_name: Disable test commands RHEL8
run_on:
- rhel87-small
expansions:
AUTH: auth
SSL: ssl
DISABLE_TEST_COMMANDS: "1"
# Doctests tests
- name: doctests-rhel8
tasks:
- name: .test-non-standard .standalone-noauth-nossl
display_name: Doctests RHEL8
run_on:
- rhel87-small
expansions:
TEST_NAME: doctest
# Encryption tests
- name: encryption-rhel8
tasks:
- name: .test-non-standard
display_name: Encryption RHEL8
run_on:
- rhel87-small
batchtime: 1440
expansions:
TEST_NAME: encryption
tags: [encryption_tag]
- name: encryption-macos
tasks:
- name: .test-non-standard !.pypy
display_name: Encryption macOS
run_on:
- macos-14
batchtime: 1440
expansions:
TEST_NAME: encryption
tags: [encryption_tag]
- name: encryption-win64
tasks:
- name: .test-non-standard !.pypy
display_name: Encryption Win64
run_on:
- windows-2022-latest-small
batchtime: 1440
expansions:
TEST_NAME: encryption
tags: [encryption_tag]
- name: encryption-crypt_shared-rhel8
tasks:
- name: .test-non-standard
display_name: Encryption crypt_shared RHEL8
run_on:
- rhel87-small
batchtime: 1440
expansions:
TEST_NAME: encryption
TEST_CRYPT_SHARED: "true"
tags: [encryption_tag]
- name: encryption-crypt_shared-macos
tasks:
- name: .test-non-standard !.pypy
display_name: Encryption crypt_shared macOS
run_on:
- macos-14
batchtime: 1440
expansions:
TEST_NAME: encryption
TEST_CRYPT_SHARED: "true"
tags: [encryption_tag]
- name: encryption-crypt_shared-win64
tasks:
- name: .test-non-standard !.pypy
display_name: Encryption crypt_shared Win64
run_on:
- windows-2022-latest-small
batchtime: 1440
expansions:
TEST_NAME: encryption
TEST_CRYPT_SHARED: "true"
tags: [encryption_tag]
- name: encryption-pyopenssl-rhel8
tasks:
- name: .test-non-standard
display_name: Encryption PyOpenSSL RHEL8
run_on:
- rhel87-small
batchtime: 1440
expansions:
TEST_NAME: encryption
SUB_TEST_NAME: pyopenssl
tags: [encryption_tag]
# Enterprise auth tests
- name: auth-enterprise-rhel8
tasks:
- name: .test-standard-auth .auth !.free-threaded
display_name: Auth Enterprise RHEL8
run_on:
- rhel87-small
expansions:
TEST_NAME: enterprise_auth
AUTH: auth
- name: auth-enterprise-macos
tasks:
- name: .test-standard-auth !.pypy .auth !.free-threaded
display_name: Auth Enterprise macOS
run_on:
- macos-14
expansions:
TEST_NAME: enterprise_auth
AUTH: auth
- name: auth-enterprise-win64
tasks:
- name: .test-standard-auth !.pypy .auth !.free-threaded
display_name: Auth Enterprise Win64
run_on:
- windows-2022-latest-small
expansions:
TEST_NAME: enterprise_auth
AUTH: auth
# Green framework tests
- name: green-gevent-rhel8
tasks:
- name: .test-standard .sync !.free-threaded
display_name: Green Gevent RHEL8
run_on:
- rhel87-small
expansions:
GREEN_FRAMEWORK: gevent
# Import time tests
- name: import-time
tasks:
- name: check-import-time
display_name: Import Time
run_on:
- rhel87-small
# Kms tests
- name: kms
tasks:
- name: test-gcpkms
batchtime: 1440
- name: test-gcpkms-fail
- name: test-azurekms
batchtime: 1440
- name: test-azurekms-fail
display_name: KMS
run_on:
- debian11-small
# Load balancer tests
- name: load-balancer
tasks:
- name: .test-non-standard .server-6.0 .sharded_cluster-auth-ssl
- name: .test-non-standard .server-7.0 .sharded_cluster-auth-ssl
- name: .test-non-standard .server-8.0 .sharded_cluster-auth-ssl
- name: .test-non-standard .server-rapid .sharded_cluster-auth-ssl
- name: .test-non-standard .server-latest .sharded_cluster-auth-ssl
display_name: Load Balancer
run_on:
- rhel87-small
batchtime: 1440
expansions:
TEST_NAME: load_balancer
# Min support tests
- name: min-support-rhel8
tasks:
- name: .test-min-support
display_name: Min Support RHEL8
run_on:
- rhel87-small
# Mockupdb tests
- name: mockupdb-rhel8
tasks:
- name: .test-no-orchestration
display_name: MockupDB RHEL8
run_on:
- rhel87-small
expansions:
TEST_NAME: mockupdb
tags: [pr]
# Mod wsgi tests
- name: mod_wsgi-ubuntu-22
tasks:
- name: .mod_wsgi
display_name: Mod_WSGI Ubuntu-22
run_on:
- ubuntu2204-small
expansions:
MOD_WSGI_VERSION: "4"
# No c ext tests
- name: no-c-ext-rhel8
tasks:
- name: .test-standard
display_name: No C Ext RHEL8
run_on:
- rhel87-small
expansions:
NO_EXT: "1"
# No server tests
- name: no-server-rhel8
tasks:
- name: .test-no-orchestration
display_name: No server RHEL8
run_on:
- rhel87-small
tags: [pr]
# Ocsp tests
- name: ocsp-rhel8
tasks:
- name: .ocsp
display_name: OCSP RHEL8
run_on:
- rhel87-small
batchtime: 10080
- name: ocsp-win64
tasks:
- name: .ocsp-rsa !.ocsp-staple .latest
- name: .ocsp-rsa !.ocsp-staple .4.4
display_name: OCSP Win64
run_on:
- windows-2022-latest-small
batchtime: 10080
- name: ocsp-macos
tasks:
- name: .ocsp-rsa !.ocsp-staple .latest
- name: .ocsp-rsa !.ocsp-staple .4.4
display_name: OCSP macOS
run_on:
- macos-14
batchtime: 10080
# Oidc auth tests
- name: auth-oidc-ubuntu-22
tasks:
- name: .auth_oidc_remote
display_name: Auth OIDC Ubuntu-22
run_on:
- ubuntu2204-small
batchtime: 1440
- name: auth-oidc-local-ubuntu-22
tasks:
- name: "!.auth_oidc_remote .auth_oidc"
display_name: Auth OIDC Local Ubuntu-22
run_on:
- ubuntu2204-small
batchtime: 1440
expansions:
COVERAGE: "1"
tags: [pr]
- name: auth-oidc-macos
tasks:
- name: "!.auth_oidc_remote .auth_oidc"
display_name: Auth OIDC macOS
run_on:
- macos-14
batchtime: 1440
- name: auth-oidc-win64
tasks:
- name: "!.auth_oidc_remote .auth_oidc"
display_name: Auth OIDC Win64
run_on:
- windows-2022-latest-small
batchtime: 1440
# Perf tests
- name: performance-benchmarks
tasks:
- name: .perf
display_name: Performance Benchmarks
run_on:
- rhel90-dbx-perf-large
batchtime: 1440
# Pyopenssl tests
- name: pyopenssl-rhel8
tasks:
- name: .test-standard .sync
- name: .test-standard .async .replica_set-noauth-ssl
display_name: PyOpenSSL RHEL8
run_on:
- rhel87-small
batchtime: 1440
expansions:
SUB_TEST_NAME: pyopenssl
- name: pyopenssl-macos
tasks:
- name: .test-standard !.pypy .sync
- name: .test-standard !.pypy .async .replica_set-noauth-ssl
display_name: PyOpenSSL macOS
run_on:
- rhel87-small
batchtime: 1440
expansions:
SUB_TEST_NAME: pyopenssl
- name: pyopenssl-win64
tasks:
- name: .test-standard !.pypy .sync
- name: .test-standard !.pypy .async .replica_set-noauth-ssl
display_name: PyOpenSSL Win64
run_on:
- windows-2022-latest-small
batchtime: 1440
expansions:
SUB_TEST_NAME: pyopenssl
# Search index tests
- name: search-index-helpers-rhel8
tasks:
- name: .search_index
display_name: Search Index Helpers RHEL8
run_on:
- rhel87-small
# Server version tests
- name: mongodb-v4.2
tasks:
- name: .server-version
display_name: "* MongoDB v4.2"
run_on:
- rhel87-small
expansions:
VERSION: "4.2"
tags: [coverage_tag]
- name: mongodb-v4.4
tasks:
- name: .server-version
display_name: "* MongoDB v4.4"
run_on:
- rhel87-small
expansions:
VERSION: "4.4"
tags: [coverage_tag]
- name: mongodb-v5.0
tasks:
- name: .server-version
display_name: "* MongoDB v5.0"
run_on:
- rhel87-small
expansions:
VERSION: "5.0"
tags: [coverage_tag]
- name: mongodb-v6.0
tasks:
- name: .server-version
display_name: "* MongoDB v6.0"
run_on:
- rhel87-small
expansions:
VERSION: "6.0"
tags: [coverage_tag]
- name: mongodb-v7.0
tasks:
- name: .server-version
display_name: "* MongoDB v7.0"
run_on:
- rhel87-small
expansions:
VERSION: "7.0"
tags: [coverage_tag]
- name: mongodb-v8.0
tasks:
- name: .server-version
display_name: "* MongoDB v8.0"
run_on:
- rhel87-small
expansions:
VERSION: "8.0"
tags: [coverage_tag]
- name: mongodb-rapid
tasks:
- name: .server-version
display_name: "* MongoDB rapid"
run_on:
- rhel87-small
expansions:
VERSION: rapid
tags: [coverage_tag]
- name: mongodb-latest
tasks:
- name: .server-version
display_name: "* MongoDB latest"
run_on:
- rhel87-small
expansions:
VERSION: latest
tags: [coverage_tag]
# Stable api tests
- name: stable-api-require-v1-rhel8-auth
tasks:
- name: .test-standard !.replica_set-noauth-ssl .server-5.0
- name: .test-standard !.replica_set-noauth-ssl .server-6.0
- name: .test-standard !.replica_set-noauth-ssl .server-7.0
- name: .test-standard !.replica_set-noauth-ssl .server-8.0
- name: .test-standard !.replica_set-noauth-ssl .server-rapid
- name: .test-standard !.replica_set-noauth-ssl .server-latest
display_name: Stable API require v1 RHEL8 Auth
run_on:
- rhel87-small
expansions:
AUTH: auth
REQUIRE_API_VERSION: "1"
MONGODB_API_VERSION: "1"
tags: [versionedApi_tag]
- name: stable-api-accept-v2-rhel8-auth
tasks:
- name: .test-standard .server-5.0 .standalone-noauth-nossl
- name: .test-standard .server-6.0 .standalone-noauth-nossl
- name: .test-standard .server-7.0 .standalone-noauth-nossl
- name: .test-standard .server-8.0 .standalone-noauth-nossl
- name: .test-standard .server-rapid .standalone-noauth-nossl
- name: .test-standard .server-latest .standalone-noauth-nossl
display_name: Stable API accept v2 RHEL8 Auth
run_on:
- rhel87-small
expansions:
AUTH: auth
ORCHESTRATION_FILE: versioned-api-testing.json
tags: [versionedApi_tag]
# Standard nonlinux tests
- name: test-macos
tasks:
- name: .test-standard !.pypy
display_name: "* Test macOS"
run_on:
- macos-14
tags: [standard-non-linux]
- name: test-macos-arm64
tasks:
- name: .test-standard !.pypy .server-6.0
- name: .test-standard !.pypy .server-7.0
- name: .test-standard !.pypy .server-8.0
- name: .test-standard !.pypy .server-rapid
- name: .test-standard !.pypy .server-latest
display_name: "* Test macOS Arm64"
run_on:
- macos-14-arm64
tags: [standard-non-linux]
- name: test-win64
tasks:
- name: .test-standard !.pypy
- name: .test-no-orchestration !.pypy
display_name: "* Test Win64"
run_on:
- windows-2022-latest-small
tags: [standard-non-linux]
- name: test-win32
tasks:
- name: .test-standard !.pypy
display_name: "* Test Win32"
run_on:
- windows-64-vsMulti-small
expansions:
IS_WIN32: "1"
tags: [standard-non-linux]
# Storage engine tests
- name: storage-inmemory-rhel8
tasks:
- name: .test-standard .standalone-noauth-nossl
display_name: Storage InMemory RHEL8
run_on:
- rhel87-small
expansions:
STORAGE_ENGINE: inmemory
# Test numpy tests
- name: test-numpy-rhel8
tasks:
- name: .test-numpy
display_name: Test Numpy RHEL8
run_on:
- rhel87-small
tags: [binary, vector, pr]
- name: test-numpy-macos
tasks:
- name: .test-numpy
display_name: Test Numpy macOS
run_on:
- macos-14
tags: [binary, vector]
- name: test-numpy-macos-arm64
tasks:
- name: .test-numpy
display_name: Test Numpy macOS Arm64
run_on:
- macos-14-arm64
tags: [binary, vector]
- name: test-numpy-win64
tasks:
- name: .test-numpy
display_name: Test Numpy Win64
run_on:
- windows-2022-latest-small
tags: [binary, vector]
- name: test-numpy-win32
tasks:
- name: .test-numpy
display_name: Test Numpy Win32
run_on:
- windows-64-vsMulti-small
expansions:
IS_WIN32: "1"
tags: [binary, vector]

View File

@ -0,0 +1,19 @@
#!/bin/bash
set -o xtrace # Write all commands first to stderr
set -o errexit # Exit the script with error if any of the commands fail
# Copy PyMongo's test certificates over driver-evergreen-tools'
cp ${PROJECT_DIRECTORY}/test/certificates/* ${DRIVERS_TOOLS}/.evergreen/x509gen/
# Replace MongoOrchestration's client certificate.
cp ${PROJECT_DIRECTORY}/test/certificates/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem
if [ -w /etc/hosts ]; then
SUDO=""
else
SUDO="sudo"
fi
# Add 'server' and 'hostname_not_in_cert' as a hostnames
echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts
echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts

View File

@ -1,5 +0,0 @@
#!/bin/bash
set -eu
. .evergreen/scripts/setup-dev-env.sh
just "$@"

9
.evergreen/release.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash -ex
if [ "$(uname -s)" = "Darwin" ]; then
.evergreen/build-mac.sh
elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
.evergreen/build-windows.sh
else
.evergreen/build-manylinux.sh
fi

View File

@ -1,56 +0,0 @@
#!/bin/bash
PYMONGO=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)")
rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894
rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873
rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-application-error.json # PYTHON-4918
rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-checkout-error.json # PYTHON-4918
rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-min-pool-size-error.json # PYTHON-4918
rm $PYMONGO/test/client-side-encryption/spec/unified/client-bulkWrite-qe.json # PYTHON-4929
# Python doesn't implement DRIVERS-3064
rm $PYMONGO/test/collection_management/listCollections-rawdata.json
rm $PYMONGO/test/crud/unified/aggregate-rawdata.json
rm $PYMONGO/test/crud/unified/bulkWrite-deleteMany-rawdata.json
rm $PYMONGO/test/crud/unified/bulkWrite-deleteOne-rawdata.json
rm $PYMONGO/test/crud/unified/bulkWrite-replaceOne-rawdata.json
rm $PYMONGO/test/crud/unified/bulkWrite-updateMany-rawdata.json
rm $PYMONGO/test/crud/unified/bulkWrite-updateOne-rawdata.json
rm $PYMONGO/test/crud/unified/client-bulkWrite-delete-rawdata.json
rm $PYMONGO/test/crud/unified/client-bulkWrite-replaceOne-rawdata.json
rm $PYMONGO/test/crud/unified/client-bulkWrite-update-rawdata.json
rm $PYMONGO/test/crud/unified/count-rawdata.json
rm $PYMONGO/test/crud/unified/countDocuments-rawdata.json
rm $PYMONGO/test/crud/unified/db-aggregate-rawdata.json
rm $PYMONGO/test/crud/unified/deleteMany-rawdata.json
rm $PYMONGO/test/crud/unified/deleteOne-rawdata.json
rm $PYMONGO/test/crud/unified/distinct-rawdata.json
rm $PYMONGO/test/crud/unified/estimatedDocumentCount-rawdata.json
rm $PYMONGO/test/crud/unified/find-rawdata.json
rm $PYMONGO/test/crud/unified/findOneAndDelete-rawdata.json
rm $PYMONGO/test/crud/unified/findOneAndReplace-rawdata.json
rm $PYMONGO/test/crud/unified/findOneAndUpdate-rawdata.json
rm $PYMONGO/test/crud/unified/insertMany-rawdata.json
rm $PYMONGO/test/crud/unified/insertOne-rawdata.json
rm $PYMONGO/test/crud/unified/replaceOne-rawdata.json
rm $PYMONGO/test/crud/unified/updateMany-rawdata.json
rm $PYMONGO/test/crud/unified/updateOne-rawdata.json
rm $PYMONGO/test/index_management/index-rawdata.json
# PyMongo does not support modifyCollection
rm $PYMONGO/test/collection_management/modifyCollection-*.json
# PYTHON-5248 - Remove support for MongoDB 4.0
find /$PYMONGO/test -type f -name 'pre-42-*.json' -delete
# PYTHON-3359 - Remove Database and Collection level timeout override
rm $PYMONGO/test/csot/override-collection-timeoutMS.json
rm $PYMONGO/test/csot/override-database-timeoutMS.json
# PYTHON-2943 - Socks5 Proxy Support
rm $PYMONGO/test/uri_options/proxy-options.json
# PYTHON-5517 - Avoid clearing the connection pool when the server connection rate limiter triggers
rm $PYMONGO/test/discovery_and_monitoring/unified/backpressure-*.json
echo "Done removing unimplemented tests"

View File

@ -1,6 +1,6 @@
#!/bin/bash
# Resync test files from the specifications repo.
set -eu
# exit when any command fails
set -e
PYMONGO=$(dirname "$(cd "$(dirname "$0")"; pwd)")
SPECS=${MDB_SPECS:-~/Work/specifications}
@ -45,12 +45,9 @@ then
fi
# Ensure the JSON files are up to date.
if ! [ -n "${CI:-}" ]
then
cd $SPECS/source
make
cd -
fi
cd $SPECS/source
make
cd -
# cpjson unified-test-format/tests/invalid unified-test-format/invalid
# * param1: Path to spec tests dir in specifications repo
# * param2: Path to where the corresponding tests live in Python.
@ -76,8 +73,8 @@ do
auth)
cpjson auth/tests/ auth
;;
bson-binary-vector|bson_binary_vector)
cpjson bson-binary-vector/tests/ bson_binary_vector
atlas-data-lake-testing|data_lake)
cpjson atlas-data-lake-testing/tests/ data_lake
;;
bson-corpus|bson_corpus)
cpjson bson-corpus/tests/ bson_corpus
@ -94,9 +91,6 @@ do
change-streams|change_streams)
cpjson change-streams/tests/ change_streams/
;;
client-backpressure|client_backpressure)
cpjson client-backpressure/tests client-backpressure
;;
client-side-encryption|csfle|fle)
cpjson client-side-encryption/tests/ client-side-encryption/spec
cpjson client-side-encryption/corpus/ client-side-encryption/corpus
@ -113,6 +107,7 @@ do
cmap|CMAP|connection-monitoring-and-pooling)
cpjson connection-monitoring-and-pooling/tests/logging connection_logging
cpjson connection-monitoring-and-pooling/tests/cmap-format connection_monitoring
rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873
;;
apm|APM|command-monitoring|command_monitoring)
cpjson command-logging-and-monitoring/tests/monitoring command_monitoring
@ -133,9 +128,6 @@ do
gridfs)
cpjson gridfs/tests gridfs
;;
handshake)
cpjson mongodb-handshake/tests handshake
;;
index|index-management)
cpjson index-management/tests index_management
;;
@ -145,9 +137,6 @@ do
srv|SRV|initial-dns-seedlist-discovery|srv_seedlist)
cpjson initial-dns-seedlist-discovery/tests/ srv_seedlist
;;
read-write-concern|read_write_concern)
cpjson read-write-concern/tests/operation read_write_concern/operation
;;
retryable-reads|retryable_reads)
cpjson retryable-reads/tests/ retryable_reads
;;
@ -176,7 +165,7 @@ do
;;
server-selection|server_selection)
cpjson server-selection/tests/ server_selection
rm -rf $PYMONGO/test/server_selection/logging # these tests live in server_selection_logging
rm -rf $PYMONGO/test/server_selection/logging
cpjson server-selection/tests/logging server_selection_logging
;;
server-selection-logging|server_selection_logging)
@ -188,6 +177,7 @@ do
transactions|transactions-convenient-api)
cpjson transactions/tests/ transactions
cpjson transactions-convenient-api/tests/ transactions-convenient-api
rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894
;;
unified|unified-test-format)
cpjson unified-test-format/tests/ unified-test-format/

View File

@ -0,0 +1,10 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
KEY_NAME="${AZUREKMS_KEYNAME}" \
KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \
SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
./.evergreen/tox.sh -m test-eg

View File

@ -0,0 +1,21 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh
echo "Copying files ... begin"
export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
tar czf /tmp/mongo-python-driver.tgz .
# shellcheck disable=SC2088
AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
echo "Untarring file ... end"
echo "Running test ... begin"
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
echo "Running test ... end"

View File

@ -0,0 +1,10 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
export PATH="/opt/python/3.9/bin:${PATH}"
python --version
pushd ./test/lambda
. build.sh
popd
. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh

View File

@ -0,0 +1,18 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
source ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/secrets-export.sh
echo "Copying files ... begin"
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
tar czf /tmp/mongo-python-driver.tgz .
GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Untarring file ... end"
echo "Running test ... begin"
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Running test ... end"

View File

@ -1,20 +1,13 @@
#!/bin/bash
# Check for regressions in the import time of pymongo.
set -eu
#!/bin/bash -ex
HERE=$(dirname ${BASH_SOURCE:-$0})
set -o errexit # Exit the script with error if any of the commands fail
set -x
source $HERE/env.sh
. .evergreen/utils.sh
pushd $HERE/../.. >/dev/null
BASE_SHA="$1"
HEAD_SHA="$2"
# Set up the virtual env.
. $HERE/setup-dev-env.sh
uv venv --seed
source .venv/bin/activate
if [ -z "$PYTHON_BINARY" ]; then
PYTHON_BINARY=$(find_python3)
fi
# Use the previous commit if this was not a PR run.
if [ "$BASE_SHA" == "$HEAD_SHA" ]; then
@ -23,6 +16,7 @@ fi
function get_import_time() {
local log_file
createvirtualenv "$PYTHON_BINARY" import-venv
python -m pip install -q ".[aws,encryption,gssapi,ocsp,snappy,zstd]"
# Import once to cache modules
python -c "import pymongo"
@ -31,11 +25,7 @@ function get_import_time() {
}
get_import_time $HEAD_SHA
git stash || true
git checkout $BASE_SHA
get_import_time $BASE_SHA
git checkout $HEAD_SHA
git stash apply || true
python tools/compare_import_time.py $HEAD_SHA $BASE_SHA
popd >/dev/null

View File

@ -0,0 +1,47 @@
#!/bin/bash
set -o xtrace
set -o errexit
APACHE=$(command -v apache2 || command -v /usr/lib/apache2/mpm-prefork/apache2) || true
if [ -n "$APACHE" ]; then
APACHE_CONFIG=apache24ubuntu161404.conf
else
APACHE=$(command -v httpd) || true
if [ -z "$APACHE" ]; then
echo "Could not find apache2 binary"
exit 1
else
APACHE_CONFIG=apache22amazon.conf
fi
fi
PYTHON_VERSION=$(${PYTHON_BINARY} -c "import sys; sys.stdout.write('.'.join(str(val) for val in sys.version_info[:2]))")
# Ensure the C extensions are installed.
${PYTHON_BINARY} setup.py build_ext -i
export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_version/$MOD_WSGI_VERSION/mod_wsgi.so
export PYTHONHOME=/opt/python/$PYTHON_VERSION
# If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead
# of daemon mode (WSGIDaemonProcess).
if [ -n "$MOD_WSGI_EMBEDDED" ]; then
export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf
else
export MOD_WSGI_CONF=mod_wsgi_test.conf
fi
cd ..
$APACHE -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${APACHE_CONFIG}
trap '$APACHE -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${APACHE_CONFIG}' EXIT HUP
wget -t 1 -T 10 -O - "http://localhost:8080/interpreter1${PROJECT_DIRECTORY}" || (cat error_log && exit 1)
wget -t 1 -T 10 -O - "http://localhost:8080/interpreter2${PROJECT_DIRECTORY}" || (cat error_log && exit 1)
${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel \
http://localhost:8080/interpreter1${PROJECT_DIRECTORY} http://localhost:8080/interpreter2${PROJECT_DIRECTORY} || \
(tail -n 100 error_log && exit 1)
${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial \
http://localhost:8080/interpreter1${PROJECT_DIRECTORY} http://localhost:8080/interpreter2${PROJECT_DIRECTORY} || \
(tail -n 100 error_log && exit 1)

View File

@ -1,6 +1,7 @@
#!/bin/bash
# Script run on an ECS host to test MONGODB-AWS.
set -eu
# Don't trace since the URI contains a password that shouldn't show up in the logs
set -o errexit # Exit the script with error if any of the commands fail
############################################
# Main Program #
@ -19,14 +20,15 @@ fi
# Now we can safely enable xtrace
set -o xtrace
# Install a c compiler.
apt-get -qq update < /dev/null > /dev/null
apt-get -q install -y build-essential
# Install python with pip.
PYTHON_VER="python3.9"
apt-get update
apt-get install $PYTHON_VER python3-pip build-essential $PYTHON_VER-dev -y
export PYTHON_BINARY=$PYTHON_VER
export TEST_AUTH_AWS=1
export AUTH="auth"
export SET_XTRACE_ON=1
cd src
rm -rf .venv
rm -f .evergreen/scripts/test-env.sh || true
rm -f .evergreen/scripts/env.sh || true
bash ./.evergreen/just.sh setup-tests auth_aws ecs-remote
bash .evergreen/just.sh run-tests
$PYTHON_BINARY -m pip install -q --user tox
bash .evergreen/tox.sh -m test-eg

View File

@ -0,0 +1,27 @@
#!/bin/bash
set -o xtrace
set -o errexit # Exit the script with error if any of the commands fail
############################################
# Main Program #
############################################
# Supported/used environment variables:
# MONGODB_URI Set the URI, including an optional username/password to use
# to connect to the server via MONGODB-AWS authentication
# mechanism.
# PYTHON_BINARY The Python version to use.
echo "Running MONGODB-AWS authentication tests"
# Handle credentials and environment setup.
. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
# show test output
set -x
export TEST_AUTH_AWS=1
export AUTH="auth"
export SET_XTRACE_ON=1
bash ./.evergreen/tox.sh -m test-eg

View File

@ -1,17 +1,32 @@
#!/bin/bash
# Script run on a remote host to test MONGODB-OIDC.
set +x # Disable debug trace
set -eu
echo "Running MONGODB-OIDC authentication tests on ${OIDC_ENV}..."
echo "Running MONGODB-OIDC authentication tests"
OIDC_ENV=${OIDC_ENV:-"test"}
if [ $OIDC_ENV == "test" ]; then
# Make sure DRIVERS_TOOLS is set.
if [ -z "$DRIVERS_TOOLS" ]; then
echo "Must specify DRIVERS_TOOLS"
exit 1
fi
source ${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh
elif [ $OIDC_ENV == "azure" ]; then
source ./env.sh
elif [ $OIDC_ENV == "gcp" ]; then
source ./secrets-export.sh
if [ ${OIDC_ENV} == "k8s" ]; then
SUB_TEST_NAME=$K8S_VARIANT-remote
else
SUB_TEST_NAME=$OIDC_ENV-remote
sudo apt-get install -y python3-dev build-essential
echo "Unrecognized OIDC_ENV $OIDC_ENV"
exit 1
fi
bash ./.evergreen/just.sh setup-tests auth_oidc $SUB_TEST_NAME
bash ./.evergreen/just.sh run-tests "${@:1}"
echo "Running MONGODB-OIDC authentication tests on ${OIDC_ENV}... done."
export TEST_AUTH_OIDC=1
export COVERAGE=1
export AUTH="auth"
bash ./.evergreen/tox.sh -m test-eg -- "${@:1}"

View File

@ -0,0 +1,19 @@
#!/bin/bash
set -o xtrace
set -o errexit
git clone --depth 1 https://github.com/mongodb/specifications.git
pushd specifications/source/benchmarking/data
tar xf extended_bson.tgz
tar xf parallel.tgz
tar xf single_and_multi_document.tgz
popd
export TEST_PATH="${PROJECT_DIRECTORY}/specifications/source/benchmarking/data"
export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
export PERF_TEST=1
bash ./.evergreen/tox.sh -m test-eg

View File

@ -1,45 +1,286 @@
#!/bin/bash
# Run a test suite that was configured with setup-tests.sh.
set -eu
set -o errexit # Exit the script with error if any of the commands fail
set -o xtrace
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
SCRIPT_DIR="$( cd -- "$SCRIPT_DIR" > /dev/null 2>&1 && pwd )"
ROOT_DIR="$(dirname $SCRIPT_DIR)"
# Note: It is assumed that you have already set up a virtual environment before running this file.
PREV_DIR=$(pwd)
cd $ROOT_DIR
# Supported/used environment variables:
# AUTH Set to enable authentication. Defaults to "noauth"
# SSL Set to enable SSL. Defaults to "nossl"
# GREEN_FRAMEWORK The green framework to test with, if any.
# COVERAGE If non-empty, run the test suite with coverage.
# COMPRESSORS If non-empty, install appropriate compressor.
# LIBMONGOCRYPT_URL The URL to download libmongocrypt.
# TEST_DATA_LAKE If non-empty, run data lake tests.
# TEST_ENCRYPTION If non-empty, run encryption tests.
# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib.
# TEST_SERVERLESS If non-empy, test on serverless.
# TEST_LOADBALANCER If non-empy, test load balancing.
# TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure
# TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP
# TEST_PYOPENSSL If non-empy, test with PyOpenSSL
# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth
# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism
# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism
# TEST_PERF If non-empty, run performance tests
# TEST_OCSP If non-empty, run OCSP tests
# TEST_ATLAS If non-empty, test Atlas connections
# TEST_INDEX_MANAGEMENT If non-empty, run index management tests
# TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL
# Try to source the env file.
if [ -f $SCRIPT_DIR/scripts/env.sh ]; then
echo "Sourcing env inputs"
. $SCRIPT_DIR/scripts/env.sh
AUTH=${AUTH:-noauth}
SSL=${SSL:-nossl}
TEST_ARGS="${*:1}"
PYTHON=$(which python)
# TODO: Remove when we drop PyPy 3.8 support.
OLD_PYPY=$(python -c "import sys; print(sys.implementation.name.lower() == 'pypy' and sys.implementation.version < (7, 3, 12))")
export PIP_QUIET=1 # Quiet by default
export PIP_PREFER_BINARY=1 # Prefer binary dists by default
python -c "import sys; sys.exit(sys.prefix == sys.base_prefix)" || (echo "Not inside a virtual env!"; exit 1)
# Try to source local Drivers Secrets
if [ -f ./secrets-export.sh ]; then
echo "Sourcing secrets"
source ./secrets-export.sh
else
echo "Not sourcing env inputs"
echo "Not sourcing secrets"
fi
# Handle test inputs.
if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then
echo "Sourcing test inputs"
. $SCRIPT_DIR/scripts/test-env.sh
else
echo "Missing test inputs, please run 'just setup-tests'"
exit 1
if [ "$AUTH" != "noauth" ]; then
set +x
if [ ! -z "$TEST_DATA_LAKE" ]; then
export DB_USER="mhuser"
export DB_PASSWORD="pencil"
elif [ ! -z "$TEST_SERVERLESS" ]; then
source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh
export DB_USER=$SERVERLESS_ATLAS_USER
export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
export MONGODB_URI="$SERVERLESS_URI"
echo "MONGODB_URI=$MONGODB_URI"
export SINGLE_MONGOS_LB_URI=$MONGODB_URI
export MULTI_MONGOS_LB_URI=$MONGODB_URI
elif [ ! -z "$TEST_AUTH_OIDC" ]; then
export DB_USER=$OIDC_ADMIN_USER
export DB_PASSWORD=$OIDC_ADMIN_PWD
export DB_IP="$MONGODB_URI"
else
export DB_USER="bob"
export DB_PASSWORD="pwd123"
fi
echo "Added auth, DB_USER: $DB_USER"
set -x
fi
cleanup_tests() {
# Avoid leaving the lock file in a changed state when we change the resolution type.
if [ -n "${TEST_MIN_DEPS:-}" ]; then
git checkout uv.lock || true
fi
cd $PREV_DIR
}
if [ -n "$TEST_ENTERPRISE_AUTH" ]; then
if [ "Windows_NT" = "$OS" ]; then
echo "Setting GSSAPI_PASS"
export GSSAPI_PASS=${SASL_PASS}
export GSSAPI_CANONICALIZE="true"
else
# BUILD-3830
touch krb5.conf.empty
export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty
trap "cleanup_tests" SIGINT ERR
echo "Writing keytab"
echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab
echo "Running kinit"
kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL}
fi
echo "Setting GSSAPI variables"
export GSSAPI_HOST=${SASL_HOST}
export GSSAPI_PORT=${SASL_PORT}
export GSSAPI_PRINCIPAL=${PRINCIPAL}
fi
# Start the test runner.
echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..."
echo "UV_ARGS=${UV_ARGS}"
uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@"
echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done."
if [ -n "$TEST_LOADBALANCER" ]; then
export LOAD_BALANCER=1
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}"
export TEST_ARGS="test/test_load_balancer.py"
fi
cleanup_tests
if [ "$SSL" != "nossl" ]; then
export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
if [ -n "$TEST_LOADBALANCER" ]; then
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}&tls=true"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}&tls=true"
fi
fi
if [ "$COMPRESSORS" = "snappy" ]; then
python -m pip install '.[snappy]'
if [ "$OLD_PYPY" == "True" ]; then
pip install "python-snappy<0.7.0"
fi
PYTHON=python
elif [ "$COMPRESSORS" = "zstd" ]; then
python -m pip install zstandard
fi
# PyOpenSSL test setup.
if [ -n "$TEST_PYOPENSSL" ]; then
python -m pip install '.[ocsp]'
fi
if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
python -m pip install '.[encryption]'
# Install libmongocrypt if necessary.
if [ ! -d "libmongocrypt" ]; then
bash ./.evergreen/setup-libmongocrypt.sh
fi
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
BASE=$(pwd)/libmongocrypt/nocrypto
if [ -f "${BASE}/lib/libmongocrypt.so" ]; then
PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so
elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then
PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib
elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then
PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll
# libmongocrypt's windows dll is not marked executable.
chmod +x $PYMONGOCRYPT_LIB
PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB)
elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then
PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so
else
echo "Cannot find libmongocrypt shared object file"
exit 1
fi
export PYMONGOCRYPT_LIB
# TODO: Test with 'pip install pymongocrypt'
if [ ! -d "libmongocrypt_git" ]; then
git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
fi
python -m pip install ./libmongocrypt_git/bindings/python
python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
# PATH is updated by PREPARE_SHELL for access to mongocryptd.
fi
if [ -n "$TEST_ENCRYPTION" ]; then
if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then
python -m pip install '.[ocsp]'
fi
if [ -n "$TEST_CRYPT_SHARED" ]; then
CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH`
echo "using crypt_shared_dir $CRYPT_SHARED_DIR"
export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:$DYLD_FALLBACK_LIBRARY_PATH
export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:$LD_LIBRARY_PATH
export PATH=$CRYPT_SHARED_DIR:$PATH
fi
# Only run the encryption tests.
if [ -z "$TEST_ARGS" ]; then
TEST_ARGS="test/test_encryption.py"
fi
fi
if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
if [[ -z "$SUCCESS" ]]; then
echo "Must define SUCCESS"
exit 1
fi
if echo "$MONGODB_URI" | grep -q "@"; then
echo "MONGODB_URI unexpectedly contains user credentials in FLE test!";
exit 1
fi
if [ -z "$TEST_ARGS" ]; then
TEST_ARGS="test/test_on_demand_csfle.py"
fi
fi
if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh
export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}"
set +x
export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
set -x
TEST_ARGS="test/test_index_management.py"
fi
if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then
TEST_ARGS="test/test_data_lake.py"
fi
if [ -n "$TEST_ATLAS" ]; then
TEST_ARGS="test/atlas/test_connection.py"
fi
if [ -n "$TEST_OCSP" ]; then
python -m pip install ".[ocsp]"
TEST_ARGS="test/ocsp/test_ocsp.py"
fi
if [ -n "$TEST_AUTH_AWS" ]; then
python -m pip install ".[aws]"
TEST_ARGS="test/auth_aws/test_auth_aws.py"
fi
if [ -n "$TEST_AUTH_OIDC" ]; then
python -m pip install ".[aws]"
TEST_ARGS="test/auth_oidc/test_auth_oidc.py $TEST_ARGS"
fi
if [ -n "$PERF_TEST" ]; then
python -m pip install simplejson
start_time=$(date +%s)
TEST_ARGS="test/performance/perf_test.py"
fi
echo "Running $AUTH tests over $SSL with python $PYTHON"
python -c 'import sys; print(sys.version)'
# Run the tests, and store the results in Evergreen compatible XUnit XML
# files in the xunit-results/ directory.
# Run the tests with coverage if requested and coverage is installed.
# Only cover CPython. PyPy reports suspiciously low coverage.
PYTHON_IMPL=$($PYTHON -c "import platform; print(platform.python_implementation())")
if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
# Keep in sync with combine-coverage.sh.
# coverage >=5 is needed for relative_files=true.
python -m pip install pytest-cov "coverage>=5,<=7.5"
TEST_ARGS="$TEST_ARGS --cov"
fi
if [ -n "$GREEN_FRAMEWORK" ]; then
python -m pip install $GREEN_FRAMEWORK
fi
# Show the installed packages
PIP_QUIET=0 python -m pip list
if [ -z "$GREEN_FRAMEWORK" ]; then
# Use --capture=tee-sys so pytest prints test output inline:
# https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_ARGS
else
python green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
fi
# Handle perf test post actions.
if [ -n "$PERF_TEST" ]; then
end_time=$(date +%s)
elapsed_secs=$((end_time-start_time))
cat results.json
echo "{\"failures\": 0, \"results\": [{\"status\": \"pass\", \"exit_code\": 0, \"test_file\": \"BenchMarkTests\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json
cat report.json
fi
# Handle coverage post actions.
if [ -n "$COVERAGE" ]; then
rm -rf .pytest_cache
fi

View File

@ -1,14 +0,0 @@
#!/bin/bash
# Clean up resources at the end of an evergreen run.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
# Try to source the env file.
if [ -f $HERE/env.sh ]; then
echo "Sourcing env file"
source $HERE/env.sh
fi
rm -rf "${DRIVERS_TOOLS}" || true
rm -f $HERE/../../secrets-export.sh || true

View File

@ -1,114 +0,0 @@
#!/bin/bash
# Configure an evergreen test environment.
set -eu
# Get the current unique version of this checkout
# shellcheck disable=SC2154
if [ "${is_patch:-}" = "true" ]; then
# shellcheck disable=SC2154
CURRENT_VERSION="$(git describe)-patch-$version_id"
else
CURRENT_VERSION=latest
fi
PROJECT_DIRECTORY="$(pwd)"
DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
if [ "${CI:-}" == "true" ]; then
PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
# We want to use a path that's already on PATH on spawn hosts.
else
PYMONGO_BIN_DIR=$HOME/cli_bin
fi
PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH"
# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
CARGO_HOME=$(cygpath -m $CARGO_HOME)
UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES")
MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES")
PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR")
fi
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
if [ -f "$SCRIPT_DIR/env.sh" ]; then
echo "Reading $SCRIPT_DIR/env.sh file"
. "$SCRIPT_DIR/env.sh"
exit 0
fi
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
cat <<EOT > "$SCRIPT_DIR"/env.sh
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export CURRENT_VERSION="$CURRENT_VERSION"
export DRIVERS_TOOLS="$DRIVERS_TOOLS"
export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
export MONGODB_BINARIES="$MONGODB_BINARIES"
export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export CARGO_HOME="$CARGO_HOME"
export UV_TOOL_DIR="$UV_TOOL_DIR"
export UV_CACHE_DIR="$UV_CACHE_DIR"
export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR"
export PATH="$PATH_EXT"
# shellcheck disable=SC2154
export PROJECT="${project:-mongo-python-driver}"
export PIP_QUIET=1
EOT
# Write the .env file for drivers-tools.
rm -rf $DRIVERS_TOOLS
BRANCH=master
ORG=mongodb-labs
git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS
cat <<EOT > ${DRIVERS_TOOLS}/.env
SKIP_LEGACY_SHELL=1
DRIVERS_TOOLS="$DRIVERS_TOOLS"
MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
MONGODB_BINARIES="$MONGODB_BINARIES"
EOT
# Add these expansions to make it easier to call out tests scripts from the EVG yaml
cat <<EOT > expansion.yml
DRIVERS_TOOLS: "$DRIVERS_TOOLS"
PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
EOT
# If the toolchain is available, symlink binaries to the bin dir. This has to be done
# after drivers-tools is cloned, since we might be using its binary dir.
_bin_path=""
if [ "Windows_NT" == "${OS:-}" ]; then
_bin_path="/cygdrive/c/Python/Current/Scripts"
elif [ "$(uname -s)" == "Darwin" ]; then
_bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin"
else
_bin_path="/opt/python/Current/bin"
fi
if [ -d "${_bin_path}" ]; then
_suffix=""
if [ "Windows_NT" == "${OS:-}" ]; then
_suffix=".exe"
fi
echo "Symlinking binaries from toolchain"
mkdir -p $PYMONGO_BIN_DIR
ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix}
ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix}
ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix}
fi

View File

@ -1,50 +0,0 @@
#!/usr/bin/env bash
tools="$(realpath -s "../drivers-tools")"
pushd $tools/.evergreen/github_app || exit
owner="mongodb"
repo="mongo-python-driver"
# Bootstrap the app.
echo "bootstrapping"
source utils.sh
bootstrap drivers/comment-bot
# Run the app.
source ./secrets-export.sh
# Get a github access token for the git checkout.
echo "Getting github token..."
token=$(bash ./get-access-token.sh $repo $owner)
if [ -z "${token}" ]; then
echo "Failed to get github access token!"
popd || exit
exit 1
fi
echo "Getting github token... done."
popd || exit
# Make the git checkout and create a new branch.
echo "Creating the git checkout..."
branch="spec-resync-"$(date '+%m-%d-%Y')
git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git
git checkout -b $branch "origin/master"
git add ./test
git commit -am "resyncing specs $(date '+%m-%d-%Y')"
echo "Creating the git checkout... done."
git push origin $branch
resp=$(curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $token" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"master\"}" \
--url https://api.github.com/repos/$owner/$repo/pulls)
echo $resp | jq '.html_url'
echo "Creating the PR... done."
rm -rf $tools

View File

@ -1,4 +0,0 @@
#!/bin/bash
# Download all the task coverage files.
set -eu
aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/

View File

@ -1,6 +0,0 @@
#!/bin/bash
# Entry point for the generate-config pre-commit hook.
set -eu
python .evergreen/scripts/generate_config.py

File diff suppressed because it is too large Load Diff

View File

@ -1,357 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass
from inspect import getmembers, isfunction
from itertools import cycle, zip_longest
from pathlib import Path
from typing import Any
from shrub.v3.evg_build_variant import BuildVariant
from shrub.v3.evg_command import (
EvgCommandType,
ec2_assume_role,
s3_put,
subprocess_exec,
)
from shrub.v3.evg_project import EvgProject
from shrub.v3.evg_task import EvgTaskRef
from shrub.v3.shrub_service import ShrubService
##############
# Globals
##############
ALL_VERSIONS = ["4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"]
CPYTHONS = ["3.10", "3.11", "3.12", "3.13", "3.14t", "3.14"]
PYPYS = ["pypy3.11"]
MIN_SUPPORT_VERSIONS = ["3.9", "pypy3.9", "pypy3.10"]
ALL_PYTHONS = CPYTHONS + PYPYS
MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]]
BATCHTIME_WEEK = 10080
BATCHTIME_DAY = 1440
AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")]
TOPOLOGIES = ["standalone", "replica_set", "sharded_cluster"]
C_EXTS = ["without_ext", "with_ext"]
SYNCS = ["sync", "async"]
DISPLAY_LOOKUP = dict(
ssl=dict(ssl="SSL", nossl="NoSSL"),
auth=dict(auth="Auth", noauth="NoAuth"),
topology=dict(
standalone="Standalone", replica_set="Replica Set", sharded_cluster="Sharded Cluster"
),
test_suites=dict(default="Sync", default_async="Async"),
sync={"sync": "Sync", "async": "Async"},
coverage={"1": "cov"},
no_ext={"1": "No C"},
test_min_deps={"1": "Min Deps"},
)
HOSTS = dict()
@dataclass
class Host:
name: str
run_on: str
display_name: str
variables: dict[str, str] | None
# Hosts with toolchains.
HOSTS["rhel8"] = Host("rhel8", "rhel87-small", "RHEL8", dict())
HOSTS["win64"] = Host("win64", "windows-64-vsMulti-small", "Win64", dict())
HOSTS["win-latest"] = Host("win-latest", "windows-2022-latest-small", "WinLatest", dict())
HOSTS["win32"] = Host("win32", "windows-64-vsMulti-small", "Win32", dict())
HOSTS["macos"] = Host("macos", "macos-14", "macOS", dict())
HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64", dict())
HOSTS["ubuntu22"] = Host("ubuntu22", "ubuntu2204-small", "Ubuntu-22", dict())
HOSTS["ubuntu24"] = Host("ubuntu24", "ubuntu2404-small", "Ubuntu-24", dict())
HOSTS["perf"] = Host("perf", "rhel90-dbx-perf-large", "", dict())
HOSTS["debian11"] = Host("debian11", "debian11-small", "Debian11", dict())
DEFAULT_HOST = HOSTS["rhel8"]
# Other hosts
OTHER_HOSTS = ["RHEL9-FIPS", "RHEL8-zseries", "RHEL8-POWER8", "RHEL8-arm64", "Amazon2023"]
for name, run_on in zip(
OTHER_HOSTS,
[
"rhel92-fips",
"rhel8-zseries-small",
"rhel8-power-small",
"rhel82-arm64-small",
"amazon2023-arm64-latest-large-m8g",
],
):
HOSTS[name] = Host(name, run_on, name, dict())
##############
# Helpers
##############
def create_variant_generic(
tasks: list[str | EvgTaskRef],
display_name: str,
*,
host: Host | str | None = None,
default_run_on="rhel87-small",
expansions: dict | None = None,
**kwargs: Any,
) -> BuildVariant:
"""Create a build variant for the given inputs."""
task_refs = []
if isinstance(host, str):
host = HOSTS[host]
for t in tasks:
if isinstance(t, EvgTaskRef):
task_refs.append(t)
else:
task_refs.append(EvgTaskRef(name=t))
expansions = expansions and expansions.copy() or dict()
if "run_on" in kwargs:
run_on = kwargs.pop("run_on")
elif host:
run_on = [host.run_on]
if host.variables:
expansions.update(host.variables)
else:
run_on = [default_run_on]
if isinstance(run_on, str):
run_on = [run_on]
name = display_name.replace(" ", "-").replace("*-", "").lower()
return BuildVariant(
name=name,
display_name=display_name,
tasks=task_refs,
expansions=expansions or None,
run_on=run_on,
**kwargs,
)
def create_variant(
tasks: list[str | EvgTaskRef],
display_name: str,
*,
version: str | None = None,
host: Host | str | None = None,
expansions: dict | None = None,
**kwargs: Any,
) -> BuildVariant:
expansions = expansions and expansions.copy() or dict()
if version:
expansions["VERSION"] = version
# 8.0+ Windows builds must run on win-latest
if (
"win64" in display_name.lower()
or "win32" in display_name.lower()
and version
and version >= "8.0"
):
kwargs["run_on"] = HOSTS["win-latest"].run_on
return create_variant_generic(
tasks, display_name, version=version, host=host, expansions=expansions, **kwargs
)
def get_versions_from(min_version: str) -> list[str]:
"""Get all server versions starting from a minimum version."""
min_version_float = float(min_version)
rapid_latest = ["rapid", "latest"]
versions = [v for v in ALL_VERSIONS if v not in rapid_latest]
return [v for v in versions if float(v) >= min_version_float] + rapid_latest
def get_versions_until(max_version: str) -> list[str]:
"""Get all server version up to a max version."""
max_version_float = float(max_version)
versions = [v for v in ALL_VERSIONS if v not in ["rapid", "latest"]]
versions = [v for v in versions if float(v) <= max_version_float]
if not len(versions):
raise ValueError(f"No server versions found less <= {max_version}")
return versions
def get_common_name(base: str, sep: str, **kwargs) -> str:
display_name = base
version = kwargs.pop("VERSION", None)
version = version or kwargs.pop("version", None)
if version:
if version not in ["rapid", "latest"]:
version = f"v{version}"
display_name = f"{display_name}{sep}{version}"
for key, value in kwargs.items():
name = value
if key.lower() in ["python", "toolchain_version"]:
if not value.startswith("pypy"):
name = f"Python{value}"
else:
name = f"PyPy{value.replace('pypy', '')}"
elif key.lower() in DISPLAY_LOOKUP and value in DISPLAY_LOOKUP[key.lower()]:
name = DISPLAY_LOOKUP[key.lower()][value]
else:
continue
display_name = f"{display_name}{sep}{name}"
return display_name
def get_variant_name(base: str, host: str | Host | None = None, **kwargs) -> str:
"""Get the display name of a variant."""
display_name = base
if isinstance(host, str):
host = HOSTS[host]
if host is not None:
display_name += f" {host.display_name}"
return get_common_name(display_name, " ", **kwargs)
def get_task_name(base: str, **kwargs):
return get_common_name(base, "-", **kwargs).replace(" ", "-").lower()
def zip_cycle(*iterables, empty_default=None):
"""Get all combinations of the inputs, cycling over the shorter list(s)."""
cycles = [cycle(i) for i in iterables]
for _ in zip_longest(*iterables):
yield tuple(next(i, empty_default) for i in cycles)
def handle_c_ext(c_ext, expansions) -> None:
"""Handle c extension option."""
if c_ext == C_EXTS[0]:
expansions["NO_EXT"] = "1"
def get_standard_auth_ssl(topology):
auth = "auth" if topology == "sharded_cluster" else "noauth"
ssl = "nossl" if topology == "standalone" else "ssl"
return auth, ssl
def get_assume_role(**kwargs):
kwargs.setdefault("command_type", EvgCommandType.SETUP)
kwargs.setdefault("role_arn", "${assume_role_arn}")
return ec2_assume_role(**kwargs)
def get_subprocess_exec(**kwargs):
kwargs.setdefault("binary", "bash")
kwargs.setdefault("working_dir", "src")
kwargs.setdefault("command_type", EvgCommandType.TEST)
return subprocess_exec(**kwargs)
def get_s3_put(**kwargs):
kwargs["aws_key"] = "${AWS_ACCESS_KEY_ID}"
kwargs["aws_secret"] = "${AWS_SECRET_ACCESS_KEY}" # noqa:S105
kwargs["aws_session_token"] = "${AWS_SESSION_TOKEN}" # noqa:S105
kwargs["bucket"] = "${bucket_name}"
kwargs.setdefault("optional", "true")
kwargs.setdefault("permissions", "public-read")
kwargs.setdefault("content_type", "${content_type|application/x-gzip}")
kwargs.setdefault("command_type", EvgCommandType.SETUP)
return s3_put(**kwargs)
def generate_yaml(tasks=None, variants=None):
"""Generate the yaml for a given set of tasks and variants."""
project = EvgProject(tasks=tasks, buildvariants=variants)
out = ShrubService.generate_yaml(project)
# Dedent by two spaces to match what we use in config.yml
lines = [line[2:] for line in out.splitlines()]
print("\n".join(lines))
##################
# Generate Config
##################
def write_variants_to_file(mod):
here = Path(__file__).absolute().parent
target = here.parent / "generated_configs" / "variants.yml"
if target.exists():
target.unlink()
with target.open("w") as fid:
fid.write("buildvariants:\n")
for name, func in sorted(getmembers(mod, isfunction)):
if not name.endswith("_variants"):
continue
if not name.startswith("create_"):
raise ValueError("Variant creators must start with create_")
title = name.replace("create_", "").replace("_variants", "").replace("_", " ").capitalize()
project = EvgProject(tasks=None, buildvariants=func())
out = ShrubService.generate_yaml(project).splitlines()
with target.open("a") as fid:
fid.write(f" # {title} tests\n")
for line in out[1:]:
fid.write(f"{line}\n")
fid.write("\n")
# Remove extra trailing newline:
data = target.read_text().splitlines()
with target.open("w") as fid:
for line in data[:-1]:
fid.write(f"{line}\n")
def write_tasks_to_file(mod):
here = Path(__file__).absolute().parent
target = here.parent / "generated_configs" / "tasks.yml"
if target.exists():
target.unlink()
with target.open("w") as fid:
fid.write("tasks:\n")
for name, func in sorted(getmembers(mod, isfunction)):
if name.startswith("_") or not name.endswith("_tasks"):
continue
if not name.startswith("create_"):
raise ValueError("Task creators must start with create_")
title = name.replace("create_", "").replace("_tasks", "").replace("_", " ").capitalize()
project = EvgProject(tasks=func(), buildvariants=None)
out = ShrubService.generate_yaml(project).splitlines()
with target.open("a") as fid:
fid.write(f" # {title} tests\n")
for line in out[1:]:
fid.write(f"{line}\n")
fid.write("\n")
# Remove extra trailing newline:
data = target.read_text().splitlines()
with target.open("w") as fid:
for line in data[:-1]:
fid.write(f"{line}\n")
def write_functions_to_file(mod):
here = Path(__file__).absolute().parent
target = here.parent / "generated_configs" / "functions.yml"
if target.exists():
target.unlink()
with target.open("w") as fid:
fid.write("functions:\n")
functions = dict()
for name, func in sorted(getmembers(mod, isfunction)):
if name.startswith("_") or not name.endswith("_func"):
continue
if not name.startswith("create_"):
raise ValueError("Function creators must start with create_")
title = name.replace("create_", "").replace("_func", "").replace("_", " ").capitalize()
func_name, cmds = func()
functions = dict()
functions[func_name] = cmds
project = EvgProject(functions=functions, tasks=None, buildvariants=None)
out = ShrubService.generate_yaml(project).splitlines()
with target.open("a") as fid:
fid.write(f" # {title}\n")
for line in out[1:]:
fid.write(f"{line}\n")
fid.write("\n")
# Remove extra trailing newline:
data = target.read_text().splitlines()
with target.open("w") as fid:
for line in data[:-1]:
fid.write(f"{line}\n")

View File

@ -1,36 +0,0 @@
#!/bin/bash
# Install the necessary dependencies.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
pushd "$(dirname "$(dirname $HERE)")" > /dev/null
# Source the env files to pick up common variables.
if [ -f $HERE/env.sh ]; then
. $HERE/env.sh
fi
# Set up the default bin directory.
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
PYMONGO_BIN_DIR="$HOME/.local/bin"
fi
# Ensure uv is installed.
if ! command -v uv &>/dev/null; then
_BIN_DIR=$PYMONGO_BIN_DIR
mkdir -p ${_BIN_DIR}
echo "Installing uv..."
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh
if [ "Windows_NT" = "${OS:-}" ]; then
chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
fi
export PATH="$PYMONGO_BIN_DIR:$PATH"
echo "Installing uv... done."
fi
# Ensure just is installed.
if ! command -v just &>/dev/null; then
uv tool install rust-just
fi
popd > /dev/null

View File

@ -1,144 +0,0 @@
from __future__ import annotations
import os
from utils import (
DRIVERS_TOOLS,
LOGGER,
TMP_DRIVER_FILE,
create_archive,
read_env,
run_command,
write_env,
)
DIRS = dict(
gcp=f"{DRIVERS_TOOLS}/.evergreen/csfle/gcpkms",
azure=f"{DRIVERS_TOOLS}/.evergreen/csfle/azurekms",
)
def _setup_azure_vm(base_env: dict[str, str]) -> None:
LOGGER.info("Setting up Azure VM...")
azure_dir = DIRS["azure"]
env = base_env.copy()
env["AZUREKMS_SRC"] = TMP_DRIVER_FILE
env["AZUREKMS_DST"] = "~/"
run_command(f"{azure_dir}/copy-file.sh", env=env)
env = base_env.copy()
env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz"
run_command(f"{azure_dir}/run-command.sh", env=env)
env["AZUREKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
run_command(f"{azure_dir}/run-command.sh", env=env)
env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote"
run_command(f"{azure_dir}/run-command.sh", env=env)
LOGGER.info("Setting up Azure VM... done.")
def _setup_gcp_vm(base_env: dict[str, str]) -> None:
LOGGER.info("Setting up GCP VM...")
gcp_dir = DIRS["gcp"]
env = base_env.copy()
env["GCPKMS_SRC"] = TMP_DRIVER_FILE
env["GCPKMS_DST"] = f"{env['GCPKMS_INSTANCENAME']}:"
run_command(f"{gcp_dir}/copy-file.sh", env=env)
env = base_env.copy()
env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz"
run_command(f"{gcp_dir}/run-command.sh", env=env)
env["GCPKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
run_command(f"{gcp_dir}/run-command.sh", env=env)
env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote"
run_command(f"{gcp_dir}/run-command.sh", env=env)
LOGGER.info("Setting up GCP VM...")
def _load_kms_config(sub_test_target: str) -> dict[str, str]:
target_dir = DIRS[sub_test_target]
config = read_env(f"{target_dir}/secrets-export.sh")
base_env = os.environ.copy()
for key, value in config.items():
base_env[key] = str(value)
return base_env
def setup_kms(sub_test_name: str) -> None:
if "-" in sub_test_name:
sub_test_target, sub_test_type = sub_test_name.split("-")
else:
sub_test_target = sub_test_name
sub_test_type = ""
assert sub_test_target in ["azure", "gcp"], sub_test_target
assert sub_test_type in ["", "remote", "fail"], sub_test_type
success = sub_test_type != "fail"
kms_dir = DIRS[sub_test_target]
if sub_test_target == "azure":
write_env("TEST_FLE_AZURE_AUTO")
else:
write_env("TEST_FLE_GCP_AUTO")
write_env("SUCCESS", success)
# For remote tests, there is no further work required.
if sub_test_type == "remote":
return
if sub_test_target == "azure":
run_command("./setup-secrets.sh", cwd=kms_dir)
if success:
create_archive()
if sub_test_target == "azure":
os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER"
# Found using "az vm image list --output table"
os.environ[
"AZUREKMS_IMAGE"
] = "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest"
else:
os.environ["GCPKMS_IMAGEFAMILY"] = "debian-12"
run_command("./setup.sh", cwd=kms_dir)
base_env = _load_kms_config(sub_test_target)
if sub_test_target == "azure":
_setup_azure_vm(base_env)
else:
_setup_gcp_vm(base_env)
if sub_test_target == "azure":
config = read_env(f"{kms_dir}/secrets-export.sh")
if success:
write_env("AZUREKMS_VMNAME", config["AZUREKMS_VMNAME"])
write_env("KEY_NAME", config["AZUREKMS_KEYNAME"])
write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"])
def test_kms_send_to_remote(sub_test_name: str) -> None:
env = _load_kms_config(sub_test_name)
if sub_test_name == "azure":
key_name = os.environ["KEY_NAME"]
key_vault_endpoint = os.environ["KEY_VAULT_ENDPOINT"]
env[
"AZUREKMS_CMD"
] = f'KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh run-tests'
else:
env["GCPKMS_CMD"] = "./.evergreen/just.sh run-tests"
cmd = f"{DIRS[sub_test_name]}/run-command.sh"
run_command(cmd, env=env)
def teardown_kms(sub_test_name: str) -> None:
run_command(f"{DIRS[sub_test_name]}/teardown.sh")
if __name__ == "__main__":
setup_kms()

View File

@ -1,93 +0,0 @@
from __future__ import annotations
import os
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
from shutil import which
from utils import LOGGER, ROOT, run_command, write_env
def make_request(url, timeout=10):
for _ in range(int(timeout)):
try:
urllib.request.urlopen(url) # noqa: S310
return
except urllib.error.HTTPError:
pass
time.sleep(1)
raise TimeoutError(f"Failed to access {url}")
def setup_mod_wsgi(sub_test_name: str) -> None:
env = os.environ.copy()
if sub_test_name == "embedded":
env["MOD_WSGI_CONF"] = "mod_wsgi_test_embedded.conf"
elif sub_test_name == "standalone":
env["MOD_WSGI_CONF"] = "mod_wsgi_test.conf"
else:
raise ValueError("mod_wsgi sub test must be either 'standalone' or 'embedded'")
write_env("MOD_WSGI_CONF", env["MOD_WSGI_CONF"])
apache = which("apache2")
if not apache and Path("/usr/lib/apache2/mpm-prefork/apache2").exists():
apache = "/usr/lib/apache2/mpm-prefork/apache2"
if apache:
apache_config = "apache24ubuntu161404.conf"
else:
apache = which("httpd")
if not apache:
raise ValueError("Could not find apache2 or httpd")
apache_config = "apache22amazon.conf"
python_version = ".".join(str(val) for val in sys.version_info[:2])
mod_wsgi_version = 4
so_file = f"/opt/python/mod_wsgi/python_version/{python_version}/mod_wsgi_version/{mod_wsgi_version}/mod_wsgi.so"
write_env("MOD_WSGI_SO", so_file)
env["MOD_WSGI_SO"] = so_file
env["PYTHONHOME"] = f"/opt/python/{python_version}"
env["PROJECT_DIRECTORY"] = project_directory = str(ROOT)
write_env("APACHE_BINARY", apache)
write_env("APACHE_CONFIG", apache_config)
uri1 = f"http://localhost:8080/interpreter1{project_directory}"
write_env("TEST_URI1", uri1)
uri2 = f"http://localhost:8080/interpreter2{project_directory}"
write_env("TEST_URI2", uri2)
run_command(f"{apache} -k start -f {ROOT}/test/mod_wsgi_test/{apache_config}", env=env)
# Wait for the endpoints to be available.
try:
make_request(uri1, 10)
make_request(uri2, 10)
except Exception as e:
LOGGER.error(Path("error_log").read_text())
raise e
def test_mod_wsgi() -> None:
sys.path.insert(0, ROOT)
from test.mod_wsgi_test.test_client import main, parse_args
uri1 = os.environ["TEST_URI1"]
uri2 = os.environ["TEST_URI2"]
args = f"-n 25000 -t 100 parallel {uri1} {uri2}"
try:
main(*parse_args(args.split()))
args = f"-n 25000 serial {uri1} {uri2}"
main(*parse_args(args.split()))
except Exception as e:
LOGGER.error(Path("error_log").read_text())
raise e
def teardown_mod_wsgi() -> None:
apache = os.environ["APACHE_BINARY"]
apache_config = os.environ["APACHE_CONFIG"]
run_command(f"{apache} -k stop -f {ROOT}/test/mod_wsgi_test/{apache_config}")
if __name__ == "__main__":
setup_mod_wsgi()

View File

@ -1,111 +0,0 @@
from __future__ import annotations
import os
from utils import (
DRIVERS_TOOLS,
TMP_DRIVER_FILE,
create_archive,
read_env,
run_command,
write_env,
)
K8S_NAMES = ["aks", "gke", "eks"]
K8S_REMOTE_NAMES = [f"{n}-remote" for n in K8S_NAMES]
def _get_target_dir(sub_test_name: str) -> str:
if sub_test_name == "default":
target_dir = "auth_oidc"
elif sub_test_name.startswith("azure"):
target_dir = "auth_oidc/azure"
elif sub_test_name.startswith("gcp"):
target_dir = "auth_oidc/gcp"
elif sub_test_name in K8S_NAMES + K8S_REMOTE_NAMES:
target_dir = "auth_oidc/k8s"
else:
raise ValueError(f"Invalid sub test name '{sub_test_name}'")
return f"{DRIVERS_TOOLS}/.evergreen/{target_dir}"
def setup_oidc(sub_test_name: str) -> dict[str, str] | None:
target_dir = _get_target_dir(sub_test_name)
env = os.environ.copy()
if sub_test_name == "eks" and "AWS_ACCESS_KEY_ID" in os.environ:
# Store AWS creds for kubectl access.
for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]:
if key in os.environ:
write_env(key, os.environ[key])
if sub_test_name == "azure":
env["AZUREOIDC_VMNAME_PREFIX"] = "PYTHON_DRIVER"
if "-remote" not in sub_test_name:
if sub_test_name == "azure":
# Found using "az vm image list --output table"
env["AZUREOIDC_IMAGE"] = "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest"
else:
env["GCPKMS_IMAGEFAMILY"] = "debian-12"
run_command(f"bash {target_dir}/setup.sh", env=env)
if sub_test_name in K8S_NAMES:
run_command(f"bash {target_dir}/setup-pod.sh {sub_test_name}")
run_command(f"bash {target_dir}/run-self-test.sh")
return None
source_file = None
if sub_test_name == "default":
source_file = f"{target_dir}/secrets-export.sh"
elif sub_test_name in ["azure-remote", "gcp-remote"]:
source_file = "./secrets-export.sh"
if sub_test_name in K8S_REMOTE_NAMES:
return os.environ.copy()
if source_file is None:
return None
config = read_env(source_file)
write_env("MONGODB_URI_SINGLE", config["MONGODB_URI_SINGLE"])
write_env("MONGODB_URI", config["MONGODB_URI"])
write_env("DB_IP", config["MONGODB_URI"])
if sub_test_name == "default":
write_env("OIDC_TOKEN_FILE", config["OIDC_TOKEN_FILE"])
write_env("OIDC_TOKEN_DIR", config["OIDC_TOKEN_DIR"])
if "OIDC_DOMAIN" in config:
write_env("OIDC_DOMAIN", config["OIDC_DOMAIN"])
elif sub_test_name == "azure-remote":
write_env("AZUREOIDC_RESOURCE", config["AZUREOIDC_RESOURCE"])
elif sub_test_name == "gcp-remote":
write_env("GCPOIDC_AUDIENCE", config["GCPOIDC_AUDIENCE"])
return config
def test_oidc_send_to_remote(sub_test_name: str) -> None:
env = os.environ.copy()
target_dir = _get_target_dir(sub_test_name)
create_archive()
if sub_test_name in ["azure", "gcp"]:
upper_name = sub_test_name.upper()
env[f"{upper_name}OIDC_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
env[
f"{upper_name}OIDC_TEST_CMD"
] = f"OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
elif sub_test_name in K8S_NAMES:
env["K8S_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
env["K8S_TEST_CMD"] = "OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh"
run_command(f"bash {target_dir}/run-driver-test.sh", env=env)
def teardown_oidc(sub_test_name: str) -> None:
target_dir = _get_target_dir(sub_test_name)
# For k8s, make sure an error while tearing down the pod doesn't prevent
# the Altas server teardown.
error = None
if sub_test_name in K8S_NAMES:
try:
run_command(f"bash {target_dir}/teardown-pod.sh")
except Exception as e:
error = e
run_command(f"bash {target_dir}/teardown.sh")
if error:
raise error

View File

@ -1,15 +0,0 @@
#!/bin/bash
# We use the requester expansion to determine whether the data is from a mainline evergreen run or not
set -eu
# shellcheck disable=SC2154
if [ "${requester}" == "commit" ]; then
echo "is_mainline: true" >> expansion.yml
else
echo "is_mainline: false" >> expansion.yml
fi
# We parse the username out of the order_id as patches append that in and SPS does not need that information
# shellcheck disable=SC2154
echo "parsed_order_id: $(echo "${revision_order_id}" | awk -F'_' '{print $NF}')" >> expansion.yml

View File

@ -1,25 +0,0 @@
#!/bin/bash
# We use the requester expansion to determine whether the data is from a mainline evergreen run or not
set -eu
# Submit the performance data to the SPS endpoint
# shellcheck disable=SC2154
response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
"https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=${parsed_order_id}&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=${is_mainline}" \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d @results.json)
http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
# We want to throw an error if the data was not successfully submitted
if [ "$http_status" -ne 200 ]; then
echo "Error: Received HTTP status $http_status"
echo "Response Body: $response_body"
exit 1
fi
echo "Response Body: $response_body"
echo "HTTP Status: $http_status"

View File

@ -1,150 +0,0 @@
from __future__ import annotations
import argparse
import os
import pathlib
import subprocess
from argparse import Namespace
from subprocess import CalledProcessError
JIRA_FILTER = "https://jira.mongodb.org/issues/?jql=labels%20%3D%20automated-sync%20AND%20status%20!%3D%20Closed"
def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None:
"""Actually sync the specs"""
print("Beginning to sync specs")
for spec in os.scandir(directory):
if not spec.is_dir():
continue
if spec.name in ["asynchronous"]:
continue
try:
subprocess.run(
["bash", "./.evergreen/resync-specs.sh", spec.name], # noqa: S603, S607
capture_output=True,
text=True,
check=True,
)
except CalledProcessError as exc:
errored[spec.name] = exc.stderr
print("Done syncing specs")
def apply_patches(errored):
print("Beginning to apply patches")
subprocess.run(
["bash", "./.evergreen/remove-unimplemented-tests.sh"], # noqa: S603, S607
check=True,
)
try:
# Avoid shell=True by passing arguments as a list.
# Note: glob expansion doesn't work in shell=False, so we use a list of files.
patches = [str(p) for p in pathlib.Path("./.evergreen/spec-patch/").glob("*")]
if patches:
subprocess.run(
[ # noqa: S603, S607
"git",
"apply",
"-R",
"--allow-empty",
"--whitespace=fix",
*patches,
],
check=True,
stderr=subprocess.PIPE,
)
except CalledProcessError as exc:
errored["applying patches"] = exc.stderr
def check_new_spec_directories(directory: pathlib.Path) -> list[str]:
"""Check to see if there are any directories in the spec repo that don't exist in pymongo/test"""
spec_dir = pathlib.Path(os.environ["MDB_SPECS"]) / "source"
spec_set = {
entry.name.replace("-", "_")
for entry in os.scandir(spec_dir)
if entry.is_dir()
and (pathlib.Path(entry.path) / "tests").is_dir()
and len(list(os.scandir(pathlib.Path(entry.path) / "tests"))) > 1
}
test_set = {entry.name.replace("-", "_") for entry in os.scandir(directory) if entry.is_dir()}
known_mappings = {
"ocsp_support": "ocsp",
"client_side_operations_timeout": "csot",
"mongodb_handshake": "handshake",
"load_balancers": "load_balancer",
"connection_monitoring_and_pooling": "connection_monitoring",
"command_logging_and_monitoring": "command_logging",
"initial_dns_seedlist_discovery": "srv_seedlist",
"server_discovery_and_monitoring": "sdam_monitoring",
}
for k, v in known_mappings.items():
if k in spec_set:
spec_set.remove(k)
spec_set.add(v)
return list(spec_set - test_set)
def write_summary(errored: dict[str, str], new: list[str], filename: str | None) -> None:
"""Generate the PR description"""
pr_body = ""
# Avoid shell=True and complex pipes by using Python to process git output
process = subprocess.run(
["git", "diff", "--name-only"], # noqa: S603, S607
capture_output=True,
text=True,
check=True,
)
changed_files = process.stdout.strip().splitlines()
succeeded_set = set()
for f in changed_files:
parts = f.split("/")
if len(parts) > 1:
succeeded_set.add(parts[1])
succeeded = sorted(succeeded_set)
if len(succeeded) > 0:
pr_body += "The following specs were changed:\n -"
pr_body += "\n -".join(succeeded)
pr_body += "\n"
if len(errored) > 0:
pr_body += "\n\nThe following spec syncs encountered errors:"
for k, v in errored.items():
pr_body += f"\n -{k}\n```{v}\n```"
pr_body += "\n"
if len(new) > 0:
pr_body += "\n\nThe following directories are in the specification repository and not in our test directory:\n -"
pr_body += "\n -".join(new)
pr_body += "\n"
if pr_body != "":
pr_body = f"Jira tickets: {JIRA_FILTER}\n\n" + pr_body
if filename is None:
print(f"\n{pr_body}")
else:
with open(filename, "w") as f:
# replacements made for proper json
f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t"))
def main(args: Namespace):
directory = pathlib.Path("./test")
errored: dict[str, str] = {}
resync_specs(directory, errored)
apply_patches(errored)
new = check_new_spec_directories(directory)
write_summary(errored, new, args.filename)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Python Script to resync all specs and generate summary for PR."
)
parser.add_argument(
"--filename",
help="Name of file for the summary to be written into.",
default=None,
)
args = parser.parse_args()
main(args)

View File

@ -1,43 +0,0 @@
#!/usr/bin/env bash
# Run spec syncing script and create PR
set -eu
# SETUP
SRC_URL="https://github.com/mongodb/specifications.git"
# needs to be set for resync-specs.sh
SPEC_SRC="$(realpath "../specifications")"
SCRIPT="$(realpath "./.evergreen/resync-specs.sh")"
# Clone the spec repo if the directory does not exist
if [[ ! -d $SPEC_SRC ]]; then
git clone $SRC_URL $SPEC_SRC
if [[ $? -ne 0 ]]; then
echo "Error: Failed to clone repository."
exit 1
fi
fi
# Set environment variable to the cloned spec repo for resync-specs.sh
export MDB_SPECS="$SPEC_SRC"
# Check that resync-specs.sh exists and is executable
if [[ ! -x $SCRIPT ]]; then
echo "Error: $SCRIPT not found or is not executable."
exit 1
fi
PR_DESC="spec_sync.txt"
# run python script that actually does all the resyncing
if ! [ -n "${CI:-}" ]
then
# we're running locally
python3 ./.evergreen/scripts/resync-all-specs.py
else
/opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC"
if [[ -f $PR_DESC ]]; then
# changes were made -> call scrypt to create PR for us
.evergreen/scripts/create-spec-pr.sh "$PR_DESC"
rm "$PR_DESC"
fi
fi

View File

@ -1,26 +0,0 @@
#!/bin/bash
# Get the debug data for an evergreen task.
set -eu
. ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true
get_distro || true
echo $DISTRO
echo $MARCH
echo $OS
set -x
uname -a || true
ls /etc/*release* || true
cc --version || true
gcc --version || true
clang --version || true
gcov --version || true
lcov --version || true
llvm-cov --version || true
echo $PATH
ls -la /usr/local/Cellar/llvm/*/bin/ || true
ls -la /usr/local/Cellar/ || true
scan-build --version || true
genhtml --version || true
valgrind --version || true
set +x

View File

@ -1,13 +0,0 @@
#!/bin/bash
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
# Try to source the env file.
if [ -f $HERE/env.sh ]; then
echo "Sourcing env file"
source $HERE/env.sh
fi
uv run $HERE/run_server.py "$@"

View File

@ -1,59 +0,0 @@
from __future__ import annotations
import os
from typing import Any
from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command
def set_env(name: str, value: Any = "1") -> None:
os.environ[name] = str(value)
def start_server():
opts, extra_opts = get_test_options(
"Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.",
require_sub_test_name=False,
allow_extra_opts=True,
)
test_name = opts.test_name
# drivers-evergreen-tools expects the version variable to be named MONGODB_VERSION.
if "VERSION" in os.environ:
os.environ["MONGODB_VERSION"] = os.environ["VERSION"]
if test_name == "auth_aws":
set_env("AUTH_AWS")
elif test_name == "load_balancer":
set_env("LOAD_BALANCER")
elif test_name == "search_index":
os.environ["TOPOLOGY"] = "replica_set"
os.environ["MONGODB_VERSION"] = "7.0"
if not os.environ.get("TEST_CRYPT_SHARED"):
set_env("SKIP_CRYPT_SHARED")
if opts.ssl:
extra_opts.append("--ssl")
if test_name != "ocsp":
certs = ROOT / "test/certificates"
set_env("TLS_CERT_KEY_FILE", certs / "client.pem")
set_env("TLS_PEM_KEY_FILE", certs / "server.pem")
set_env("TLS_CA_FILE", certs / "ca.pem")
if opts.auth:
extra_opts.append("--auth")
if opts.verbose:
extra_opts.append("-v")
elif opts.quiet:
extra_opts.append("-q")
cmd = ["bash", f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh", "start", *extra_opts]
run_command(cmd, cwd=DRIVERS_TOOLS)
if __name__ == "__main__":
start_server()

View File

@ -1,228 +0,0 @@
from __future__ import annotations
import json
import logging
import os
import platform
import shlex
import shutil
import subprocess
import sys
from datetime import datetime
from pathlib import Path
from shutil import which
try:
import importlib_metadata
except ImportError:
from importlib import metadata as importlib_metadata
import pytest
from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
AUTH = os.environ.get("AUTH", "noauth")
SSL = os.environ.get("SSL", "nossl")
UV_ARGS = os.environ.get("UV_ARGS", "")
TEST_PERF = os.environ.get("TEST_PERF")
GREEN_FRAMEWORK = os.environ.get("GREEN_FRAMEWORK")
TEST_ARGS = os.environ.get("TEST_ARGS", "").split()
TEST_NAME = os.environ.get("TEST_NAME")
SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME")
def list_packages():
packages = set()
for distribution in importlib_metadata.distributions():
if distribution.name:
packages.add(distribution.name)
print("Package Version URL")
print("------------------- ----------- ----------------------------------------------------")
for name in sorted(packages):
distribution = importlib_metadata.distribution(name)
url = ""
if distribution.origin is not None:
url = distribution.origin.url
print(f"{name:20s}{distribution.version:12s}{url}")
print("------------------- ----------- ----------------------------------------------------\n")
def handle_perf(start_time: datetime):
end_time = datetime.now()
elapsed_secs = (end_time - start_time).total_seconds()
with open("results.json") as fid:
results = json.load(fid)
LOGGER.info("results.json:\n%s", json.dumps(results, indent=2))
results = dict(
status="PASS",
exit_code=0,
test_file="BenchMarkTests",
start=int(start_time.timestamp()),
end=int(end_time.timestamp()),
elapsed=elapsed_secs,
)
report = dict(failures=0, results=[results])
LOGGER.info("report.json\n%s", json.dumps(report, indent=2))
with open("report.json", "w", newline="\n") as fid:
json.dump(report, fid)
def handle_green_framework() -> None:
if GREEN_FRAMEWORK == "gevent":
from gevent import monkey
monkey.patch_all()
# Never run async tests with a framework.
if len(TEST_ARGS) <= 1:
TEST_ARGS.extend(["-m", "not default_async and default"])
else:
for i in range(len(TEST_ARGS) - 1):
if "-m" in TEST_ARGS[i]:
TEST_ARGS[i + 1] = f"not default_async and {TEST_ARGS[i + 1]}"
LOGGER.info(f"Running tests with {GREEN_FRAMEWORK}...")
def handle_c_ext() -> None:
if platform.python_implementation() != "CPython":
return
sys.path.insert(0, str(ROOT / "tools"))
from fail_if_no_c import main as fail_if_no_c
fail_if_no_c()
def handle_pymongocrypt() -> None:
import pymongocrypt
LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})")
LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})")
def handle_aws_lambda() -> None:
env = os.environ.copy()
target_dir = ROOT / "test/lambda"
env["TEST_LAMBDA_DIRECTORY"] = str(target_dir)
env.setdefault("AWS_REGION", "us-east-1")
dirs = ["pymongo", "gridfs", "bson"]
# Remove the original .so files.
for dname in dirs:
so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")]
for so_path in list(so_paths):
Path(so_path).unlink()
# Build the c extensions.
docker = which("docker") or which("podman")
if not docker:
raise ValueError("Could not find docker!")
image = "quay.io/pypa/manylinux2014_x86_64:latest"
run_command(
f'{docker} run --rm -v "{ROOT}:/src" --platform linux/amd64 {image} /src/test/lambda/build_internal.sh'
)
for dname in dirs:
target = ROOT / "test/lambda/mongodb" / dname
shutil.rmtree(target, ignore_errors=True)
shutil.copytree(ROOT / dname, target)
# Remove the new so files from the ROOT directory.
for dname in dirs:
so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")]
for so_path in list(so_paths):
Path(so_path).unlink()
script_name = "run-deployed-lambda-aws-tests.sh"
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/aws_lambda/{script_name}", env=env)
def run() -> None:
# Add diagnostic for python version.
print("Running with python", sys.version)
# List the installed packages.
list_packages()
# Handle green framework first so they can patch modules.
if GREEN_FRAMEWORK:
handle_green_framework()
# Ensure C extensions if applicable.
if not os.environ.get("NO_EXT"):
handle_c_ext()
if os.environ.get("PYMONGOCRYPT_LIB"):
handle_pymongocrypt()
LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}")
# Record the start time for a perf test.
if TEST_PERF:
start_time = datetime.now()
# Run mod_wsgi tests using the helper.
if TEST_NAME == "mod_wsgi":
from mod_wsgi_tester import test_mod_wsgi
test_mod_wsgi()
return
# Send kms tests to run remotely.
if TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]:
from kms_tester import test_kms_send_to_remote
test_kms_send_to_remote(SUB_TEST_NAME)
return
# Handle doctests.
if TEST_NAME == "doctest":
from sphinx.cmd.build import main
result = main("-E -b doctest doc ./doc/_build/doctest".split())
sys.exit(result)
# Send ecs tests to run remotely.
if TEST_NAME == "auth_aws" and SUB_TEST_NAME == "ecs":
run_command(f"{DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh ecs")
return
# Send OIDC tests to run remotely.
if (
TEST_NAME == "auth_oidc"
and SUB_TEST_NAME != "default"
and not SUB_TEST_NAME.endswith("-remote")
):
from oidc_tester import test_oidc_send_to_remote
test_oidc_send_to_remote(SUB_TEST_NAME)
return
# Run deployed aws lambda tests.
if TEST_NAME == "aws_lambda":
handle_aws_lambda()
return
if os.environ.get("DEBUG_LOG"):
TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG}".split())
if os.environ.get("COVERAGE"):
binary = sys.executable.replace(os.sep, "/")
cmd = f"{binary} -m coverage run -m pytest {' '.join(TEST_ARGS)} {' '.join(sys.argv[1:])}"
result = subprocess.run(shlex.split(cmd), check=False) # noqa: S603
cmd = f"{binary} -m coverage report"
subprocess.run(shlex.split(cmd), check=False) # noqa: S603
if result.returncode != 0:
print(result.stderr)
sys.exit(result.returncode)
# Run local tests.
ret = pytest.main(TEST_ARGS + sys.argv[1:])
if ret != 0:
sys.exit(ret)
# Handle perf test post actions.
if TEST_PERF:
handle_perf(start_time)
if __name__ == "__main__":
run()

View File

@ -1,58 +0,0 @@
#!/bin/bash
# Set up development environment.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
ROOT=$(dirname "$(dirname $HERE)")
# Source the env files to pick up common variables.
if [ -f $HERE/env.sh ]; then
. $HERE/env.sh
fi
# Get variables defined in test-env.sh.
if [ -f $HERE/test-env.sh ]; then
. $HERE/test-env.sh
fi
# Ensure dependencies are installed.
bash $HERE/install-dependencies.sh
# Handle the value for UV_PYTHON.
. $HERE/setup-uv-python.sh
# Only run the next part if not running on CI.
if [ -z "${CI:-}" ]; then
# Add the default install path to the path if needed.
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
export PATH="$PATH:$HOME/.local/bin"
fi
# Set up venv, making sure c extensions build unless disabled.
if [ -z "${NO_EXT:-}" ]; then
export PYMONGO_C_EXT_MUST_BUILD=1
fi
(
cd $ROOT && uv sync
)
# Set up build utilities on Windows spawn hosts.
if [ -f $HOME/.visualStudioEnv.sh ]; then
set +u
SSH_TTY=1 source $HOME/.visualStudioEnv.sh
set -u
fi
# Only set up pre-commit if we are in a git checkout.
if [ -f $HERE/.git ]; then
if ! command -v pre-commit &>/dev/null; then
uv tool install pre-commit
fi
if [ ! -f .git/hooks/pre-commit ]; then
uvx pre-commit install
fi
fi
fi

View File

@ -1,55 +0,0 @@
#!/bin/bash
# Set up the system on an evergreen host.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
pushd "$(dirname "$(dirname $HERE)")"
echo "Setting up system..."
bash .evergreen/scripts/configure-env.sh
source .evergreen/scripts/env.sh
bash $DRIVERS_TOOLS/.evergreen/setup.sh
popd
# Run spawn host-specific tasks.
if [ -z "${CI:-}" ]; then
bash $HERE/setup-dev-env.sh
fi
# Enable core dumps if enabled on the machine
# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
if [ -f /proc/self/coredump_filter ]; then
# Set the shell process (and its children processes) to dump ELF headers (bit 4),
# anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
echo 0x13 >/proc/self/coredump_filter
if [ -f /sbin/sysctl ]; then
# Check that the core pattern is set explicitly on our distro image instead
# of being the OS's default value. This ensures that coredump names are consistent
# across distros and can be picked up by Evergreen.
core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
if [ "$core_pattern" = "dump_%e.%p.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
fi
if [ "$(uname -s)" = "Darwin" ]; then
core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
if [ -w /etc/hosts ]; then
SUDO=""
else
SUDO="sudo"
fi
# Add 'server' and 'hostname_not_in_cert' as a hostnames
echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts
echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts
echo "Setting up system... done."

View File

@ -1,26 +0,0 @@
#!/bin/bash
# Set up the test environment, including secrets and services.
set -eu
# Supported/used environment variables:
# AUTH Set to enable authentication. Defaults to "noauth"
# SSL Set to enable SSL. Defaults to "nossl"
# GREEN_FRAMEWORK The green framework to test with, if any.
# COVERAGE If non-empty, run the test suite with coverage.
# COMPRESSORS If non-empty, install appropriate compressor.
# LIBMONGOCRYPT_URL The URL to download libmongocrypt.
# TEST_CRYPT_SHARED If non-empty, install crypt_shared lib.
# MONGODB_API_VERSION The mongodb api version to use in tests.
# MONGODB_URI If non-empty, use as the MONGODB_URI in tests.
# USE_ACTIVE_VENV If non-empty, use the active virtual environment.
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
# Try to source the env file.
if [ -f $SCRIPT_DIR/env.sh ]; then
source $SCRIPT_DIR/env.sh
fi
echo "Setting up tests with args \"$*\"..."
uv run ${USE_ACTIVE_VENV:+--active} "$SCRIPT_DIR/setup_tests.py" "$@"
echo "Setting up tests with args \"$*\"... done."

View File

@ -1,53 +0,0 @@
#!/bin/bash
# Set up the UV_PYTHON variable.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
# Use min supported version by default.
_python="3.10"
# Source the env files to pick up common variables.
if [ -f $HERE/env.sh ]; then
. $HERE/env.sh
fi
# Get variables defined in test-env.sh.
if [ -f $HERE/test-env.sh ]; then
. $HERE/test-env.sh
fi
if [ -z "${UV_PYTHON:-}" ]; then
set -x
# Translate a TOOLCHAIN_VERSION to UV_PYTHON.
if [ -n "${TOOLCHAIN_VERSION:-}" ]; then
_python=$TOOLCHAIN_VERSION
if [ "$(uname -s)" = "Darwin" ]; then
if [[ "$_python" == *"t"* ]]; then
binary_name="python3t"
framework_dir="PythonT"
else
binary_name="python3"
framework_dir="Python"
fi
_python=$(echo "$_python" | sed 's/t//g')
_python="/Library/Frameworks/$framework_dir.Framework/Versions/$_python/bin/$binary_name"
elif [ "Windows_NT" = "${OS:-}" ]; then
_python=$(echo $_python | cut -d. -f1,2 | sed 's/\.//g; s/t//g')
if [[ "$TOOLCHAIN_VERSION" == *"t"* ]]; then
_exe="python${TOOLCHAIN_VERSION}.exe"
else
_exe="python.exe"
fi
if [ -n "${IS_WIN32:-}" ]; then
_python="C:/python/32/Python${_python}/${_exe}"
else
_python="C:/python/Python${_python}/${_exe}"
fi
elif [ -d "/opt/python/$_python/bin" ]; then
_python="/opt/python/$_python/bin/python3"
fi
fi
export UV_PYTHON="$_python"
fi

View File

@ -1,491 +0,0 @@
from __future__ import annotations
import base64
import os
import platform
import shutil
import stat
from pathlib import Path
from urllib import request
from utils import (
DRIVERS_TOOLS,
ENV_FILE,
HERE,
LOGGER,
PLATFORM,
ROOT,
TEST_SUITE_MAP,
Distro,
get_test_options,
read_env,
run_command,
write_env,
)
# Passthrough environment variables.
PASS_THROUGH_ENV = [
"GREEN_FRAMEWORK",
"NO_EXT",
"MONGODB_API_VERSION",
"DEBUG_LOG",
"UV_PYTHON",
"REQUIRE_FIPS",
"IS_WIN32",
]
# Map the test name to test extra.
EXTRAS_MAP = {
"auth_aws": "aws",
"auth_oidc": "aws",
"encryption": "encryption",
"enterprise_auth": "gssapi",
"kms": "encryption",
"ocsp": "ocsp",
"pyopenssl": "ocsp",
}
# Map the test name to test group.
GROUP_MAP = dict(mockupdb="mockupdb", perf="perf")
# The python version used for perf tests.
PERF_PYTHON_VERSION = "3.10.11"
def is_set(var: str) -> bool:
value = os.environ.get(var, "")
return len(value.strip()) > 0
def get_distro() -> Distro:
name = ""
version_id = ""
arch = platform.machine()
with open("/etc/os-release") as fid:
for line in fid.readlines():
line = line.replace('"', "") # noqa: PLW2901
if line.startswith("NAME="):
_, _, name = line.strip().partition("=")
if line.startswith("VERSION_ID="):
_, _, version_id = line.strip().partition("=")
return Distro(name=name, version_id=version_id, arch=arch)
def setup_libmongocrypt():
target = ""
if PLATFORM == "windows":
# PYTHON-2808 Ensure this machine has the CA cert for google KMS.
if is_set("TEST_FLE_GCP_AUTO"):
run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"')
target = "windows-test"
elif PLATFORM == "darwin":
target = "macos"
else:
distro = get_distro()
if distro.name.startswith("Debian"):
target = f"debian{distro.version_id}"
elif distro.name.startswith("Ubuntu"):
if distro.version_id == "20.04":
target = "debian11"
elif distro.version_id == "22.04":
target = "debian12"
elif distro.version_id == "24.04":
target = "debian13"
elif distro.name.startswith("Red Hat"):
if distro.version_id.startswith("7"):
target = "rhel-70-64-bit"
elif distro.version_id.startswith("8"):
if distro.arch == "aarch64":
target = "rhel-82-arm64"
else:
target = "rhel-80-64-bit"
if not is_set("LIBMONGOCRYPT_URL"):
if not target:
raise ValueError("Cannot find libmongocrypt target for current platform!")
url = f"https://s3.amazonaws.com/mciuploads/libmongocrypt/{target}/master/latest/libmongocrypt.tar.gz"
else:
url = os.environ["LIBMONGOCRYPT_URL"]
shutil.rmtree(HERE / "libmongocrypt", ignore_errors=True)
LOGGER.info(f"Fetching {url}...")
with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310
if response.status == 200:
with Path("libmongocrypt.tar.gz").open("wb") as f:
f.write(response.read())
Path("libmongocrypt").mkdir()
run_command("tar -xzf libmongocrypt.tar.gz -C libmongocrypt")
LOGGER.info(f"Fetching {url}... done.")
run_command("ls -la libmongocrypt")
run_command("ls -la libmongocrypt/nocrypto")
if PLATFORM == "windows":
# libmongocrypt's windows dll is not marked executable.
run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
def load_config_from_file(path: str | Path) -> dict[str, str]:
config = read_env(path)
for key, value in config.items():
write_env(key, value)
return config
def get_secrets(name: str) -> dict[str, str]:
secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling")
run_command(f"bash {secrets_dir.as_posix()}/setup-secrets.sh {name}", cwd=secrets_dir)
return load_config_from_file(secrets_dir / "secrets-export.sh")
def handle_test_env() -> None:
opts, _ = get_test_options("Set up the test environment and services.")
test_name = opts.test_name
sub_test_name = opts.sub_test_name
AUTH = "auth" if opts.auth else "noauth"
SSL = "ssl" if opts.ssl else "nossl"
TEST_ARGS = ""
# Start compiling the args we'll pass to uv.
UV_ARGS = ["--extra test --no-group dev"]
# If USE_ACTIVE_VENV is set, add --active to UV_ARGS so run-tests.sh uses the active venv.
if is_set("USE_ACTIVE_VENV"):
UV_ARGS.append("--active")
test_title = test_name
if sub_test_name:
test_title += f" {sub_test_name}"
# Create the test env file with the initial set of values.
with ENV_FILE.open("w", newline="\n") as fid:
fid.write("#!/usr/bin/env bash\n")
fid.write("set +x\n")
ENV_FILE.chmod(ENV_FILE.stat().st_mode | stat.S_IEXEC)
write_env("PIP_QUIET") # Quiet by default.
write_env("PIP_PREFER_BINARY") # Prefer binary dists by default.
# Set an environment variable for the test name and sub test name.
write_env(f"TEST_{test_name.upper()}")
write_env("TEST_NAME", test_name)
write_env("SUB_TEST_NAME", sub_test_name)
# Handle pass through env vars.
for var in PASS_THROUGH_ENV:
if is_set(var) or getattr(opts, var.lower(), ""):
write_env(var, os.environ.get(var, getattr(opts, var.lower(), "")))
if extra := EXTRAS_MAP.get(test_name, ""):
UV_ARGS.append(f"--extra {extra}")
if group := GROUP_MAP.get(test_name, ""):
UV_ARGS.append(f"--group {group}")
if opts.test_min_deps:
UV_ARGS.append("--resolution=lowest-direct")
if test_name == "auth_oidc":
from oidc_tester import setup_oidc
config = setup_oidc(sub_test_name)
if not config:
AUTH = "noauth"
if test_name in ["aws_lambda", "search_index"]:
env = os.environ.copy()
env["MONGODB_VERSION"] = "7.0"
env["LAMBDA_STACK_NAME"] = "dbx-python-lambda"
write_env("LAMBDA_STACK_NAME", env["LAMBDA_STACK_NAME"])
run_command(
f"bash {DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh",
env=env,
cwd=DRIVERS_TOOLS,
)
if test_name == "search_index":
AUTH = "auth"
if test_name == "ocsp":
SSL = "ssl"
write_env("AUTH", AUTH)
write_env("SSL", SSL)
LOGGER.info(f"Setting up '{test_title}' with {AUTH=} and {SSL=}...")
if test_name == "aws_lambda":
UV_ARGS.append("--group pip")
# Store AWS creds if they were given.
if "AWS_ACCESS_KEY_ID" in os.environ:
for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]:
if key in os.environ:
write_env(key, os.environ[key])
if AUTH != "noauth":
if test_name == "auth_oidc":
DB_USER = config["OIDC_ADMIN_USER"]
DB_PASSWORD = config["OIDC_ADMIN_PWD"]
elif test_name == "search_index":
config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh")
DB_USER = config["DRIVERS_ATLAS_LAMBDA_USER"]
DB_PASSWORD = config["DRIVERS_ATLAS_LAMBDA_PASSWORD"]
write_env("MONGODB_URI", config["MONGODB_URI"])
else:
DB_USER = "bob"
DB_PASSWORD = "pwd123" # noqa: S105
write_env("DB_USER", DB_USER)
write_env("DB_PASSWORD", DB_PASSWORD)
LOGGER.info("Added auth, DB_USER: %s", DB_USER)
if is_set("MONGODB_URI"):
write_env("PYMONGO_MUST_CONNECT", "true")
if opts.disable_test_commands:
write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
if test_name == "enterprise_auth":
config = get_secrets("drivers/enterprise_auth")
if PLATFORM == "windows":
LOGGER.info("Setting GSSAPI_PASS")
write_env("GSSAPI_PASS", config["SASL_PASS"])
write_env("GSSAPI_CANONICALIZE", "true")
else:
# BUILD-3830
krb_conf = ROOT / ".evergreen/krb5.conf.empty"
krb_conf.touch()
write_env("KRB5_CONFIG", krb_conf)
LOGGER.info("Writing keytab")
keytab = base64.b64decode(config["KEYTAB_BASE64"])
keytab_file = ROOT / ".evergreen/drivers.keytab"
with keytab_file.open("wb") as fid:
fid.write(keytab)
principal = config["PRINCIPAL"]
LOGGER.info("Running kinit")
os.environ["KRB5_CONFIG"] = str(krb_conf)
cmd = f"kinit -k -t {keytab_file} -p {principal}"
run_command(cmd)
LOGGER.info("Setting GSSAPI variables")
write_env("GSSAPI_HOST", config["SASL_HOST"])
write_env("GSSAPI_PORT", config["SASL_PORT"])
write_env("GSSAPI_PRINCIPAL", config["PRINCIPAL"])
if test_name == "doctest":
UV_ARGS.append("--extra docs")
if test_name == "load_balancer":
SINGLE_MONGOS_LB_URI = os.environ.get(
"SINGLE_MONGOS_LB_URI", "mongodb://127.0.0.1:8000/?loadBalanced=true"
)
MULTI_MONGOS_LB_URI = os.environ.get(
"MULTI_MONGOS_LB_URI", "mongodb://127.0.0.1:8001/?loadBalanced=true"
)
if SSL != "nossl":
SINGLE_MONGOS_LB_URI += "&tls=true"
MULTI_MONGOS_LB_URI += "&tls=true"
write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI)
write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI)
if not DRIVERS_TOOLS:
raise RuntimeError("Missing DRIVERS_TOOLS")
cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start'
run_command(cmd)
if test_name == "mod_wsgi":
from mod_wsgi_tester import setup_mod_wsgi
setup_mod_wsgi(sub_test_name)
if test_name == "ocsp":
if sub_test_name:
os.environ["OCSP_SERVER_TYPE"] = sub_test_name
for name in ["OCSP_SERVER_TYPE", "ORCHESTRATION_FILE"]:
if name not in os.environ:
raise ValueError(f"Please set {name}")
server_type = os.environ["OCSP_SERVER_TYPE"]
orch_file = os.environ["ORCHESTRATION_FILE"]
ocsp_algo = orch_file.split("-")[0]
if server_type == "no-responder":
tls_should_succeed = "false" if "mustStaple-disableStapling" in orch_file else "true"
else:
tls_should_succeed = "true" if "valid" in server_type else "false"
write_env("OCSP_TLS_SHOULD_SUCCEED", tls_should_succeed)
write_env("CA_FILE", f"{DRIVERS_TOOLS}/.evergreen/ocsp/{ocsp_algo}/ca.pem")
if server_type != "no-responder":
env = os.environ.copy()
env["SERVER_TYPE"] = server_type
env["OCSP_ALGORITHM"] = ocsp_algo
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh", env=env)
# The mock OCSP responder MUST BE started before the mongod as the mongod expects that
# a responder will be available upon startup.
version = os.environ.get("VERSION", "latest")
cmd = [
"bash",
f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh",
"start",
"--ssl",
"--version",
version,
]
if opts.verbose:
cmd.append("-v")
elif opts.quiet:
cmd.append("-q")
run_command(cmd, cwd=DRIVERS_TOOLS)
if SSL != "nossl":
if not DRIVERS_TOOLS:
raise RuntimeError("Missing DRIVERS_TOOLS")
write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
compressors = os.environ.get("COMPRESSORS") or opts.compressor
if compressors == "snappy":
UV_ARGS.append("--extra snappy")
elif compressors == "zstd":
UV_ARGS.append("--extra zstd")
if test_name in ["encryption", "kms"]:
# Check for libmongocrypt download.
if not (ROOT / "libmongocrypt").exists():
setup_libmongocrypt()
if not opts.test_min_deps:
UV_ARGS.append(
"--with pymongocrypt@git+https://github.com/mongodb/libmongocrypt@master#subdirectory=bindings/python"
)
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
BASE = ROOT / "libmongocrypt/nocrypto"
if PLATFORM == "linux":
if (BASE / "lib/libmongocrypt.so").exists():
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so"
else:
PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so"
elif PLATFORM == "darwin":
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib"
else:
PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll"
if not PYMONGOCRYPT_LIB.exists():
raise RuntimeError("Cannot find libmongocrypt shared object file")
write_env("PYMONGOCRYPT_LIB", PYMONGOCRYPT_LIB.as_posix())
# PATH is updated by configure-env.sh for access to mongocryptd.
if test_name == "encryption":
if not DRIVERS_TOOLS:
raise RuntimeError("Missing DRIVERS_TOOLS")
csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle")
run_command(f"bash {csfle_dir.as_posix()}/setup-secrets.sh", cwd=csfle_dir)
load_config_from_file(csfle_dir / "secrets-export.sh")
run_command(f"bash {csfle_dir.as_posix()}/start-servers.sh")
if sub_test_name == "pyopenssl":
UV_ARGS.append("--extra ocsp")
if opts.crypt_shared:
config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh")
CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
if PLATFORM == "windows":
write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH")
else:
write_env(
"DYLD_FALLBACK_LIBRARY_PATH",
f"{CRYPT_SHARED_DIR}:${{DYLD_FALLBACK_LIBRARY_PATH:-}}",
)
write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}")
if test_name == "kms":
from kms_tester import setup_kms
setup_kms(sub_test_name)
if test_name == "auth_aws" and sub_test_name != "ecs-remote":
auth_aws_dir = f"{DRIVERS_TOOLS}/.evergreen/auth_aws"
if "AWS_ROLE_SESSION_NAME" in os.environ:
write_env("AWS_ROLE_SESSION_NAME")
if sub_test_name != "ecs":
aws_setup = f"{auth_aws_dir}/aws_setup.sh"
run_command(f"bash {aws_setup} {sub_test_name}")
creds = read_env(f"{auth_aws_dir}/test-env.sh")
for name, value in creds.items():
write_env(name, value)
else:
run_command(f"bash {auth_aws_dir}/setup-secrets.sh")
if test_name == "atlas_connect":
secrets = get_secrets("drivers/atlas_connect")
# Write file with Atlas X509 client certificate:
decoded = base64.b64decode(secrets["ATLAS_X509_DEV_CERT_BASE64"]).decode("utf8")
cert_file = ROOT / ".evergreen/atlas_x509_dev_client_certificate.pem"
with cert_file.open("w") as file:
file.write(decoded)
write_env(
"ATLAS_X509_DEV_WITH_CERT",
secrets["ATLAS_X509_DEV"] + "&tlsCertificateKeyFile=" + str(cert_file),
)
# We do not want the default client_context to be initialized.
write_env("DISABLE_CONTEXT")
if test_name == "numpy":
UV_ARGS.append("--with numpy")
if test_name == "perf":
data_dir = ROOT / "specifications/source/benchmarking/data"
if not data_dir.exists():
run_command("git clone --depth 1 https://github.com/mongodb/specifications.git")
run_command("tar xf extended_bson.tgz", cwd=data_dir)
run_command("tar xf parallel.tgz", cwd=data_dir)
run_command("tar xf single_and_multi_document.tgz", cwd=data_dir)
write_env("TEST_PATH", str(data_dir))
write_env("OUTPUT_FILE", str(ROOT / "results.json"))
# Overwrite the UV_PYTHON from the env.sh file.
write_env("UV_PYTHON", "")
UV_ARGS.append(f"--python={PERF_PYTHON_VERSION}")
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
# affects the benchmark results.
if sub_test_name == "sync":
TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}"
else:
TEST_ARGS = f"test/performance/async_perf_test.py {TEST_ARGS}"
# Add coverage if requested.
# Only cover CPython. PyPy reports suspiciously low coverage.
if opts.cov and platform.python_implementation() == "CPython":
# Keep in sync with combine-coverage.sh.
# coverage >=5 is needed for relative_files=true.
UV_ARGS.append("--group coverage")
write_env("COVERAGE")
if opts.green_framework:
framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"]
UV_ARGS.append(f"--group {framework}")
if framework == "gevent" and opts.test_min_deps:
# PYTHON-5729. This can be removed when the min supported gevent is moved to 25.9.1.
UV_ARGS.append('--with "setuptools==81.0"')
else:
TEST_ARGS = f"-v --durations=5 {TEST_ARGS}"
TEST_SUITE = TEST_SUITE_MAP.get(test_name)
if TEST_SUITE:
TEST_ARGS = f"-m {TEST_SUITE} {TEST_ARGS}"
write_env("TEST_ARGS", TEST_ARGS)
write_env("UV_ARGS", " ".join(UV_ARGS))
LOGGER.info(f"Setting up test '{test_title}' with {AUTH=} and {SSL=}... done.")
if __name__ == "__main__":
handle_test_env()

View File

@ -1,14 +0,0 @@
#!/bin/bash
# Stop a server that was started using run-mongodb.sh in DRIVERS_TOOLS.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
# Try to source the env file.
if [ -f $HERE/env.sh ]; then
echo "Sourcing env file"
source $HERE/env.sh
fi
bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop

View File

@ -1,24 +0,0 @@
#!/bin/bash
# Tear down any services that were used by tests.
set -eu
SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
# Try to source the env file.
if [ -f $SCRIPT_DIR/env.sh ]; then
echo "Sourcing env inputs"
. $SCRIPT_DIR/env.sh
else
echo "Not sourcing env inputs"
fi
# Handle test inputs.
if [ -f $SCRIPT_DIR/test-env.sh ]; then
echo "Sourcing test inputs"
. $SCRIPT_DIR/test-env.sh
else
echo "Missing test inputs, please run 'just setup-tests'"
fi
# Teardown the test runner.
uv run $SCRIPT_DIR/teardown_tests.py

View File

@ -1,64 +0,0 @@
from __future__ import annotations
import os
import shutil
import sys
from pathlib import Path
from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
TEST_NAME = os.environ.get("TEST_NAME", "unconfigured")
SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME")
LOGGER.info(f"Tearing down tests of type '{TEST_NAME}'...")
# Shut down csfle servers if applicable.
if TEST_NAME == "encryption":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh")
# Shut down load balancer if applicable.
elif TEST_NAME == "load-balancer":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop")
# Tear down kms VM if applicable.
elif TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]:
from kms_tester import teardown_kms
teardown_kms(SUB_TEST_NAME)
# Tear down OIDC if applicable.
elif TEST_NAME == "auth_oidc":
from oidc_tester import teardown_oidc
teardown_oidc(SUB_TEST_NAME)
# Tear down ocsp if applicable.
elif TEST_NAME == "ocsp":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh")
# Tear down atlas cluster if applicable.
if TEST_NAME in ["aws_lambda", "search_index"]:
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh")
# Tear down auth_aws if applicable.
# We do not run web-identity hosts on macos, because the hosts lack permissions,
# so there is no reason to run the teardown, which would error with a 401.
elif TEST_NAME == "auth_aws" and sys.platform != "darwin":
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/auth_aws/teardown.sh")
# Tear down perf if applicable.
elif TEST_NAME == "perf":
shutil.rmtree(ROOT / "specifications", ignore_errors=True)
Path(os.environ["OUTPUT_FILE"]).unlink(missing_ok=True)
# Tear down mog_wsgi if applicable.
elif TEST_NAME == "mod_wsgi":
from mod_wsgi_tester import teardown_mod_wsgi
teardown_mod_wsgi()
# Tear down coverage if applicable.
if os.environ.get("COVERAGE"):
shutil.rmtree(".pytest_cache", ignore_errors=True)
LOGGER.info(f"Tearing down tests of type '{TEST_NAME}'... done.")

View File

@ -1,57 +0,0 @@
#!/bin/bash
# shellcheck disable=SC2154
# Upload a coverate report to codecov.
set -eu
HERE=$(dirname ${BASH_SOURCE:-$0})
ROOT=$(dirname "$(dirname $HERE)")
pushd $ROOT > /dev/null
export FNAME=coverage.xml
REQUESTER=${requester:-}
if [ ! -f ".coverage" ]; then
echo "There are no coverage results, not running codecov"
exit 0
fi
if [[ "${REQUESTER}" == "github_pr" || "${REQUESTER}" == "commit" ]]; then
echo "Uploading codecov for $REQUESTER..."
else
echo "Error: requester must be 'github_pr' or 'commit', got '${REQUESTER}'" >&2
exit 1
fi
printf 'sha: %s\n' "$github_commit"
printf 'flag: %s-%s\n' "$build_variant" "$task_name"
printf 'file: %s\n' "$FNAME"
uv tool run --with "coverage[toml]" coverage xml
codecov_args=(
upload-process
--report-type coverage
--disable-search
--fail-on-error
--git-service github
--token "${CODECOV_TOKEN}"
--sha "${github_commit}"
--flag "${build_variant}-${task_name}"
--file "${FNAME}"
)
if [ -n "${github_pr_number:-}" ]; then
printf 'branch: %s:%s\n' "$github_author" "$github_pr_head_branch"
printf 'pr: %s\n' "$github_pr_number"
uv tool run --from codecov-cli codecovcli \
"${codecov_args[@]}" \
--pr "${github_pr_number}" \
--branch "${github_author}:${github_pr_head_branch}"
else
printf 'branch: %s\n' "$branch_name"
uv tool run --from codecov-cli codecovcli \
"${codecov_args[@]}" \
--branch "${branch_name}"
fi
echo "Uploading codecov for $REQUESTER... done."
popd > /dev/null

View File

@ -1,4 +0,0 @@
#!/bin/bash
# Upload a coverate report to s3.
set -eu
aws s3 cp htmlcov/ s3://"$1"/coverage/"$2"/"$3"/htmlcov/ --recursive --acl public-read --region us-east-1

View File

@ -1,228 +0,0 @@
from __future__ import annotations
import argparse
import dataclasses
import logging
import os
import shlex
import subprocess
import sys
from pathlib import Path
from typing import Any
HERE = Path(__file__).absolute().parent
ROOT = HERE.parent.parent
DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108
LOGGER = logging.getLogger("test")
logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
ENV_FILE = HERE / "test-env.sh"
PLATFORM = "windows" if os.name == "nt" else sys.platform.lower()
@dataclasses.dataclass
class Distro:
name: str
version_id: str
arch: str
# Map the test name to a test suite.
TEST_SUITE_MAP = {
"atlas_connect": "atlas_connect",
"auth_aws": "auth_aws",
"auth_oidc": "auth_oidc",
"default": "",
"default_async": "default_async",
"default_sync": "default",
"encryption": "encryption",
"enterprise_auth": "auth",
"search_index": "search_index",
"kms": "kms",
"load_balancer": "load_balancer",
"mockupdb": "mockupdb",
"ocsp": "ocsp",
"perf": "perf",
"numpy": "",
}
# Tests that require a sub test suite.
SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "doctest"]
# Tests that do not use run-mongodb directly.
NO_RUN_ORCHESTRATION = [
"auth_oidc",
"atlas_connect",
"aws_lambda",
"mockupdb",
"ocsp",
]
# Mapping of env variables to options
OPTION_TO_ENV_VAR = {"cov": "COVERAGE", "crypt_shared": "TEST_CRYPT_SHARED"}
def get_test_options(
description, require_sub_test_name=True, allow_extra_opts=False
) -> tuple[argparse.Namespace, list[str]]:
parser = argparse.ArgumentParser(
description=description, formatter_class=argparse.RawDescriptionHelpFormatter
)
if require_sub_test_name:
parser.add_argument(
"test_name",
choices=sorted(list(TEST_SUITE_MAP) + EXTRA_TESTS),
nargs="?",
default="default",
help="The optional name of the test suite to set up, typically the same name as a pytest marker.",
)
parser.add_argument(
"sub_test_name", nargs="?", help="The optional sub test name, for example 'azure'."
)
else:
parser.add_argument(
"test_name",
choices=set(list(TEST_SUITE_MAP) + EXTRA_TESTS) - set(NO_RUN_ORCHESTRATION),
nargs="?",
default="default",
help="The optional name of the test suite to be run, which informs the server configuration.",
)
parser.add_argument(
"--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level."
)
parser.add_argument(
"--quiet", "-q", action="store_true", help="Whether to log at the WARNING level."
)
parser.add_argument("--auth", action="store_true", help="Whether to add authentication.")
parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.")
parser.add_argument(
"--test-min-deps", action="store_true", help="Test against minimum dependency versions"
)
# Add the test modifiers.
if require_sub_test_name:
parser.add_argument(
"--debug-log", action="store_true", help="Enable pymongo standard logging."
)
parser.add_argument("--cov", action="store_true", help="Add test coverage.")
parser.add_argument(
"--green-framework",
nargs=1,
choices=["gevent"],
help="Optional green framework to test against.",
)
parser.add_argument(
"--compressor",
nargs=1,
choices=["zlib", "zstd", "snappy"],
help="Optional compression algorithm.",
)
parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.")
parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.")
parser.add_argument(
"--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use."
)
parser.add_argument(
"--disable-test-commands", action="store_true", help="Disable test commands."
)
# Get the options.
if not allow_extra_opts:
opts, extra_opts = parser.parse_args(), []
else:
opts, extra_opts = parser.parse_known_args()
# Convert list inputs to strings.
for name in vars(opts):
value = getattr(opts, name)
if isinstance(value, list):
setattr(opts, name, value[0])
# Handle validation and environment variable overrides.
test_name = opts.test_name
sub_test_name = opts.sub_test_name if require_sub_test_name else ""
if require_sub_test_name and test_name in SUB_TEST_REQUIRED and not sub_test_name:
raise ValueError(f"Test '{test_name}' requires a sub_test_name")
handle_env_overrides(parser, opts)
if "auth" in test_name:
opts.auth = True
# 'auth_aws ecs' shouldn't have extra auth set.
if test_name == "auth_aws" and sub_test_name == "ecs":
opts.auth = False
if opts.verbose:
LOGGER.setLevel(logging.DEBUG)
elif opts.quiet:
LOGGER.setLevel(logging.WARNING)
return opts, extra_opts
def handle_env_overrides(parser: argparse.ArgumentParser, opts: argparse.Namespace) -> None:
# Get the options, and then allow environment variable overrides.
for key in vars(opts):
if key in OPTION_TO_ENV_VAR:
env_var = OPTION_TO_ENV_VAR[key]
else:
env_var = key.upper()
if env_var in os.environ:
if parser.get_default(key) != getattr(opts, key):
LOGGER.info("Overriding env var '%s' with cli option", env_var)
elif env_var == "AUTH":
opts.auth = os.environ.get("AUTH") == "auth"
elif env_var == "SSL":
ssl_opt = os.environ.get("SSL", "")
opts.ssl = ssl_opt and ssl_opt.lower() != "nossl"
elif isinstance(getattr(opts, key), bool):
if os.environ[env_var]:
setattr(opts, key, True)
else:
setattr(opts, key, os.environ[env_var])
def read_env(path: Path | str) -> dict[str, str]:
config = dict()
with Path(path).open() as fid:
for line in fid.readlines():
if "=" not in line:
continue
name, _, value = line.strip().partition("=")
if value.startswith(('"', "'")):
value = value[1:-1]
name = name.replace("export ", "")
config[name] = value
return config
def write_env(name: str, value: Any = "1") -> None:
with ENV_FILE.open("a", newline="\n") as fid:
# Remove any existing quote chars.
value = str(value).replace('"', "")
fid.write(f'export {name}="{value}"\n')
def run_command(cmd: str | list[str], **kwargs: Any) -> None:
if isinstance(cmd, list):
cmd = " ".join(cmd)
LOGGER.info("Running command '%s'...", cmd)
kwargs.setdefault("check", True)
# Prevent overriding the python used by other tools.
env = kwargs.pop("env", os.environ).copy()
if "UV_PYTHON" in env:
del env["UV_PYTHON"]
kwargs["env"] = env
try:
subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603
except subprocess.CalledProcessError as e:
LOGGER.error(e.output)
LOGGER.error(str(e))
sys.exit(e.returncode)
LOGGER.info("Running command '%s'... done.", cmd)
def create_archive() -> str:
run_command("git add .", cwd=ROOT)
run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT)
run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT)
return TMP_DRIVER_FILE

View File

@ -0,0 +1,47 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
set -o xtrace
TARGET=""
if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
# PYTHON-2808 Ensure this machine has the CA cert for google KMS.
powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
TARGET="windows-test"
fi
if [ "$(uname -s)" = "Darwin" ]; then
TARGET="macos"
fi
if [ "$(uname -s)" = "Linux" ]; then
rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release)
arch=$(uname -m)
echo "RHEL $rhel_ver $arch"
if [[ $rhel_ver =~ 7 ]]; then
TARGET="rhel-70-64-bit"
elif [[ $rhel_ver =~ 8 ]]; then
if [ "$arch" = "x86_64" ]; then
TARGET="rhel-80-64-bit"
elif [ "$arch" = "arm" ]; then
TARGET="rhel-82-arm64"
fi
fi
fi
if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then
LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz"
fi
if [ -z "$LIBMONGOCRYPT_URL" ]; then
echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
exit 1
fi
rm -rf libmongocrypt libmongocrypt.tar.gz
echo "Fetching $LIBMONGOCRYPT_URL..."
curl -O "$LIBMONGOCRYPT_URL"
echo "Fetching $LIBMONGOCRYPT_URL...done"
mkdir libmongocrypt
tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
ls -la libmongocrypt
ls -la libmongocrypt/nocrypto

View File

@ -1,18 +0,0 @@
#!/bin/bash
# Set up a remote evergreen spawn host.
set -eu
if [ -z "$1" ]
then
echo "Must supply a spawn host URL!"
fi
target=$1
user=${target%@*}
remote_dir=/home/$user/mongo-python-driver
echo "Copying files to $target..."
rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_dir
echo "Copying files to $target... done"
ssh $target "$remote_dir/.evergreen/scripts/setup-system.sh"

View File

@ -1,24 +0,0 @@
diff --git a/test/connection_monitoring/pool-create-min-size-error.json b/test/connection_monitoring/pool-create-min-size-error.json
index 1c744b85..509b2a23 100644
--- a/test/connection_monitoring/pool-create-min-size-error.json
+++ b/test/connection_monitoring/pool-create-min-size-error.json
@@ -49,15 +49,15 @@
"type": "ConnectionCreated",
"address": 42
},
+ {
+ "type": "ConnectionPoolCleared",
+ "address": 42
+ },
{
"type": "ConnectionClosed",
"address": 42,
"connectionId": 42,
"reason": "error"
- },
- {
- "type": "ConnectionPoolCleared",
- "address": 42
}
],
"ignore": [

View File

@ -1,440 +0,0 @@
diff --git a/test/unified-test-format/invalid/entity-client-observeTracingMessages-additionalProperties.json b/test/unified-test-format/invalid/entity-client-observeTracingMessages-additionalProperties.json
new file mode 100644
index 00000000..aa8046d2
--- /dev/null
+++ b/test/unified-test-format/invalid/entity-client-observeTracingMessages-additionalProperties.json
@@ -0,0 +1,20 @@
+{
+ "description": "entity-client-observeTracingMessages-additionalProperties",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0",
+ "observeTracingMessages": {
+ "foo": "bar"
+ }
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "observeTracingMessages must not have additional properties'",
+ "operations": []
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/entity-client-observeTracingMessages-additionalPropertyType.json b/test/unified-test-format/invalid/entity-client-observeTracingMessages-additionalPropertyType.json
new file mode 100644
index 00000000..0b3a65f5
--- /dev/null
+++ b/test/unified-test-format/invalid/entity-client-observeTracingMessages-additionalPropertyType.json
@@ -0,0 +1,20 @@
+{
+ "description": "entity-client-observeTracingMessages-additionalPropertyType",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0",
+ "observeTracingMessages": {
+ "enableCommandPayload": 0
+ }
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "observeTracingMessages enableCommandPayload must be boolean",
+ "operations": []
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/entity-client-observeTracingMessages-type.json b/test/unified-test-format/invalid/entity-client-observeTracingMessages-type.json
new file mode 100644
index 00000000..de3ef39a
--- /dev/null
+++ b/test/unified-test-format/invalid/entity-client-observeTracingMessages-type.json
@@ -0,0 +1,18 @@
+{
+ "description": "entity-client-observeTracingMessages-type",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0",
+ "observeTracingMessages": "foo"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "observeTracingMessages must be an object",
+ "operations": []
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-additionalProperties.json b/test/unified-test-format/invalid/expectedTracingSpans-additionalProperties.json
new file mode 100644
index 00000000..5947a286
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-additionalProperties.json
@@ -0,0 +1,30 @@
+{
+ "description": "expectedTracingSpans-additionalProperties",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "additional property foo not allowed in expectTracingMessages",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "ignoreExtraSpans": false,
+ "spans": [
+ {
+ "name": "command",
+ "tags": {
+ "db.system": "mongodb"
+ }
+ }
+ ],
+ "foo": 0
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-clientType.json b/test/unified-test-format/invalid/expectedTracingSpans-clientType.json
new file mode 100644
index 00000000..2fe7faea
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-clientType.json
@@ -0,0 +1,28 @@
+{
+ "description": "expectedTracingSpans-clientType",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "client type must be string",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": 0,
+ "spans": [
+ {
+ "name": "command",
+ "tags": {
+ "db.system": "mongodb"
+ }
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-emptyNestedSpan.json b/test/unified-test-format/invalid/expectedTracingSpans-emptyNestedSpan.json
new file mode 100644
index 00000000..8a98d5ba
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-emptyNestedSpan.json
@@ -0,0 +1,29 @@
+{
+ "description": "expectedTracingSpans-emptyNestedSpan",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "nested spans must not have fewer than 1 items'",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "name": "command",
+ "tags": {
+ "db.system": "mongodb"
+ },
+ "nested": []
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-invalidNestedSpan.json b/test/unified-test-format/invalid/expectedTracingSpans-invalidNestedSpan.json
new file mode 100644
index 00000000..79a86744
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-invalidNestedSpan.json
@@ -0,0 +1,31 @@
+{
+ "description": "expectedTracingSpans-invalidNestedSpan",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "nested span must have required property name",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "name": "command",
+ "tags": {
+ "db.system": "mongodb"
+ },
+ "nested": [
+ {}
+ ]
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-missingPropertyClient.json b/test/unified-test-format/invalid/expectedTracingSpans-missingPropertyClient.json
new file mode 100644
index 00000000..2fb1cd5b
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-missingPropertyClient.json
@@ -0,0 +1,27 @@
+{
+ "description": "expectedTracingSpans-missingPropertyClient",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "missing required property client",
+ "operations": [],
+ "expectTracingMessages": {
+ "spans": [
+ {
+ "name": "command",
+ "tags": {
+ "db.system": "mongodb"
+ }
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-missingPropertySpans.json b/test/unified-test-format/invalid/expectedTracingSpans-missingPropertySpans.json
new file mode 100644
index 00000000..acd10307
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-missingPropertySpans.json
@@ -0,0 +1,20 @@
+{
+ "description": "expectedTracingSpans-missingPropertySpans",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "missing required property spans",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0"
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedAdditionalProperties.json b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedAdditionalProperties.json
new file mode 100644
index 00000000..17299f86
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedAdditionalProperties.json
@@ -0,0 +1,28 @@
+{
+ "description": "expectedTracingSpans-spanMalformedAdditionalProperties",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "Span must not have additional properties",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "name": "foo",
+ "tags": {},
+ "nested": [],
+ "foo": "bar"
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedMissingName.json b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedMissingName.json
new file mode 100644
index 00000000..0257cd9b
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedMissingName.json
@@ -0,0 +1,27 @@
+{
+ "description": "expectedTracingSpans-spanMalformedMissingName",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "missing required span name",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "tags": {
+ "db.system": "mongodb"
+ }
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedMissingTags.json b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedMissingTags.json
new file mode 100644
index 00000000..a09ca31c
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedMissingTags.json
@@ -0,0 +1,25 @@
+{
+ "description": "expectedTracingSpans-spanMalformedMissingTags",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "missing required span tags",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "name": "foo"
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedNestedMustBeArray.json b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedNestedMustBeArray.json
new file mode 100644
index 00000000..ccff0410
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedNestedMustBeArray.json
@@ -0,0 +1,27 @@
+{
+ "description": "expectedTracingSpans-spanMalformedNestedMustBeArray",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "nested spans must be an array",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "name": "foo",
+ "tags": {},
+ "nested": {}
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedTagsMustBeObject.json b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedTagsMustBeObject.json
new file mode 100644
index 00000000..72af1c29
--- /dev/null
+++ b/test/unified-test-format/invalid/expectedTracingSpans-spanMalformedTagsMustBeObject.json
@@ -0,0 +1,26 @@
+{
+ "description": "expectedTracingSpans-spanMalformedNestedMustBeObject",
+ "schemaVersion": "1.26",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "span tags must be an object",
+ "operations": [],
+ "expectTracingMessages": {
+ "client": "client0",
+ "spans": [
+ {
+ "name": "foo",
+ "tags": []
+ }
+ ]
+ }
+ }
+ ]
+}

View File

@ -1,26 +0,0 @@
diff --git a/test/auth/legacy/connection-string.json b/test/auth/legacy/connection-string.json
index 3a099c813..8982b61d5 100644
--- a/test/auth/legacy/connection-string.json
+++ b/test/auth/legacy/connection-string.json
@@ -440,6 +440,21 @@
}
}
},
+ {
+ "description": "should throw an exception if username provided (MONGODB-AWS)",
+ "uri": "mongodb://user@localhost.com/?authMechanism=MONGODB-AWS",
+ "valid": false
+ },
+ {
+ "description": "should throw an exception if username and password provided (MONGODB-AWS)",
+ "uri": "mongodb://user:pass@localhost.com/?authMechanism=MONGODB-AWS",
+ "valid": false
+ },
+ {
+ "description": "should throw an exception if AWS_SESSION_TOKEN provided (MONGODB-AWS)",
+ "uri": "mongodb://localhost/?authMechanism=MONGODB-AWS&authMechanismProperties=AWS_SESSION_TOKEN:token",
+ "valid": false
+ },
{
"description": "should recognise the mechanism with test environment (MONGODB-OIDC)",
"uri": "mongodb://localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:test",

View File

@ -1,50 +0,0 @@
diff --git a/test/connection_logging/connection-logging.json b/test/connection_logging/connection-logging.json
index 5799e834..72103b3c 100644
--- a/test/connection_logging/connection-logging.json
+++ b/test/connection_logging/connection-logging.json
@@ -446,6 +446,22 @@
}
}
},
+ {
+ "level": "debug",
+ "component": "connection",
+ "data": {
+ "message": "Connection pool cleared",
+ "serverHost": {
+ "$$type": "string"
+ },
+ "serverPort": {
+ "$$type": [
+ "int",
+ "long"
+ ]
+ }
+ }
+ },
{
"level": "debug",
"component": "connection",
@@ -498,22 +514,6 @@
]
}
}
- },
- {
- "level": "debug",
- "component": "connection",
- "data": {
- "message": "Connection pool cleared",
- "serverHost": {
- "$$type": "string"
- },
- "serverPort": {
- "$$type": [
- "int",
- "long"
- ]
- }
- }
}
]
}

View File

@ -1,815 +0,0 @@
diff --git a/test/sessions/snapshot-sessions.json b/test/sessions/snapshot-sessions.json
index 260f8b6f4..8f806ea75 100644
--- a/test/sessions/snapshot-sessions.json
+++ b/test/sessions/snapshot-sessions.json
@@ -988,6 +988,810 @@
}
}
]
+ },
+ {
+ "description": "Find operation with snapshot and snapshot time",
+ "operations": [
+ {
+ "name": "find",
+ "object": "collection0",
+ "arguments": {
+ "session": "session0",
+ "filter": {}
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ },
+ {
+ "_id": 2,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "getSnapshotTime",
+ "object": "session0",
+ "saveResultAsEntity": "savedSnapshotTime"
+ },
+ {
+ "name": "insertOne",
+ "object": "collection0",
+ "arguments": {
+ "document": {
+ "_id": 3,
+ "x": 33
+ }
+ }
+ },
+ {
+ "name": "createEntities",
+ "object": "testRunner",
+ "arguments": {
+ "entities": [
+ {
+ "session": {
+ "id": "session2",
+ "client": "client0",
+ "sessionOptions": {
+ "snapshot": true,
+ "snapshotTime": "savedSnapshotTime"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "find",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "filter": {}
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ },
+ {
+ "_id": 2,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "find",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "filter": {}
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ },
+ {
+ "_id": 2,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "find",
+ "object": "collection0",
+ "arguments": {
+ "filter": {}
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ },
+ {
+ "_id": 2,
+ "x": 11
+ },
+ {
+ "_id": 3,
+ "x": 33
+ }
+ ]
+ }
+ ],
+ "expectEvents": [
+ {
+ "client": "client0",
+ "events": [
+ {
+ "commandStartedEvent": {
+ "command": {
+ "find": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$exists": false
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "find": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "find": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "find": "collection0",
+ "readConcern": {
+ "$$exists": false
+ }
+ },
+ "databaseName": "database0"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "description": "Distinct operation with snapshot and snapshot time",
+ "operations": [
+ {
+ "name": "distinct",
+ "object": "collection0",
+ "arguments": {
+ "session": "session0",
+ "filter": {},
+ "fieldName": "x"
+ },
+ "expectResult": [
+ 11
+ ]
+ },
+ {
+ "name": "getSnapshotTime",
+ "object": "session0",
+ "saveResultAsEntity": "savedSnapshotTime"
+ },
+ {
+ "name": "insertOne",
+ "object": "collection0",
+ "arguments": {
+ "document": {
+ "_id": 3,
+ "x": 33
+ }
+ }
+ },
+ {
+ "name": "createEntities",
+ "object": "testRunner",
+ "arguments": {
+ "entities": [
+ {
+ "session": {
+ "id": "session2",
+ "client": "client0",
+ "sessionOptions": {
+ "snapshot": true,
+ "snapshotTime": "savedSnapshotTime"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "distinct",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "filter": {},
+ "fieldName": "x"
+ },
+ "expectResult": [
+ 11
+ ]
+ },
+ {
+ "name": "distinct",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "filter": {},
+ "fieldName": "x"
+ },
+ "expectResult": [
+ 11
+ ]
+ },
+ {
+ "name": "distinct",
+ "object": "collection0",
+ "arguments": {
+ "filter": {},
+ "fieldName": "x"
+ },
+ "expectResult": [
+ 11,
+ 33
+ ]
+ }
+ ],
+ "expectEvents": [
+ {
+ "client": "client0",
+ "events": [
+ {
+ "commandStartedEvent": {
+ "command": {
+ "distinct": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$exists": false
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "distinct": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "distinct": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "distinct": "collection0",
+ "readConcern": {
+ "$$exists": false
+ }
+ },
+ "databaseName": "database0"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "description": "Aggregate operation with snapshot and snapshot time",
+ "operations": [
+ {
+ "name": "aggregate",
+ "object": "collection0",
+ "arguments": {
+ "session": "session0",
+ "pipeline": [
+ {
+ "$match": {
+ "_id": 1
+ }
+ }
+ ]
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "getSnapshotTime",
+ "object": "session0",
+ "saveResultAsEntity": "savedSnapshotTime"
+ },
+ {
+ "name": "findOneAndUpdate",
+ "object": "collection0",
+ "arguments": {
+ "filter": {
+ "_id": 1
+ },
+ "update": {
+ "$inc": {
+ "x": 1
+ }
+ },
+ "returnDocument": "After"
+ },
+ "expectResult": {
+ "_id": 1,
+ "x": 12
+ }
+ },
+ {
+ "name": "createEntities",
+ "object": "testRunner",
+ "arguments": {
+ "entities": [
+ {
+ "session": {
+ "id": "session2",
+ "client": "client0",
+ "sessionOptions": {
+ "snapshot": true,
+ "snapshotTime": "savedSnapshotTime"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "aggregate",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "pipeline": [
+ {
+ "$match": {
+ "_id": 1
+ }
+ }
+ ]
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "aggregate",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "pipeline": [
+ {
+ "$match": {
+ "_id": 1
+ }
+ }
+ ]
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "aggregate",
+ "object": "collection0",
+ "arguments": {
+ "pipeline": [
+ {
+ "$match": {
+ "_id": 1
+ }
+ }
+ ]
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 12
+ }
+ ]
+ }
+ ],
+ "expectEvents": [
+ {
+ "client": "client0",
+ "events": [
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$exists": false
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "$$exists": false
+ }
+ },
+ "databaseName": "database0"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "description": "countDocuments operation with snapshot and snapshot time",
+ "operations": [
+ {
+ "name": "countDocuments",
+ "object": "collection0",
+ "arguments": {
+ "session": "session0",
+ "filter": {}
+ },
+ "expectResult": 2
+ },
+ {
+ "name": "getSnapshotTime",
+ "object": "session0",
+ "saveResultAsEntity": "savedSnapshotTime"
+ },
+ {
+ "name": "insertOne",
+ "object": "collection0",
+ "arguments": {
+ "document": {
+ "_id": 3,
+ "x": 33
+ }
+ }
+ },
+ {
+ "name": "createEntities",
+ "object": "testRunner",
+ "arguments": {
+ "entities": [
+ {
+ "session": {
+ "id": "session2",
+ "client": "client0",
+ "sessionOptions": {
+ "snapshot": true,
+ "snapshotTime": "savedSnapshotTime"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "countDocuments",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "filter": {}
+ },
+ "expectResult": 2
+ },
+ {
+ "name": "countDocuments",
+ "object": "collection0",
+ "arguments": {
+ "session": "session2",
+ "filter": {}
+ },
+ "expectResult": 2
+ },
+ {
+ "name": "countDocuments",
+ "object": "collection0",
+ "arguments": {
+ "filter": {}
+ },
+ "expectResult": 3
+ }
+ ],
+ "expectEvents": [
+ {
+ "client": "client0",
+ "events": [
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$exists": false
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ },
+ "databaseName": "database0"
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "$$exists": false
+ }
+ },
+ "databaseName": "database0"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "description": "Mixed operation with snapshot and snapshotTime",
+ "operations": [
+ {
+ "name": "find",
+ "object": "collection0",
+ "arguments": {
+ "session": "session0",
+ "filter": {
+ "_id": 1
+ }
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "getSnapshotTime",
+ "object": "session0",
+ "saveResultAsEntity": "savedSnapshotTime"
+ },
+ {
+ "name": "findOneAndUpdate",
+ "object": "collection0",
+ "arguments": {
+ "filter": {
+ "_id": 1
+ },
+ "update": {
+ "$inc": {
+ "x": 1
+ }
+ },
+ "returnDocument": "After"
+ },
+ "expectResult": {
+ "_id": 1,
+ "x": 12
+ }
+ },
+ {
+ "name": "createEntities",
+ "object": "testRunner",
+ "arguments": {
+ "entities": [
+ {
+ "session": {
+ "id": "session2",
+ "client": "client0",
+ "sessionOptions": {
+ "snapshot": true,
+ "snapshotTime": "savedSnapshotTime"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "find",
+ "object": "collection0",
+ "arguments": {
+ "filter": {
+ "_id": 1
+ }
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 12
+ }
+ ]
+ },
+ {
+ "name": "aggregate",
+ "object": "collection0",
+ "arguments": {
+ "pipeline": [
+ {
+ "$match": {
+ "_id": 1
+ }
+ }
+ ],
+ "session": "session2"
+ },
+ "expectResult": [
+ {
+ "_id": 1,
+ "x": 11
+ }
+ ]
+ },
+ {
+ "name": "distinct",
+ "object": "collection0",
+ "arguments": {
+ "fieldName": "x",
+ "filter": {},
+ "session": "session2"
+ },
+ "expectResult": [
+ 11
+ ]
+ }
+ ],
+ "expectEvents": [
+ {
+ "client": "client0",
+ "events": [
+ {
+ "commandStartedEvent": {
+ "command": {
+ "find": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$exists": false
+ }
+ }
+ }
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "find": "collection0",
+ "readConcern": {
+ "$$exists": false
+ }
+ }
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "aggregate": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ }
+ }
+ },
+ {
+ "commandStartedEvent": {
+ "command": {
+ "distinct": "collection0",
+ "readConcern": {
+ "level": "snapshot",
+ "atClusterTime": {
+ "$$matchesEntity": "savedSnapshotTime"
+ }
+ }
+ }
+ }
+ }
+ ]
+ }
+ ]
}
]
}

View File

@ -1,460 +0,0 @@
diff --git a/test/client-side-encryption/spec/unified/accessToken-azure.json b/test/client-side-encryption/spec/unified/accessToken-azure.json
new file mode 100644
index 00000000..510d8795
--- /dev/null
+++ b/test/client-side-encryption/spec/unified/accessToken-azure.json
@@ -0,0 +1,186 @@
+{
+ "description": "accessToken-azure",
+ "schemaVersion": "1.28",
+ "runOnRequirements": [
+ {
+ "minServerVersion": "4.1.10",
+ "csfle": {
+ "minLibmongocryptVersion": "1.6.0"
+ }
+ }
+ ],
+ "createEntities": [
+ {
+ "client": {
+ "id": "client",
+ "autoEncryptOpts": {
+ "keyVaultNamespace": "keyvault.datakeys",
+ "kmsProviders": {
+ "azure": {
+ "accessToken": {
+ "$$placeholder": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ {
+ "database": {
+ "id": "db",
+ "client": "client",
+ "databaseName": "db"
+ }
+ },
+ {
+ "collection": {
+ "id": "coll",
+ "database": "db",
+ "collectionName": "coll"
+ }
+ },
+ {
+ "clientEncryption": {
+ "id": "clientEncryption",
+ "clientEncryptionOpts": {
+ "keyVaultClient": "client",
+ "keyVaultNamespace": "keyvault.datakeys",
+ "kmsProviders": {
+ "azure": {
+ "accessToken": {
+ "$$placeholder": 1
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "initialData": [
+ {
+ "databaseName": "db",
+ "collectionName": "coll",
+ "documents": [],
+ "createOptions": {
+ "validator": {
+ "$jsonSchema": {
+ "properties": {
+ "secret": {
+ "encrypt": {
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "AZURE+AAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
+ }
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ }
+ },
+ "bsonType": "object"
+ }
+ }
+ }
+ },
+ {
+ "databaseName": "keyvault",
+ "collectionName": "datakeys",
+ "documents": [
+ {
+ "_id": {
+ "$binary": {
+ "base64": "AZURE+AAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
+ },
+ "keyAltNames": [
+ "my-key"
+ ],
+ "keyMaterial": {
+ "$binary": {
+ "base64": "n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==",
+ "subType": "00"
+ }
+ },
+ "creationDate": {
+ "$date": {
+ "$numberLong": "1552949630483"
+ }
+ },
+ "updateDate": {
+ "$date": {
+ "$numberLong": "1552949630483"
+ }
+ },
+ "status": {
+ "$numberInt": "0"
+ },
+ "masterKey": {
+ "provider": "azure",
+ "keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
+ "keyName": "key-name-csfle"
+ }
+ }
+ ]
+ }
+ ],
+ "tests": [
+ {
+ "description": "Auto encrypt using access token Azure credentials",
+ "operations": [
+ {
+ "name": "insertOne",
+ "arguments": {
+ "document": {
+ "_id": 1,
+ "secret": "string0"
+ }
+ },
+ "object": "coll"
+ }
+ ],
+ "outcome": [
+ {
+ "documents": [
+ {
+ "_id": 1,
+ "secret": {
+ "$binary": {
+ "base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
+ "subType": "06"
+ }
+ }
+ }
+ ],
+ "collectionName": "coll",
+ "databaseName": "db"
+ }
+ ]
+ },
+ {
+ "description": "Explicit encrypt using access token Azure credentials",
+ "operations": [
+ {
+ "name": "encrypt",
+ "object": "clientEncryption",
+ "arguments": {
+ "value": "string0",
+ "opts": {
+ "keyAltName": "my-key",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ },
+ "expectResult": {
+ "$binary": {
+ "base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
+ "subType": "06"
+ }
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/test/client-side-encryption/spec/unified/accessToken-gcp.json b/test/client-side-encryption/spec/unified/accessToken-gcp.json
new file mode 100644
index 00000000..f5cf8914
--- /dev/null
+++ b/test/client-side-encryption/spec/unified/accessToken-gcp.json
@@ -0,0 +1,188 @@
+{
+ "description": "accessToken-gcp",
+ "schemaVersion": "1.28",
+ "runOnRequirements": [
+ {
+ "minServerVersion": "4.1.10",
+ "csfle": {
+ "minLibmongocryptVersion": "1.6.0"
+ }
+ }
+ ],
+ "createEntities": [
+ {
+ "client": {
+ "id": "client",
+ "autoEncryptOpts": {
+ "keyVaultNamespace": "keyvault.datakeys",
+ "kmsProviders": {
+ "gcp": {
+ "accessToken": {
+ "$$placeholder": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ {
+ "database": {
+ "id": "db",
+ "client": "client",
+ "databaseName": "db"
+ }
+ },
+ {
+ "collection": {
+ "id": "coll",
+ "database": "db",
+ "collectionName": "coll"
+ }
+ },
+ {
+ "clientEncryption": {
+ "id": "clientEncryption",
+ "clientEncryptionOpts": {
+ "keyVaultClient": "client",
+ "keyVaultNamespace": "keyvault.datakeys",
+ "kmsProviders": {
+ "gcp": {
+ "accessToken": {
+ "$$placeholder": 1
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "initialData": [
+ {
+ "databaseName": "db",
+ "collectionName": "coll",
+ "documents": [],
+ "createOptions": {
+ "validator": {
+ "$jsonSchema": {
+ "properties": {
+ "secret": {
+ "encrypt": {
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "GCP+AAAAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
+ }
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ }
+ },
+ "bsonType": "object"
+ }
+ }
+ }
+ },
+ {
+ "databaseName": "keyvault",
+ "collectionName": "datakeys",
+ "documents": [
+ {
+ "_id": {
+ "$binary": {
+ "base64": "GCP+AAAAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
+ },
+ "keyAltNames": [
+ "my-key"
+ ],
+ "keyMaterial": {
+ "$binary": {
+ "base64": "CiQAIgLj0WyktnB4dfYHo5SLZ41K4ASQrjJUaSzl5vvVH0G12G0SiQEAjlV8XPlbnHDEDFbdTO4QIe8ER2/172U1ouLazG0ysDtFFIlSvWX5ZnZUrRMmp/R2aJkzLXEt/zf8Mn4Lfm+itnjgo5R9K4pmPNvvPKNZX5C16lrPT+aA+rd+zXFSmlMg3i5jnxvTdLHhg3G7Q/Uv1ZIJskKt95bzLoe0tUVzRWMYXLIEcohnQg==",
+ "subType": "00"
+ }
+ },
+ "creationDate": {
+ "$date": {
+ "$numberLong": "1552949630483"
+ }
+ },
+ "updateDate": {
+ "$date": {
+ "$numberLong": "1552949630483"
+ }
+ },
+ "status": {
+ "$numberInt": "0"
+ },
+ "masterKey": {
+ "provider": "gcp",
+ "projectId": "devprod-drivers",
+ "location": "global",
+ "keyRing": "key-ring-csfle",
+ "keyName": "key-name-csfle"
+ }
+ }
+ ]
+ }
+ ],
+ "tests": [
+ {
+ "description": "Auto encrypt using access token GCP credentials",
+ "operations": [
+ {
+ "name": "insertOne",
+ "arguments": {
+ "document": {
+ "_id": 1,
+ "secret": "string0"
+ }
+ },
+ "object": "coll"
+ }
+ ],
+ "outcome": [
+ {
+ "documents": [
+ {
+ "_id": 1,
+ "secret": {
+ "$binary": {
+ "base64": "ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==",
+ "subType": "06"
+ }
+ }
+ }
+ ],
+ "collectionName": "coll",
+ "databaseName": "db"
+ }
+ ]
+ },
+ {
+ "description": "Explicit encrypt using access token GCP credentials",
+ "operations": [
+ {
+ "name": "encrypt",
+ "object": "clientEncryption",
+ "arguments": {
+ "value": "string0",
+ "opts": {
+ "keyAltName": "my-key",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ },
+ "expectResult": {
+ "$binary": {
+ "base64": "ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==",
+ "subType": "06"
+ }
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-accessToken-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-accessToken-type.json
new file mode 100644
index 00000000..8fe5c150
--- /dev/null
+++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-azure-accessToken-type.json
@@ -0,0 +1,31 @@
+{
+ "description": "clientEncryptionOpts-kmsProviders-azure-accessToken-type",
+ "schemaVersion": "1.28",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ },
+ {
+ "clientEncryption": {
+ "id": "clientEncryption0",
+ "clientEncryptionOpts": {
+ "keyVaultClient": "client0",
+ "keyVaultNamespace": "keyvault.datakeys",
+ "kmsProviders": {
+ "azure": {
+ "accessToken": 0
+ }
+ }
+ }
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "",
+ "operations": []
+ }
+ ]
+}
diff --git a/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-accessToken-type.json b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-accessToken-type.json
new file mode 100644
index 00000000..2284e26c
--- /dev/null
+++ b/test/unified-test-format/invalid/clientEncryptionOpts-kmsProviders-gcp-accessToken-type.json
@@ -0,0 +1,31 @@
+{
+ "description": "clientEncryptionOpts-kmsProviders-gcp-accessToken-type",
+ "schemaVersion": "1.28",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ },
+ {
+ "clientEncryption": {
+ "id": "clientEncryption0",
+ "clientEncryptionOpts": {
+ "keyVaultClient": "client0",
+ "keyVaultNamespace": "keyvault.datakeys",
+ "kmsProviders": {
+ "gcp": {
+ "accessToken": 0
+ }
+ }
+ }
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "",
+ "operations": []
+ }
+ ]
+}

View File

@ -1,20 +0,0 @@
#!/bin/bash
# Synchronize local files to a remote Evergreen spawn host.
set -eu
if [ -z "$1" ]
then
echo "Must supply a spawn host URL!"
fi
target=$1
user=${target%@*}
remote_dir=/home/$user/mongo-python-driver
echo "Copying files to $target..."
rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_dir
echo "Copying files to $target... done."
echo "Syncing files to $target..."
# shellcheck disable=SC2034
fswatch -o . | while read f; do rsync -hazv -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:/home/$user/mongo-python-driver; done
echo "Syncing files to $target... done."

24
.evergreen/tox.sh Normal file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -o errexit # Exit the script with error if any of the commands fail
set -x
. .evergreen/utils.sh
if [ -z "$PYTHON_BINARY" ]; then
PYTHON_BINARY=$(find_python3)
fi
if $PYTHON_BINARY -m tox --version; then
run_tox() {
$PYTHON_BINARY -m tox "$@"
}
else # No toolchain present, set up virtualenv before installing tox
createvirtualenv "$PYTHON_BINARY" toxenv
trap "deactivate; rm -rf toxenv" EXIT HUP
python -m pip install -q tox
run_tox() {
python -m tox "$@"
}
fi
run_tox "${@:1}"

112
.evergreen/utils.sh Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash -ex
set -o xtrace
find_python3() {
PYTHON=""
# Add a fallback system python3 if it is available and Python 3.8+.
if is_python_38 "$(command -v python3)"; then
PYTHON="$(command -v python3)"
fi
# Find a suitable toolchain version, if available.
if [ "$(uname -s)" = "Darwin" ]; then
# macos 11.00
if [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then
PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3"
# macos 10.14
elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.8" ]; then
PYTHON="/Library/Frameworks/Python.Framework/Versions/3.8/bin/python3"
fi
elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
PYTHON="C:/python/Python38/python.exe"
else
# Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.8+.
if [ -f "/opt/python/3.8/bin/python3" ]; then
PYTHON="/opt/python/3.8/bin/python3"
elif is_python_38 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then
PYTHON="/opt/mongodbtoolchain/v4/bin/python3"
elif is_python_38 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then
PYTHON="/opt/mongodbtoolchain/v3/bin/python3"
fi
fi
if [ -z "$PYTHON" ]; then
echo "Cannot test without python3.8+ installed!"
exit 1
fi
echo "$PYTHON"
}
# Usage:
# createvirtualenv /path/to/python /output/path/for/venv
# * param1: Python binary to use for the virtualenv
# * param2: Path to the virtualenv to create
createvirtualenv () {
PYTHON=$1
VENVPATH=$2
# Prefer venv
VENV="$PYTHON -m venv"
if [ "$(uname -s)" = "Darwin" ]; then
VIRTUALENV="$PYTHON -m virtualenv"
else
VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv")
VIRTUALENV="$VIRTUALENV -p $PYTHON"
fi
if ! $VENV $VENVPATH 2>/dev/null; then
# Workaround for bug in older versions of virtualenv.
$VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH
fi
if [ "Windows_NT" = "$OS" ]; then
# Workaround https://bugs.python.org/issue32451:
# mongovenv/Scripts/activate: line 3: $'\r': command not found
dos2unix $VENVPATH/Scripts/activate || true
. $VENVPATH/Scripts/activate
else
. $VENVPATH/bin/activate
fi
export PIP_QUIET=1
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools tox
}
# Usage:
# testinstall /path/to/python /path/to/.whl ["no-virtualenv"]
# * param1: Python binary to test
# * param2: Path to the wheel to install
# * param3 (optional): If set to a non-empty string, don't create a virtualenv. Used in manylinux containers.
testinstall () {
PYTHON=$1
RELEASE=$2
NO_VIRTUALENV=$3
if [ -z "$NO_VIRTUALENV" ]; then
createvirtualenv $PYTHON venvtestinstall
PYTHON=python
fi
$PYTHON -m pip install --upgrade $RELEASE
cd tools
$PYTHON fail_if_no_c.py
$PYTHON -m pip uninstall -y pymongo
cd ..
if [ -z "$NO_VIRTUALENV" ]; then
deactivate
rm -rf venvtestinstall
fi
}
# Function that returns success if the provided Python binary is version 3.8 or later
# Usage:
# is_python_38 /path/to/python
# * param1: Python binary
is_python_38() {
if [ -z "$1" ]; then
return 1
elif $1 -c "import sys; exit(sys.version_info[:2] < (3, 8))"; then
# runs when sys.version_info[:2] >= (3, 8)
return 0
else
return 1
fi
}

1
.github/CODEOWNERS vendored
View File

@ -1 +0,0 @@
* @mongodb/dbx-python

View File

@ -1,44 +0,0 @@
When reviewing code, focus on:
## Security Critical Issues
- Check for hardcoded secrets, API keys, or credentials.
- Check for instances of potential method call injection, dynamic code execution, symbol injection or other code injection vulnerabilities.
## Performance Red Flags
- Spot inefficient loops and algorithmic issues.
- Check for memory leaks and resource cleanup.
## Code Quality Essentials
- Methods should be focused and appropriately sized. If a method is doing too much, suggest refactorings to split it up.
- Use clear, descriptive naming conventions.
- Avoid encapsulation violations and ensure proper separation of concerns.
- All public classes, modules, and methods should have clear documentation in Sphinx format.
## PyMongo-specific Concerns
- Do not review files within `pymongo/synchronous` or files in `test/` that also have a file of the same name in `test/asynchronous` unless the reviewed changes include a `_IS_SYNC` statement. PyMongo generates these files from `pymongo/asynchronous` and `test/asynchronous` using `tools/synchro.py`.
- All asynchronous functions must not call any blocking I/O.
## Review Style
- Be specific and actionable in feedback.
- Explain the "why" behind recommendations.
- Acknowledge good patterns when you see them.
- Ask clarifying questions when code intent is unclear.
Always prioritize security vulnerabilities and performance issues that could impact users.
Always suggest changes to improve readability and testability. For example, this suggestion seeks to make the code more readable, reusable, and testable:
```python
# Instead of:
if user.email and "@" in user.email and len(user.email) > 5:
submit_button.enabled = True
else:
submit_button.enabled = False
# Consider:
def valid_email(email):
return email and "@" in email and len(email) > 5
submit_button.enabled = valid_email(user.email)
```

View File

@ -5,8 +5,6 @@ updates:
directory: "/"
schedule:
interval: "weekly"
cooldown:
default-days: 7
groups:
actions:
patterns:

View File

@ -1,33 +0,0 @@
<!-- Thanks for contributing! -->
<!-- Please ensure that the title of the PR is in the following form:
[JIRA TICKET]: Issue Title
If you are an external contributor and there is no JIRA ticket associated with your change, then use your best judgement
for the PR title. A MongoDB employee will create a JIRA ticket and edit the name and links as appropriate.
Note on AI Contributions:
We only accept pull requests that are authored and submitted by human contributors who fully understand the changes they are proposing.
All contributions must be written and understood by human contributors. Please read about our policy in our contributing guide.
-->
[JIRA TICKET]
## Changes in this PR
<!-- What changes did you make to the code? What new APIs (public or private) were added, removed, or edited to generate
the desired outcome explained in the above summary? -->
## Test Plan
<!-- How did you test the code? If you added unit tests, you can say that. If you didnt introduce unit tests, explain why.
All code should be tested in some way so please list what your validation strategy was. -->
## Checklist
<!-- Do not delete the items provided on this checklist. -->
### Checklist for Author
- [ ] Did you update the changelog (if necessary)?
- [ ] Is there test coverage?
- [ ] Is any followup work tracked in a JIRA ticket? If so, add link(s).
### Checklist for Reviewer
- [ ] Does the title of the PR reference a JIRA Ticket?
- [ ] Do you fully understand the implementation? (Would you be comfortable explaining how this code works to someone else?)
- [ ] Is all relevant documentation (README or docstring) updated?

View File

@ -1,68 +0,0 @@
name: "CodeQL"
on:
push:
branches: [ "master", "v*"]
tags: ['*']
pull_request:
workflow_call:
inputs:
ref:
required: true
type: string
schedule:
- cron: '17 10 * * 2'
concurrency:
group: codeql-${{ github.ref }}
cancel-in-progress: true
jobs:
analyze:
name: Analyze (${{ matrix.language }})
runs-on: "ubuntu-latest"
timeout-minutes: 360
permissions:
# required for all workflows
security-events: write
strategy:
fail-fast: false
matrix:
include:
- language: c-cpp
build-mode: manual
- language: python
build-mode: none
- language: actions
build-mode: none
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- uses: actions/setup-python@v6
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
queries: security-extended
config: |
paths-ignore:
- 'doc/**'
- 'tools/**'
- 'test/**'
- if: matrix.build-mode == 'manual'
run: |
pip install -e .
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4
with:
category: "/language:${{matrix.language}}"

View File

@ -1,57 +0,0 @@
name: Create Release Branch
on:
workflow_dispatch:
inputs:
branch_name:
description: The name of the new branch
required: true
version:
description: The version to set on the branch
required: true
base_ref:
description: The base reference for the branch
push_changes:
description: Whether to push the changes
default: "true"
concurrency:
group: create-branch-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash -eux {0}
jobs:
create-branch:
environment: release
runs-on: ubuntu-latest
permissions:
id-token: write
contents: write
outputs:
version: ${{ steps.pre-publish.outputs.version }}
steps:
- uses: mongodb-labs/drivers-github-tools/secure-checkout@v3
with:
app_id: ${{ vars.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: mongodb-labs/drivers-github-tools/setup@v3
with:
aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
aws_region_name: ${{ vars.AWS_REGION_NAME }}
aws_secret_id: ${{ secrets.AWS_SECRET_ID }}
artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }}
- name: Get hatch
run: pip install hatch
- uses: mongodb-labs/drivers-github-tools/create-branch@v3
id: create-branch
with:
branch_name: ${{ inputs.branch_name }}
version: ${{ inputs.version }}
base_ref: ${{ inputs.base_ref }}
push_changes: ${{ inputs.push_changes }}
version_bump_script: hatch version
evergreen_project: mongo-python-driver-release
release_workflow_path: ./.github/workflows/release-python.yml

View File

@ -1,148 +0,0 @@
name: Python Dist
on:
push:
tags:
- "[0-9]+.[0-9]+.[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+.post[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+"
- "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
workflow_dispatch:
pull_request:
workflow_call:
inputs:
ref:
required: true
type: string
concurrency:
group: dist-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash -eux {0}
jobs:
build_wheels:
name: Build wheels for ${{ matrix.buildplat[1] }}
runs-on: ${{ matrix.buildplat[0] }}
strategy:
# Ensure that a wheel builder finishes even if another fails
fail-fast: false
matrix:
# Github Actions doesn't support pairing matrix values together, let's improvise
# https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
buildplat:
- [ubuntu-latest, "manylinux_x86_64", "cp3*-manylinux_x86_64"]
- [ubuntu-latest, "manylinux_aarch64", "cp3*-manylinux_aarch64"]
- [ubuntu-latest, "manylinux_ppc64le", "cp3*-manylinux_ppc64le"]
- [ubuntu-latest, "manylinux_s390x", "cp3*-manylinux_s390x"]
- [ubuntu-latest, "manylinux_i686", "cp3*-manylinux_i686"]
- [windows-2022, "win_amd6", "cp3*-win_amd64"]
- [windows-2022, "win32", "cp3*-win32"]
- [windows-11-arm, "win_arm64", "cp3*-win_arm64"]
- [macos-14, "macos", "cp*-macosx_*"]
steps:
- name: Checkout pymongo
uses: actions/checkout@v6
with:
fetch-depth: 0
persist-credentials: false
ref: ${{ inputs.ref }}
- uses: actions/setup-python@v6
with:
cache: 'pip'
python-version: 3.11
cache-dependency-path: 'pyproject.toml'
allow-prereleases: true
- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
with:
# setup-qemu-action by default uses `tonistiigi/binfmt:latest` image,
# which is out of date. This causes seg faults during build.
# Here we manually fix the version.
image: tonistiigi/binfmt:qemu-v8.1.5
platforms: all
- name: Install cibuildwheel
# Note: the default manylinux is manylinux_2_28
run: |
python -m pip install -U pip
python -m pip install "cibuildwheel>=3.2.0,<4"
- name: Build wheels
env:
CIBW_BUILD: ${{ matrix.buildplat[2] }}
run: python -m cibuildwheel --output-dir wheelhouse
- name: Assert all versions in wheelhouse
if: ${{ ! startsWith(matrix.buildplat[1], 'macos') }}
run: |
ls wheelhouse/*cp39*.whl
ls wheelhouse/*cp310*.whl
ls wheelhouse/*cp311*.whl
ls wheelhouse/*cp312*.whl
ls wheelhouse/*cp313*.whl
ls wheelhouse/*cp314*.whl
# Free-threading builds:
ls wheelhouse/*cp314t*.whl
- uses: actions/upload-artifact@v7
with:
name: wheel-${{ matrix.buildplat[1] }}
path: ./wheelhouse/*.whl
if-no-files-found: error
make_sdist:
name: Make SDist
runs-on: macos-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
persist-credentials: false
ref: ${{ inputs.ref }}
- uses: actions/setup-python@v6
with:
# Build sdist on lowest supported Python
python-version: "3.9"
- name: Build SDist
run: |
set -ex
python -m pip install -U pip build
python -m build --sdist .
- name: Test SDist
run: |
python -m pip install dist/*.gz
cd ..
python -c "from pymongo import has_c; assert has_c()"
- uses: actions/upload-artifact@v7
with:
name: "sdist"
path: ./dist/*.tar.gz
collect_dist:
runs-on: ubuntu-latest
needs: [build_wheels, make_sdist]
name: Download Wheels
steps:
- name: Download all workflow run artifacts
uses: actions/download-artifact@v8
- name: Flatten directory
working-directory: .
run: |
find . -mindepth 2 -type f -exec mv {} . \;
find . -type d -empty -delete
- uses: actions/upload-artifact@v7
with:
name: all-dist-${{ github.run_id }}
path: "./*"

View File

@ -0,0 +1,23 @@
# [JIRA Ticket ID](Link to Ticket)
<!-- Please provide explicit URL link to the corresponding JIRA ticket. -->
# Summary
<!-- Please provide a high level overview of what changes have been made. -->
# Changes in this PR
<!-- Highlight any high level architecture changes if the summary doesn't already cover the scope. -->
# Test Plan
<!-- Talk through any unit tests added, and if this is a bug fix, please add repro steps in the event the fix needs to be verified. -->
# Screenshots (Optional)
<!-- Add a before and after picture to indicate changes. -->
# Callouts or Follow-up items (Optional)
<!-- Any additional info not already specified in the PR including but not limited to:
1. Potential stakeholders
2. Slack threads etc.
3. Implementation details that need additional oversight
4. Callouts on future tactics
-->

View File

@ -1,117 +1,42 @@
name: Release
name: Python Wheels
on:
push:
tags:
- "[0-9]+.[0-9]+.[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+.post[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+"
- "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
workflow_dispatch:
inputs:
following_version:
description: "The post (dev) version to set"
dry_run:
description: "Dry Run?"
default: false
type: boolean
schedule:
- cron: '30 5 * * *'
pull_request:
env:
# Changes per repo
PRODUCT_NAME: PyMongo
# Changes per branch
EVERGREEN_PROJECT: mongo-python-driver
# Constant
# inputs will be empty on a scheduled run. so, we only set dry_run
# to 'false' when the input is set to 'false'.
DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }}
FOLLOWING_VERSION: ${{ inputs.following_version || '' }}
concurrency:
group: wheels-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash -eux {0}
jobs:
pre-publish:
environment: release
generate_sarif_report:
runs-on: ubuntu-latest
if: github.repository_owner == 'mongodb' || github.event_name == 'workflow_dispatch'
permissions:
# required for all workflows
security-events: write
id-token: write
contents: write
outputs:
version: ${{ steps.pre-publish.outputs.version }}
steps:
- uses: mongodb-labs/drivers-github-tools/secure-checkout@v3
- uses: actions/checkout@v4
- name: "Export SARIF file from code scanning alerts"
uses: "alcaeus/drivers-github-tools/code-scanning-export@export-code-scanning-report"
with:
app_id: ${{ vars.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: mongodb-labs/drivers-github-tools/setup@v3
with:
aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
aws_region_name: ${{ vars.AWS_REGION_NAME }}
aws_secret_id: ${{ secrets.AWS_SECRET_ID }}
- uses: mongodb-labs/drivers-github-tools/python/pre-publish@v3
id: pre-publish
with:
dry_run: ${{ env.DRY_RUN }}
ref: master
build-dist:
needs: [pre-publish]
uses: ./.github/workflows/dist.yml
with:
ref: ${{ needs.pre-publish.outputs.version }}
static-scan:
needs: [pre-publish]
uses: ./.github/workflows/codeql.yml
permissions:
security-events: write
with:
ref: ${{ needs.pre-publish.outputs.version }}
publish:
needs: [build-dist, static-scan]
name: Upload release to PyPI
runs-on: ubuntu-latest
environment: release
permissions:
id-token: write
steps:
- name: Download all the dists
uses: actions/download-artifact@v8
- name: "Attach generated SARIF file to build artifacts"
uses: actions/upload-artifact@v4
with:
name: all-dist-${{ github.run_id }}
path: dist/
- name: Publish package distributions to TestPyPI
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
with:
repository-url: https://test.pypi.org/legacy/
skip-existing: true
attestations: ${{ env.DRY_RUN }}
- name: Publish package distributions to PyPI
if: startsWith(env.DRY_RUN, 'false')
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
post-publish:
needs: [publish]
runs-on: ubuntu-latest
environment: release
permissions:
id-token: write
contents: write
attestations: write
security-events: write
steps:
- uses: mongodb-labs/drivers-github-tools/secure-checkout@v3
with:
app_id: ${{ vars.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: mongodb-labs/drivers-github-tools/setup@v3
with:
aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
aws_region_name: ${{ vars.AWS_REGION_NAME }}
aws_secret_id: ${{ secrets.AWS_SECRET_ID }}
- uses: mongodb-labs/drivers-github-tools/python/post-publish@v3
with:
following_version: ${{ env.FOLLOWING_VERSION }}
product_name: ${{ env.PRODUCT_NAME }}
evergreen_project: ${{ env.EVERGREEN_PROJECT }}
token: ${{ github.token }}
dry_run: ${{ env.DRY_RUN }}
name: code-scanning-alerts.json
path: code-scanning-alerts.json
retention-days: 3

View File

@ -1,104 +0,0 @@
name: Generate SBOM
# This workflow uses cyclonedx-py and publishes an sbom.json artifact.
# It runs on manual trigger or when package files change on main branch,
# and creates a PR with the updated SBOM.
# Internal documentation: go/sbom-scope
on:
workflow_dispatch: {}
push:
branches: ['master']
paths:
- 'requirements.txt'
- 'requirements/**.txt'
- '!requirements/docs.txt'
- '!requirements/test.txt'
permissions:
contents: write
pull-requests: write
jobs:
sbom:
name: Generate SBOM and Create PR
runs-on: ubuntu-latest
concurrency:
group: sbom-${{ github.ref }}
cancel-in-progress: false
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Generate SBOM
run: |
python -m venv .venv
source .venv/bin/activate
python tools/generate_sbom_requirements.py
pip install -r sbom-requirements.txt
pip install .
pip uninstall -y pip setuptools
deactivate
python -m venv .venv-sbom
source .venv-sbom/bin/activate
pip install cyclonedx-bom==7.2.1
cyclonedx-py environment --spec-version 1.5 --output-format JSON --output-file sbom.json .venv
# Add PURL for pymongo (local package doesn't get PURL automatically)
jq '(.components[] | select(.name == "pymongo" and .purl == null)) |= (. + {purl: ("pkg:pypi/pymongo@" + .version)})' sbom.json > sbom.tmp.json && mv sbom.tmp.json sbom.json
- name: Download CycloneDX CLI
run: |
curl -L -s -o /tmp/cyclonedx "https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.29.1/cyclonedx-linux-x64"
chmod +x /tmp/cyclonedx
- name: Validate SBOM
run: /tmp/cyclonedx validate --input-file sbom.json --fail-on-errors
- name: Cleanup
if: always()
run: rm -rf .venv .venv-sbom sbom-requirements.txt
- name: Upload SBOM artifact
uses: actions/upload-artifact@v7
with:
name: sbom
path: sbom.json
if-no-files-found: error
- name: Create Pull Request
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: 'chore: Update SBOM after dependency changes'
branch: auto-update-sbom-${{ github.run_id }}
delete-branch: true
title: 'Automation: Update SBOM'
body: |
## Automated SBOM Update
This PR was automatically generated because dependency manifest files changed.
### Changes
- Updated `sbom.json` to reflect current dependencies
### Verification
The SBOM was generated using cyclonedx-py v7.2.1 with the current Python environment.
### Triggered by
- Commit: ${{ github.sha }}
- Workflow run: ${{ github.run_id }}
---
_This PR was created automatically by the [SBOM workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})_
labels: |
sbom
automated
dependencies

View File

@ -14,38 +14,33 @@ defaults:
run:
shell: bash -eux {0}
permissions:
contents: read
jobs:
static:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "3.10"
- name: Install just
run: uv tool install rust-just
python-version: "3.8"
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install Python dependencies
run: |
just install
python -m pip install -U pip tox
- name: Run linters
run: |
just lint-manual
tox -m lint-manual
- name: Check Manifest
run: |
tox -m manifest
- name: Run compilation
run: |
export PYMONGO_C_EXT_MUST_BUILD=1
pip install -v -e .
pip install -e .
python tools/fail_if_no_c.py
- name: Run typecheck
run: |
just typing
tox -m typecheck
- run: |
sudo apt-get install -y cppcheck
- run: |
@ -53,199 +48,127 @@ jobs:
cppcheck pymongo
build:
# supercharge/mongodb-github-action requires containers so we don't test other platforms
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
# Tests currently only pass on ubuntu on GitHub Actions.
os: [ubuntu-latest]
python-version: ["3.10", "pypy-3.11", "3.13t"]
mongodb-version: ["8.0"]
os: [ubuntu-20.04]
python-version: ["3.8", "3.11", "pypy-3.9"]
name: CPython ${{ matrix.python-version }}-${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
- id: setup-mongodb
uses: mongodb-labs/drivers-evergreen-tools@master
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install dependencies
run: |
pip install -q tox
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.10.0
with:
version: "${{ matrix.mongodb-version }}"
mongodb-version: 4.4
- name: Run tests
run: uv run --extra test pytest -v
run: |
tox -m test
coverage:
# This enables a coverage report for a given PR, which will be augmented by
# the combined codecov report uploaded in Evergreen.
runs-on: ubuntu-latest
name: Coverage
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "3.10"
- id: setup-mongodb
uses: mongodb-labs/drivers-evergreen-tools@master
with:
version: "8.0"
- name: Install just
run: uv tool install rust-just
- name: Setup tests
run: COVERAGE=1 just setup-tests
- name: Run tests
run: just run-tests
- name: Generate xml report
run: uv tool run --with "coverage[toml]" coverage xml
- name: Upload test results to Codecov
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
doctest:
runs-on: ubuntu-latest
name: DocTest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "3.10"
- name: Install just
run: uv tool install rust-just
- id: setup-mongodb
uses: mongodb-labs/drivers-evergreen-tools@master
with:
version: "8.0"
python-version: "3.8"
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install dependencies
run: just install
run: |
pip install -q tox
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.10.0
with:
mongodb-version: 4.4
- name: Run tests
run: |
just setup-tests doctest
just run-tests
run: |
tox -m doc-test
docs:
name: Docs Checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "3.10"
- name: Install just
run: uv tool install rust-just
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
# Build docs on lowest supported Python for furo
python-version: '3.8'
- name: Install dependencies
run: just install
run: |
pip install -q tox
- name: Build docs
run: just docs
run: |
tox -m doc
linkcheck:
name: Link Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "3.10"
- name: Install just
run: uv tool install rust-just
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
# Build docs on lowest supported Python for furo
python-version: '3.8'
- name: Install dependencies
run: just install
run: |
pip install -q tox
- name: Build docs
run: just docs-linkcheck
run: |
tox -m linkcheck
typing:
name: Typing Tests
runs-on: ubuntu-latest
strategy:
matrix:
python: ["3.10", "3.11"]
python: ["3.8", "3.11"]
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "${{matrix.python}}"
- name: Install just
run: uv tool install rust-just
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install dependencies
run: |
just install
pip install -q tox
- name: Run typecheck
run: |
just typing
integration_tests:
runs-on: ubuntu-latest
name: Integration Tests
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
enable-cache: true
python-version: "3.10"
- name: Install just
run: uv tool install rust-just
- name: Install dependencies
run: |
just install
- id: setup-mongodb
uses: mongodb-labs/drivers-evergreen-tools@master
- name: Run tests
run: |
just integration-tests
- id: setup-mongodb-ssl
uses: mongodb-labs/drivers-evergreen-tools@master
with:
ssl: true
- name: Run tests
run: |
just integration-tests
tox -m typecheck
make_sdist:
runs-on: ubuntu-latest
name: "Make an sdist"
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: actions/setup-python@v6
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
# Build sdist on lowest supported Python
python-version: "3.9"
python-version: '3.8'
- name: Build SDist
shell: bash
run: |
pip install build
python -m build --sdist
- uses: actions/upload-artifact@v7
- uses: actions/upload-artifact@v4
with:
name: "sdist"
path: dist/*.tar.gz
@ -257,9 +180,7 @@ jobs:
timeout-minutes: 20
steps:
- name: Download sdist
uses: actions/download-artifact@v8
with:
path: sdist/
uses: actions/download-artifact@v4
- name: Unpack SDist
shell: bash
run: |
@ -268,44 +189,19 @@ jobs:
mkdir test
tar --strip-components=1 -zxf *.tar.gz -C ./test
ls test
- uses: actions/setup-python@v6
- uses: actions/setup-python@v5
with:
cache: 'pip'
cache-dependency-path: 'sdist/test/pyproject.toml'
# Test sdist on lowest supported Python
python-version: "3.9"
- id: setup-mongodb
uses: mongodb-labs/drivers-evergreen-tools@master
- name: Run connect test from sdist
python-version: '3.8'
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.10.0
- name: Run Test
shell: bash
run: |
cd sdist/test
ls
which python
pip install -e ".[test]"
PYMONGO_MUST_CONNECT=1 pytest -v -k client_context
test_minimum:
permissions:
contents: read
runs-on: ubuntu-latest
name: Test minimum dependencies and Python
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
python-version: "3.9"
- id: setup-mongodb
uses: mongodb-labs/drivers-evergreen-tools@master
with:
version: "8.0"
- name: Run tests
shell: bash
run: |
uv venv
source .venv/bin/activate
uv pip install -e ".[test]" --resolution=lowest-direct --force-reinstall
pytest -v test/test_srv_polling.py test/test_dns.py test/asynchronous/test_srv_polling.py test/asynchronous/test_dns.py
pytest -v

View File

@ -1,21 +0,0 @@
name: GitHub Actions Security Analysis with zizmor 🌈
on:
push:
branches: ["master"]
pull_request:
branches: ["**"]
jobs:
zizmor:
name: zizmor latest via Cargo
runs-on: ubuntu-latest
permissions:
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Run zizmor 🌈
uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2

7
.github/zizmor.yml vendored
View File

@ -1,7 +0,0 @@
rules:
unpinned-uses:
config:
policies:
actions/*: ref-pin
mongodb-labs/drivers-github-tools/*: ref-pin
mongodb-labs/drivers-evergreen-tools: ref-pin

14
.gitignore vendored
View File

@ -18,29 +18,19 @@ mongocryptd.pid
.idea/
.vscode/
.nova/
.temp/
venv/
secrets-export.sh
libmongocrypt.tar.gz
libmongocrypt/
.venv
expansion.yml
*expansions.yml
.evergreen/scripts/env.sh
.evergreen/scripts/test-env.sh
specifications/
results.json
.evergreen/atlas_x509_dev_client_certificate.pem
libmongocrypt_git/
# Lambda temp files
test/lambda/.aws-sam
test/lambda/env.json
test/lambda/mongodb/pymongo/*
test/lambda/mongodb/gridfs/*
test/lambda/mongodb/bson/*
test/lambda/*.json
# test results and logs
xunit-results/
coverage.xml
server.log
.coverage

View File

@ -6,7 +6,6 @@ repos:
- id: check-added-large-files
- id: check-case-conflict
- id: check-toml
- id: check-json
- id: check-yaml
exclude: template.yaml
- id: debug-statements
@ -26,18 +25,6 @@ repos:
args: ["--fix", "--show-fixes"]
- id: ruff-format
- repo: local
hooks:
- id: synchro
name: synchro
entry: bash ./tools/synchro.sh
language: python
require_serial: true
fail_fast: true
additional_dependencies:
- ruff==0.1.3
- unasync
- repo: https://github.com/adamchainz/blacken-docs
rev: "1.16.0"
hooks:
@ -77,14 +64,17 @@ repos:
stages: [manual]
- repo: https://github.com/sirosen/check-jsonschema
rev: 0.29.0
rev: 0.27.0
hooks:
- id: check-github-workflows
- id: check-github-actions
- id: check-dependabot
- id: check-jsonschema
name: "Check GitHub Workflows"
files: ^\.github/workflows/
types: [yaml]
args: ["--schemafile", "https://json.schemastore.org/github-workflow"]
stages: [manual]
- repo: https://github.com/ariebovenberg/slotscheck
rev: v0.19.0
rev: v0.17.0
hooks:
- id: slotscheck
files: \.py$
@ -104,32 +94,3 @@ repos:
# - test/versioned-api/crud-api-version-1-strict.json:514: nin ==> inn, min, bin, nine
# - test/test_client.py:188: te ==> the, be, we, to
args: ["-L", "fle,fo,infinit,isnt,nin,te,aks"]
- repo: local
hooks:
- id: executable-shell
name: executable-shell
entry: chmod +x
language: system
types: [shell]
exclude: |
(?x)(
.evergreen/retry-with-backoff.sh
)
- id: generate-config
name: generate-config
entry: .evergreen/scripts/generate-config.sh
language: python
require_serial: true
additional_dependencies: ["shrub.py>=3.10.0", "pyyaml>=6.0.2"]
- id: uv-lock
name: uv-lock
entry: uv lock
language: python
require_serial: true
files: ^(uv\.lock|pyproject\.toml|requirements.txt|requirements/.*\.txt)$
pass_filenames: false
fail_fast: true
additional_dependencies:
- "uv>=0.8.4"

View File

@ -16,7 +16,7 @@ be of interest or that has already been addressed.
## Supported Interpreters
PyMongo supports CPython 3.9+ and PyPy3.9+. Language features not
PyMongo supports CPython 3.8+ and PyPy3.9+. Language features not
supported by all interpreters can not be used.
## Style Guide
@ -28,10 +28,8 @@ including 4 space indents and 79 character line limits.
- Avoid backward breaking changes if at all possible.
- Write inline documentation for new classes and methods.
- We use [uv](https://docs.astral.sh/uv/) for python environment management and packaging.
- We use [just](https://just.systems/man/en/) as our task runner.
- Write tests and make sure they pass (make sure you have a mongod
running on the default port, then execute `just test` from the cmd
running on the default port, then execute `tox -e test` from the cmd
line to run the test suite).
- Add yourself to doc/contributors.rst `:)`
@ -85,53 +83,49 @@ likelihood for getting review sooner shoots up.
- `versionadded:: 3.11`
- `versionchanged:: 3.5`
### AI-Generated Contributions Policy
**Pull Request Template Breakdown**
#### Our Stance
- **Github PR Title**
We only accept pull requests that are authored and submitted by human contributors who fully understand the changes they are proposing. Pull requests that are not clearly owned and understood by a human contributor may be closed. **All contributions must be submitted, reviewed, and understood by human contributors.**
- The PR Title format should always be
`[JIRA-ID] : Jira Title or Blurb Summary`.
##### Why This Policy Exists
- **JIRA LINK**
At MongoDB, we understand the power and prevalence of AI tools in software development. With that being said, many MongoDB libraries are foundational tools used in production systems worldwide. The nature of these libraries requires:
- Convenient link to the associated JIRA ticket.
- **Deep domain expertise**: MongoDB's wire protocol, BSON specification, connection pooling, authentication mechanisms, and concurrency patterns require an understanding that AI alone cannot substantiate.
- **Summary**
- **Long-term maintainability**: Contributors need to be able to explain *why* code is written a certain way, explain design decisions, and be available to iterate on their contributions.
- Small blurb on why this is needed. The JIRA task should have
the more in-depth description, but this should still, at a
high level, give anyone looking an understanding of why the
PR has been checked in.
- **Security responsibility**: Authentication, credential handling, and TLS implementation cannot be left to probabilistic code generation.
- **Changes in this PR**
##### What This Means for Contributors
- The explicit code changes that this PR is introducing. This
should be more specific than just the task name. (Unless the
task name is very clear).
**Required:**
- **Test Plan**
- Full understanding of every line of code you submit
- Ability to explain and defend your implementation choices
- Willingness to iterate and maintain your contributions
- Everything needs a test description. Describe what you did
to validate your changes actually worked; if you did
nothing, then document you did not test it. Aim to make
these steps reproducible by other engineers, specifically
with your primary reviewer in mind.
**Encouraged:**
- **Screenshots**
- Using AI assistants as learning tools to understand concepts
- IDE autocomplete features that suggest standard patterns
- AI help for brainstorming approaches (but write the code yourself)
- Writing code using AI tools, reviewing each line and revising code as necessary.
- Any images that provide more context to the PR. Usually,
these just coincide with the test plan.
**Not allowed:**
- **Callouts or follow-up items**
- Submitting PRs generated solely by AI tools
- Copy-pasting AI-generated code without full understanding
##### Disclosure
If you used AI assistance in any way during your contribution, please disclose what the AI assistant was used for in your PR description. We would love to know what tools developers have found useful in iterating in their day to day.
##### Questions?
If you're unsure whether your contribution complies with this policy, please ask for guidance within the scope of the PR and clarify any uncertainty. We're happy to guide contributors toward successful contributions.
---
*This policy helps us maintain the reliability, security, and trustworthiness that production applications depend on. Thank you for understanding and for contributing thoughtfully to PyMongo.*
- This is a good place for identifying "to-dos" that you've
placed in the code (Must have an accompanying JIRA Ticket).
- Potential bugs that you are unsure how to test in the code.
- Opinions you want to receive about your code.
## Running Linters
@ -153,26 +147,28 @@ To run `pre-commit` manually, run:
pre-commit run --all-files
```
To run a manual hook like `ruff` manually, run:
To run a manual hook like `mypy` manually, run:
```bash
pre-commit run --all-files --hook-stage manual ruff
pre-commit run --all-files --hook-stage manual mypy
```
Typically we use `just` to run the linters, e.g.
Typically we use `tox` to run the linters, e.g.
```bash
just install # this will install a venv with pre-commit installed, and install the pre-commit hook.
just typing-mypy
just run lint-manual
tox -e typecheck-mypy
tox -e lint-manual
```
## Documentation
To contribute to the [API documentation](https://pymongo.readthedocs.io/en/stable/) just make your
changes to the inline documentation of the appropriate [source code](https://github.com/mongodb/mongo-python-driver) or
[rst file](https://github.com/mongodb/mongo-python-driver/tree/master/doc) in
a branch and submit a [pull request](https://help.github.com/articles/using-pull-requests). You
To contribute to the [API
documentation](https://pymongo.readthedocs.io/en/stable/) just make your
changes to the inline documentation of the appropriate [source
code](https://github.com/mongodb/mongo-python-driver) or [rst
file](https://github.com/mongodb/mongo-python-driver/tree/master/doc) in
a branch and submit a [pull
request](https://help.github.com/articles/using-pull-requests). You
might also use the GitHub
[Edit](https://github.com/blog/844-forking-with-the-edit-button) button.
@ -182,271 +178,49 @@ documentation including narrative docs, and the [Sphinx docstring format](https:
You can build the documentation locally by running:
```bash
just docs
tox -e doc
```
When updating docs, it can be helpful to run the live docs server as:
```bash
just docs-serve
tox -e doc-serve
```
Browse to the link provided, and then as you make changes to docstrings or narrative docs,
the pages will re-render and the browser will automatically refresh.
## Running Tests Locally
- Run `just install` to set a local virtual environment, or you can manually
create a virtual environment and run `pytest` directly. If you want to use a specific
version of Python, set `UV_PYTHON` before running `just install`.
- Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args
to set up the server. All given options will be passed to
[`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h`
for a full list of options.
- Run `just test` or `pytest` to run all of the tests.
- Ensure you have started the appropriate Mongo Server(s).
- Run `pip install tox` to use `tox` for testing or run
`pip install -e ".[test]"` to run `pytest` directly.
- Run `tox -m test` or `pytest` to run all of the tests.
- Append `test/<mod_name>.py::<class_name>::<test_name>` to run
specific tests. You can omit the `<test_name>` to test a full class
and the `<class_name>` to test a full module. For example:
`just test test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`.
`tox -m test -- test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`.
- Use the `-k` argument to select tests by pattern.
- Run `just test-coverage` to run tests with coverage and display a report. After running tests with coverage, use `just coverage-html` to generate an HTML report in `htmlcov/index.html`.
## Running Load Balancer Tests Locally
## Running tests that require secrets, services, or other configuration
### Prerequisites
- Clone `drivers-evergreen-tools`:
- Install `haproxy` (available as `brew install haproxy` on macOS).
- Clone `drivers-evergreen-tools`:
`git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`.
- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`. This can be put into a `.bashrc` file
for convenience.
- Some tests require access to [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling).
### Usage
- Run `just run-server` with optional args to set up the server.
- Run `just setup-tests` with optional args to set up the test environment, secrets, etc.
See `just setup-tests -h` for a full list of available options.
- Run `just run-tests` to run the tests in an appropriate Python environment.
- When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server.
### SSL tests
- Run `just run-server --ssl` to start the server with TLS enabled.
- Run `just setup-tests --ssl`.
- Run `just run-tests`.
Note: for general testing purposes with an TLS-enabled server, you can use the following (this should ONLY be used
for local testing):
```python
from pymongo import MongoClient
client = MongoClient(
"mongodb://localhost:27017?tls=true&tlsAllowInvalidCertificates=true"
)
```
If you want to use the actual certificate file then set `tlsCertificateKeyFile` to the local path
to `<repo_roo>/test/certificates/client.pem` and `tlsCAFile` to the local path to `<repo_roo>/test/certificates/ca.pem`.
### Encryption tests
- Run `just run-server` to start the server.
- Run `just setup-tests encryption`.
- Run the tests with `just run-tests`.
To test with `encryption` and `PyOpenSSL`, use `just setup-tests encryption pyopenssl`.
### PyOpenSSL tests
- Run `just run-server` to start the server.
- Run `just setup-tests default_sync pyopenssl`.
- Run the tests with `just run-tests`.
Note: `PyOpenSSL` is not used in async tests, but you can use `just setup-tests default_async pyopenssl`
to verify that PyMongo falls back to the standard library `OpenSSL`.
### Load balancer tests
- Install `haproxy` (available as `brew install haproxy` on macOS).
- Start the server with `just run-server load_balancer`.
- Set up the test with `just setup-tests load_balancer`.
- Run the tests with `just run-tests`.
### AWS auth tests
- Run `just run-server auth_aws` to start the server.
- Run `just setup-tests auth_aws <aws-test-type>` to set up the AWS test.
- Run the tests with `just run-tests`.
### OIDC auth tests
- Run `just setup-tests auth_oidc <oidc-test-type>` to set up the OIDC test.
- Run the tests with `just run-tests`.
The supported types are [`default`, `azure`, `gcp`, `eks`, `aks`, and `gke`].
For the `eks` test, you will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
### KMS tests
For KMS tests that are run locally, and expected to fail, in this case using `azure`:
- Run `just run-server`.
- Run `just setup-tests kms azure-fail`.
- Run `just run-tests`.
For KMS tests that run remotely and are expected to pass, in this case using `gcp`:
- Run `just setup-tests kms gcp`.
- Run `just run-tests`.
### Enterprise Auth tests
Note: these tests can only be run from an Evergreen host.
- Run `just run-server enterprise_auth`.
- Run `just setup-tests enterprise_auth`.
- Run `just run-tests`.
### Atlas Connect tests
- Run `just setup-tests atlas_connect`.
- Run `just run-tests`.
### Search Index tests
- Run `just run-server search_index`.
- Run `just setup-tests search_index`.
- Run `just run-tests`.
### MockupDB tests
- Run `just setup-tests mockupdb`.
- Run `just run-tests`.
### Doc tests
The doc tests require a running server.
- Run `just run-server`.
- Run `just setup-tests doctest`.
- Run `just run-tests`.
### Free-threaded Python Tests
In the evergreen builds, the tests are configured to use the free-threaded python from the toolchain.
Locally you can run:
- Run `just run-server`.
- Run `just setup-tests`.
- Run `UV_PYTHON=3.14t just run-tests`.
### AWS Lambda tests
You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
- Run `just setup-tests aws_lambda`.
- Run `just run-tests`.
### mod_wsgi tests
Note: these tests can only be run from an Evergreen Linux host that has the Python toolchain.
- Run `just run-server`.
- Run `just setup-tests mod_wsgi <mode>`.
- Run `just run-tests`.
The `mode` can be `standalone` or `embedded`. For the `replica_set` version of the tests, use
`TOPOLOGY=replica_set just run-server`.
### OCSP tests
- Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`.
This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`.
MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA.
NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server
as part of `setup-tests`.
- Run `just setup-tests ocsp <sub test>` (options are "valid", "revoked", "valid-delegate", "revoked-delegate").
- Run `just run-tests`
If you are running one of the `no-responder` tests, omit the `run-server` step.
### Perf Tests
- Start the appropriate server, e.g. `just run-server --version=v8.0-perf --ssl`.
- Set up the tests with `sync` or `async`: `just setup-tests perf sync`.
- Run the tests: `just run-tests`.
## Enable Debug Logs
- Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest` to output all debug logs to the terminal. **Warning**: This will output a huge amount of logs.
- Add `log_cli=1` and `log_cli_level="DEBUG"` to the `tool.pytest.ini_options` section in `pyproject.toml` to enable debug logs in this manner by default on your machine.
- Set `DEBUG_LOG=1` and run `just setup-tests`, `just-test`, or `pytest` to enable debug logs only for failed tests.
- Finally, you can use `just setup-tests --debug-log`.
- For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for failed tests in the patch.
## Testing minimum dependencies
To run any of the test suites with minimum supported dependencies, pass `--test-min-deps` to
`just setup-tests`.
## Testing time-dependent operations
- `test.utils_shared.delay` - One can trigger an arbitrarily long-running operation on the server using this delay utility
in combination with a `$where` operation. Use this to test behaviors around timeouts or signals.
## Adding a new test suite
- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add
to the list of pytest markers in `pyproject.toml`. Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`. If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead.
- If the test uses Atlas or otherwise doesn't use `run-mongodb.sh`, add it to the `NO_RUN_ORCHESTRATION` list in
`.evergreen/scripts/utils.py`.
- If there is something special required to run the local server or there is an extra flag that should always be set
like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`.
- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`. This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`.
- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`.
- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`.
- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`.
- There are some considerations about the Python version used in the test:
- If a specific version of Python is needed in a task that is running on variants with a toolchain, use
``TOOLCHAIN_VERSION`` (e.g. `TOOLCHAIN_VERSION=3.10`). The actual path lookup needs to be done on the host, since
tasks are host-agnostic.
- If a specific Python binary is needed (for example on the FIPS host), set `UV_PYTHON=/path/to/python`.
- If a specific Python version is needed and the toolchain will not be available, use `UV_PYTHON` (e.g. `UV_PYTHON=3.11`).
- The default if neither ``TOOLCHAIN_VERSION`` or ``UV_PYTHON`` is set is to use UV to install the minimum
supported version of Python and use that. This ensures a consistent behavior across host types that do not
have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`)
needed to build the C extensions.
- Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`.
- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`.
## Handling flaky tests
We have a custom `flaky` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for
tests that are `flaky`. By default the decorator only applies when not running on CPython on Linux, since other
runtimes tend to have more variation. When using the `flaky` decorator, open a corresponding ticket and
a use the ticket number as the "reason" parameter to the decorator, e.g. `@flaky(reason="PYTHON-1234")`.
When running tests locally (not in CI), the `flaky` decorator will be disabled unless `ENABLE_FLAKY` is set.
To disable the `flaky` decorator in CI, you can use `evergreen patch --param DISABLE_FLAKY=1`.
## Integration Tests
The `integration_tests` directory has a set of scripts that verify the usage of PyMongo with downstream packages or frameworks. See the [README](./integration_tests/README.md) for more information.
To run the tests, use `just integration_tests`.
The tests should be able to run with and without SSL enabled.
## Specification Tests
The MongoDB [specifications repository](https://github.com/mongodb/specifications)
holds in progress and completed specifications for features of MongoDB, drivers,
and associated products. PyMongo supports the [Unified Test Format](https://jira.mongodb.org/browse/DRIVERS-709)
for running specification tests to confirm PyMongo behaves as expected.
### Resynchronizing the Specification Tests
- Start the servers using
`LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh`.
- Start the load balancer using:
`MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`.
- Run the tests from the `pymongo` checkout directory using:
`TEST_LOADBALANCER=1 tox -m test-eg`.
## Running Encryption Tests Locally
- Run `AWS_PROFILE=<profile> tox -m setup-encryption` after setting up your AWS profile with `aws configure sso`.
- Run the tests with `TEST_ENCRYPTION=1 tox -e test-eg`.
- When done, run `tox -m teardown-encryption` to clean up.
## Re-sync Spec Tests
If you would like to re-sync the copy of the specification tests in the
PyMongo repository with that which is inside the [specifications
@ -466,115 +240,3 @@ The `-b` flag adds as a regex pattern to block files you do not wish to
update in PyMongo. This is primarily helpful if you are implementing a
new feature in PyMongo that has spec tests already implemented, or if
you are attempting to validate new spec tests in PyMongo.
### Automated Specification Test Resyncing
The (`/.evergreen/scripts/resync-all-specs.sh`) script
automatically runs once a week to resync all the specs with the [specifications repo](https://github.com/mongodb/specifications).
A PR will be generated by mongodb-drivers-pr-bot containing any changes picked up by this resync.
The PR description will display the name(s) of the updated specs along
with any errors that occurred.
Spec test changes associated with a behavioral change or bugfix that has yet to be implemented in PyMongo
must be added to a patch file in `/.evergreen/spec-patch`. Each patch
file must be named after the associated PYTHON ticket and contain the
test differences between PyMongo's current tests and the specification.
All changes listed in these patch files will be *undone* by the script and won't
be applied to PyMongo's tests.
When a new test file or folder is added to the spec repo before the associated code changes are implemented, that test's path must be added to `.evergreen/remove-unimplemented-tests.sh` along with a comment indicating the associated PYTHON ticket for those changes.
Any PR that implements a PYTHON ticket documented in a patch file or within `.evergreen/remove-unimplemented-tests.sh` must also remove the associated patch file or entry in `remove-unimplemented-tests.sh`.
#### Adding to a patch file
To add to or create a patch file, run `git diff` to show the desired changes to undo and copy the
results into the patch file.
For example: the imaginary, unimplemented PYTHON-1234 ticket has associated spec test changes. To add those changes to `PYTHON-1234.patch`), do the following:
```bash
git diff HEAD~1 path/to/file >> .evergreen/spec-patch/PYTHON-1234.patch
```
#### Running Locally
Both `resync-all-specs.sh` and `resync-all-specs.py` can be run locally (and won't generate a PR).
```bash
./.evergreen/scripts/resync-all-specs.sh
python3 ./.evergreen/scripts/resync-all-specs.py
```
## Making a Release
Follow the [Python Driver Release Process Wiki](https://wiki.corp.mongodb.com/display/DRIVERS/Python+Driver+Release+Process).
## Project Structure and Asyncio Considerations
This section describes the layout of the `pymongo/` package.
Within `pymongo/`, the code is further divided into the `pymongo/asynchronous` and `pymongo/synchronous` subdirectories.
Files in `pymongo/synchronous` are generated from `pymongo/asynchronous` using the `synchro` pre-commit hook, which uses [unasync](https://github.com/python-trio/unasync/) and some custom transforms.
As a result, **all modifications** within `pymongo` must be made in either the top-level `pymongo` directory when they have to exhibit differing behavior between sync and async contexts or the `pymongo/asynchronous` directory, not `pymongo/synchronous`.
Any changes made directly to files in the `pymongo/synchronous` directory will be overwritten by the `synchro` hook when it is run, which happens automatically on commit.
Some top-level files (e.g. `pymongo/collection.py`) are re-export files for existing import compatibility and should not be modified directly.
The other top-level files (e.g. `pymongo/network_layer.py`, `pymongo/pool_shared.py`) contain either shared code used in both the asynchronous and synchronous APIs, or code that is very different between the two APIs and therefore cannot be generated from the async version using `synchro`.
Run `pre-commit run --all-files synchro` before running tests to generate the latest version of the synchronous code.
To prevent the `synchro` hook from accidentally overwriting code, it first checks to see whether a sync version
of a file is changing and not its async counterpart, and will fail.
In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`.
Sometimes, the `synchro` hook will fail and introduce changes many previously unmodified files. This is due to static
Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues,
run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the `synchro`
hook again.
## Converting a test to async
The `tools/convert_test_to_async.py` script takes in an existing synchronous test file and outputs a
partially-converted asynchronous version of the same name to the `test/asynchronous` directory.
Use this generated file as a starting point for the completed conversion.
The script is used like so: `python tools/convert_test_to_async.py [test_file.py]`
## CPU profiling
To profile a test script and generate a flame graph, follow these steps:
1. Install `py-spy` if you haven't already:
```bash
pip install py-spy
```
2. Inside your test script, perform any required setup and then loop over the code you want to profile for improved sampling.
3. Run `py-spy record -o <output.svg> -r <sample_rate=100> -- python <path/to/script>` to generate a `.svg` file containing the flame graph.
(Note: on macOS you will need to run this command using `sudo` to allow `py-spy` to attach to the Python process.)
4. If you need to include native code (for example the C extensions), profiling should be done on a Linux system, as macOS and Windows do not support the `--native` option of `py-spy`.
Creating an ubuntu Evergreen spawn host and using `scp` to copy the flamegraph `.svg` file back to your local machine is the best way to do this.
5. You can then view the flamegraph using an SVG viewer like a browser.
## Memory profiling
To test for a memory leak or any memory-related issues, the current best tool is [memray](https://bloomberg.github.io/memray/overview.html).
In order to include code from our C extensions, it must be run in native mode, on Linux.
To do so, either spin up an Ubuntu docker container or an Ubuntu Evergreen spawn host.
From the spawn host or Ubuntu image, do the following:
1. Install `memray` if you haven't already:
```bash
pip install memray
```
2. Inside your test script, perform any required setup and then loop over the code you want to profile for improved sampling.
3. Run memray with the script under test with the `--native` flag, e.g. `python -m memray run --native -o test.bin <path/to/script>`.
4. Generate the flamegraph with `python -m memray flamegraph -o test.html test.bin`.
See the [docs](https://bloomberg.github.io/memray/flamegraph.html) for more options.
5. Then, from the host computer, use either scp or docker cp to copy the flamegraph, e.g. `scp ubuntu@ec2-3-82-52-49.compute-1.amazonaws.com:/home/ubuntu/test.html .`.
6. You can then view the flamegraph html in a browser.
## Dependabot updates
Dependabot will raise PRs at most once per week, grouped by GitHub Actions updates and Python requirement
file updates. We have a pre-commit hook that will update the `uv.lock` file when requirements change.
To update the lock file on a failing PR, you can use a method like `gh pr checkout <pr number>`, then run
`just lint uv-lock` to update the lock file, and then push the changes. If a typing dependency has changed,
also run `just typing` and handle any new findings.

32
MANIFEST.in Normal file
View File

@ -0,0 +1,32 @@
include README.md
include LICENSE
include THIRD-PARTY-NOTICES
include *.ini
include requirements.txt
exclude .coveragerc
exclude .git-blame-ignore-revs
exclude .pre-commit-config.yaml
exclude .readthedocs.yaml
exclude CONTRIBUTING.md
exclude RELEASE.md
recursive-include doc *.rst
recursive-include doc *.py
recursive-include doc *.conf
recursive-include doc *.css
recursive-include doc *.js
recursive-include doc *.png
include doc/Makefile
include doc/_templates/layout.html
include doc/make.bat
include doc/static/periodic-executor-refs.dot
recursive-include requirements *.txt
recursive-include tools *.py
include tools/README.rst
include green_framework_test.py
recursive-include test *.pem
recursive-include test *.py
recursive-include test *.json
recursive-include bson *.h
prune test/mod_wsgi_test
prune test/lambda
prune .evergreen

View File

@ -3,26 +3,18 @@
[![PyPI Version](https://img.shields.io/pypi/v/pymongo)](https://pypi.org/project/pymongo)
[![Python Versions](https://img.shields.io/pypi/pyversions/pymongo)](https://pypi.org/project/pymongo)
[![Monthly Downloads](https://static.pepy.tech/badge/pymongo/month)](https://pepy.tech/project/pymongo)
[![API Documentation Status](https://readthedocs.org/projects/pymongo/badge/?version=stable)](http://pymongo.readthedocs.io/en/stable/api?badge=stable)
[![codecov](https://codecov.io/gh/mongodb/mongo-python-driver/graph/badge.svg?branch=master)](https://codecov.io/gh/mongodb/mongo-python-driver)
[![Documentation Status](https://readthedocs.org/projects/pymongo/badge/?version=stable)](http://pymongo.readthedocs.io/en/stable/?badge=stable)
## About
The PyMongo distribution contains tools for interacting with MongoDB
database from Python. The `bson` package is an implementation of the
[BSON format](http://bsonspec.org) for Python. The `pymongo` package is
a native Python driver for MongoDB, offering both synchronous and asynchronous APIs. The `gridfs` package is a
[gridfs](https://github.com/mongodb/specifications/blob/master/source/gridfs/gridfs-spec.md/)
a native Python driver for MongoDB. The `gridfs` package is a
[gridfs](https://github.com/mongodb/specifications/blob/master/source/gridfs/gridfs-spec.rst/)
implementation on top of `pymongo`.
PyMongo supports MongoDB 4.0, 4.2, 4.4, 5.0, 6.0, 7.0, and 8.0. PyMongo follows [semantic versioning](https://semver.org/spec/v2.0.0.html) for its releases.
## Documentation
Documentation is available at
[mongodb.com](https://www.mongodb.com/docs/languages/python/pymongo-driver/current/).
[API documentation](https://pymongo.readthedocs.io/en/stable/api/) and the [full changelog](https://pymongo.readthedocs.io/en/stable/changelog.html) for each release is available at [readthedocs.io](https://pymongo.readthedocs.io/en/stable/index.html).
PyMongo supports MongoDB 3.6, 4.0, 4.2, 4.4, 5.0, 6.0, and 7.0.
## Support / Feedback
@ -86,6 +78,12 @@ PyMongo can be installed with [pip](http://pypi.python.org/pypi/pip):
python -m pip install pymongo
```
Or `easy_install` from [setuptools](http://pypi.python.org/pypi/setuptools):
```bash
python -m easy_install pymongo
```
You can also download the project source and do:
```bash
@ -98,7 +96,7 @@ package that is incompatible with PyMongo.
## Dependencies
PyMongo supports CPython 3.9+ and PyPy3.9+.
PyMongo supports CPython 3.8+ and PyPy3.9+.
Required dependencies:
@ -140,8 +138,7 @@ python -m pip install "pymongo[snappy]"
```
Wire protocol compression with zstandard requires
[backports.zstd](https://pypi.org/project/backports.zstd)
when used with Python versions before 3.14:
[zstandard](https://pypi.org/project/zstandard):
```bash
python -m pip install "pymongo[zstd]"
@ -161,6 +158,11 @@ command:
python -m pip install "pymongo[gssapi,aws,ocsp,snappy,zstd,encryption]"
```
Additional dependencies are:
- (to generate documentation or run tests)
[tox](https://tox.wiki/en/latest/index.html)
## Examples
Here's a basic example (for more see the *examples* section of the
@ -200,6 +202,14 @@ ObjectId('4aba160ee23f6b543e000002')
[8, 11]
```
## Documentation
Documentation is available at
[pymongo.readthedocs.io](https://pymongo.readthedocs.io/en/stable/).
Documentation can be generated by running **tox -m doc**. Generated
documentation can be found in the `doc/build/html/` directory.
## Learning Resources
- MongoDB Learn - [Python
@ -209,11 +219,9 @@ Center](https://www.mongodb.com/developer/languages/python/).
## Testing
The easiest way to run the tests is to run the following from the repository root.
The easiest way to run the tests is to run **tox -m test** in the root
of the distribution. For example,
```bash
pip install -e ".[test]"
pytest
tox -e test
```
For more advanced testing scenarios, see the [contributing guide](https://github.com/mongodb/mongo-python-driver/blob/master/CONTRIBUTING.md#running-tests-locally).

109
RELEASE.md Normal file
View File

@ -0,0 +1,109 @@
# Some notes on PyMongo releases
## Versioning
We follow [semver](https://semver.org/) and [pep-0440](https://www.python.org/dev/peps/pep-0440)
for versioning.
We shoot for a release every few months - that will generally just
increment the middle / minor version number (e.g. `3.5.0` -> `3.6.0`).
Patch releases are reserved for bug fixes (in general no new features or
deprecations) - they only happen in cases where there is a critical bug
in a recently released version, or when a release has no new features or
API changes.
In between releases we add `.devN` to the version number to denote the
version under development. So if we just released `3.6.0`, then the
current dev version might be `3.6.1.dev0` or `3.7.0.dev0`. When we make the
next release we replace all instances of `3.x.x.devN` in the docs with the
new version number.
## Deprecation
Changes should be backwards compatible unless absolutely necessary. When
making API changes the approach is generally to add a deprecation
warning but keeping the existing API functional. Deprecated features can
be removed in a release that changes the major version number.
## Doing a Release
1. PyMongo is tested on Evergreen. Ensure the latest commit are passing
[CI](https://spruce.mongodb.com/commits/mongo-python-driver) as expected.
2. Check Jira to ensure all the tickets in this version have been
completed.
3. Make a PR that adds the release notes to `doc/changelog.rst`. Generally just
summarize/clarify the git log, but you might add some more long form
notes for big changes.
4. Merge the PR.
5. Clone the source repository in a temporary directory and check out the
release branch.
6. Update the version number in `pymongo/_version.py`.
7. Commit the change, e.g. `git add . && git commit -m "BUMP <VERSION>"`
7. Tag w/ version_number, eg,
`git tag -a '4.1.0' -m 'BUMP 4.1.0'`.
8. Bump the version number to `<next version>.dev0` in
`pymongo/_version.py`, commit, push.
9. Push commit / tag, eg `git push && git push --tags`.
10. Pushing a tag will trigger the release process on GitHub Actions
that will require a member of the team to authorize the deployment.
Navigate to https://github.com/mongodb/mongo-python-driver/actions/workflows/release-python.yml
and wait for the publish to complete.
11. Make sure the new version appears on
`https://pymongo.readthedocs.io/en/stable/`. If the new version does not show
up automatically, trigger a rebuild of "stable" on https://readthedocs.org/projects/pymongo/builds/.
12. Publish the release version in Jira and add a description of the release, such as a the reason
or the main feature.
13. Announce the release on the [community forum](https://www.mongodb.com/community/forums/tags/c/announcements/driver-releases/110/python)
14. File a ticket for DOCSP highlighting changes in server version and
Python version compatibility or the lack thereof, for example https://jira.mongodb.org/browse/DOCSP-34040
15. Create a GitHub Release for the tag using https://github.com/mongodb/mongo-python-driver/releases/new.
The title should be "PyMongo X.Y.Z", and the description should
contain a link to the release notes on the the community forum, e.g.
"Release notes: mongodb.com/community/forums/t/pymongo-4-0-2-released/150457"
16. Wait for automated update PR on conda-forge, e.g.: https://github.com/conda-forge/pymongo-feedstock/pull/81
Update dependencies if needed.
## Doing a Bug Fix Release
1. If it is a new branch, first create the release branch and Evergreen project.
- Clone the source repository in a temporary location.
- Create a branch from the tag, e.g. `git checkout -b v4.1 4.1.0`.
- Push the branch, e.g.: `git push origin v4.6`.
- Create a new project in Evergreen for the branch by duplicating the "Mongo Python Driver" project.
Select the option to create a JIRA ticket for S3 bucket permissions.
- Update the "Display Name", "Branch Name", and "Identifier".
- Attach the project to the repository.
- Wait for the JIRA ticket to be resolved and verify S3 upload capability with a patch release on the
new project.
2. Create a PR against the release branch.
3. Create a release using the "Doing a Release" checklist above, ensuring that you
check out the appropriate release branch in the source checkout.
4. Cherry-pick the changelog PR onto the `master` branch.

View File

@ -39,60 +39,35 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
2) License Notice for _asyncio_lock.py
2) License Notice for bson-stdint-win32.h
-----------------------------------------
1. This LICENSE AGREEMENT is between the Python Software Foundation
("PSF"), and the Individual or Organization ("Licensee") accessing and
otherwise using this software ("Python") in source or binary form and
its associated documentation.
ISO C9x compliant stdint.h for Microsoft Visual Studio
Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
2. Subject to the terms and conditions of this License Agreement, PSF hereby
grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
analyze, test, perform and/or display publicly, prepare derivative works,
distribute, and otherwise use Python alone or in any derivative version,
provided, however, that PSF's License Agreement and PSF's notice of copyright,
i.e., "Copyright (c) 2001-2024 Python Software Foundation; All Rights Reserved"
are retained in Python alone or in any derivative version prepared by Licensee.
Copyright (c) 2006-2013 Alexander Chemeris
3. In the event Licensee prepares a derivative work that is based on
or incorporates Python or any part thereof, and wants to make
the derivative work available to others as provided herein, then
Licensee hereby agrees to include in any such work a brief summary of
the changes made to Python.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
4. PSF is making Python available to Licensee on an "AS IS"
basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
6. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
3. Neither the name of the product nor the names of its contributors may
be used to endorse or promote products derived from this software
without specific prior written permission.
7. Nothing in this License Agreement shall be deemed to create any
relationship of agency, partnership, or joint venture between PSF and
Licensee. This License Agreement does not grant permission to use PSF
trademarks or trade name in a trademark sense to endorse or promote
products or services of Licensee, or any third party.
8. By copying, installing or otherwise using Python, Licensee
agrees to be bound by the terms and conditions of this License
Agreement.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

152
_setup.py
View File

@ -1,152 +0,0 @@
from __future__ import annotations
import os
import sys
import warnings
# Hack to silence atexit traceback in some Python versions
try:
import multiprocessing # noqa: F401
except ImportError:
pass
from setuptools import setup
from setuptools.command.build_ext import build_ext
from setuptools.extension import Extension
class custom_build_ext(build_ext):
"""Allow C extension building to fail.
The C extension speeds up BSON encoding, but is not essential.
"""
warning_message = """
********************************************************************
WARNING: %s could not
be compiled. No C extensions are essential for PyMongo to run,
although they do result in significant speed improvements.
%s
Please see the installation docs for solutions to build issues:
https://pymongo.readthedocs.io/en/stable/installation.html
Here are some hints for popular operating systems:
If you are seeing this message on Linux you probably need to
install GCC and/or the Python development package for your
version of Python.
Debian and Ubuntu users should issue the following command:
$ sudo apt-get install build-essential python-dev
Users of Red Hat based distributions (RHEL, CentOS, Amazon Linux,
Oracle Linux, Fedora, etc.) should issue the following command:
$ sudo yum install gcc python-devel
If you are seeing this message on Microsoft Windows please install
PyMongo using pip. Modern versions of pip will install PyMongo
from binary wheels available on pypi. If you must install from
source read the documentation here:
https://pymongo.readthedocs.io/en/stable/installation.html#installing-from-source-on-windows
If you are seeing this message on macOS / OSX please install PyMongo
using pip. Modern versions of pip will install PyMongo from binary
wheels available on pypi. If wheels are not available for your version
of macOS / OSX, or you must install from source read the documentation
here:
https://pymongo.readthedocs.io/en/stable/installation.html#osx
********************************************************************
"""
def run(self):
try:
build_ext.run(self)
except Exception:
if os.environ.get("PYMONGO_C_EXT_MUST_BUILD"):
raise
e = sys.exc_info()[1]
sys.stdout.write("%s\n" % str(e))
warnings.warn(
self.warning_message
% (
"Extension modules",
"There was an issue with your platform configuration - see above.",
),
stacklevel=2,
)
def build_extension(self, ext):
# "ProgramFiles(x86)" is not a valid environment variable in Cygwin but is needed for
# the MSVCCompiler in distutils.
if os.name == "nt":
if "ProgramFiles" in os.environ and "ProgramFiles(x86)" not in os.environ:
os.environ["ProgramFiles(x86)"] = os.environ["ProgramFiles"] + " (x86)"
name = ext.name
try:
build_ext.build_extension(self, ext)
except Exception:
if os.environ.get("PYMONGO_C_EXT_MUST_BUILD"):
raise
e = sys.exc_info()[1]
sys.stdout.write("%s\n" % str(e))
warnings.warn(
self.warning_message
% (
"The %s extension module" % (name,), # noqa: UP031
"The output above this warning shows how the compilation failed.",
),
stacklevel=2,
)
ext_modules = [
Extension(
"bson._cbson",
include_dirs=["bson"],
sources=["bson/_cbsonmodule.c", "bson/time64.c", "bson/buffer.c"],
),
Extension(
"pymongo._cmessage",
include_dirs=["bson"],
sources=[
"pymongo/_cmessagemodule.c",
"bson/_cbsonmodule.c",
"bson/time64.c",
"bson/buffer.c",
],
),
]
if "--no_ext" in sys.argv or os.environ.get("NO_EXT"):
try:
sys.argv.remove("--no_ext")
except ValueError:
pass
ext_modules = []
elif (
sys.platform.startswith("java")
or sys.platform == "cli"
or sys.implementation.name in ("pypy", "graalpy")
):
sys.stdout.write(
"""
*****************************************************\n
The optional C extensions are currently not supported\n
by this python implementation.\n
*****************************************************\n
"""
)
ext_modules = []
setup(
cmdclass={"build_ext": custom_build_ext},
ext_modules=ext_modules,
packages=["bson", "pymongo", "gridfs"],
) # type:ignore

View File

@ -22,46 +22,30 @@ Python Type BSON Type Supported Direction
None null both
bool boolean both
int [#int]_ int32 / int64 py -> bson
:class:`bson.int64.Int64` int64 both
`bson.int64.Int64` int64 both
float number (real) both
str string both
list array both
dict object both
:class:`~bson.son.SON` object both
:py:class:`~collections.abc.Mapping` object py -> bson
:class:`~bson.raw_bson.RawBSONDocument` object both [#raw]_
datetime.datetime [#dt]_ [#dt2]_ UTC datetime both
:class:`~bson.datetime_ms.DatetimeMS` UTC datetime both [#dt3]_
:class:`~bson.regex.Regex` regex both
dict / `SON` object both
datetime.datetime [#dt]_ [#dt2]_ date both
`bson.regex.Regex` regex both
compiled re [#re]_ regex py -> bson
:class:`~bson.binary.Binary` binary both
:py:class:`uuid.UUID` [#uuid]_ binary both
:class:`~bson.objectid.ObjectId` oid both
:class:`~bson.dbref.DBRef` dbref both
:class:`~bson.dbref.DBRef` dbpointer bson -> py
`bson.binary.Binary` binary both
`bson.objectid.ObjectId` oid both
`bson.dbref.DBRef` dbref both
None undefined bson -> py
:class:`~bson.code.Code` code both
`bson.code.Code` code both
str symbol bson -> py
bytes [#bytes]_ binary both
:class:`~bson.timestamp.Timestamp` timestamp both
:class:`~bson.decimal128.Decimal128` decimal128 both
:class:`~bson.min_key.MinKey` min key both
:class:`~bson.max_key.MaxKey` max key both
======================================= ============= ===================
.. [#int] A Python int will be saved as a BSON int32 or BSON int64 depending
on its size. A BSON int32 will always decode to a Python int. A BSON
int64 will always decode to a :class:`~bson.int64.Int64`.
.. [#raw] Decoding a bson object to :class:`~bson.raw_bson.RawBSONDocument` can be
optionally configured via :attr:`~bson.codec_options.CodecOptions.document_class`.
.. [#dt] datetime.datetime instances are encoded with millisecond precision so
the microsecond field is truncated.
.. [#dt2] all datetime.datetime instances are encoded as UTC. By default, they
are decoded as *naive* but timezone aware datetimes are also supported.
See `Dates and Times <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#dates-and-times>`_ for examples.
.. [#dt3] To enable decoding a bson UTC datetime to a :class:`~bson.datetime_ms.DatetimeMS`
instance see `handling out of range datetimes <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes>`_.
.. [#uuid] For :py:class:`uuid.UUID` encoding and decoding behavior see `<https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#universally-unique-ids--uuids->`_.
.. [#dt] datetime.datetime instances will be rounded to the nearest
millisecond when saved
.. [#dt2] all datetime.datetime instances are treated as *naive*. clients
should always use UTC.
.. [#re] :class:`~bson.regex.Regex` instances and regular expression
objects from ``re.compile()`` are both saved as BSON regular expressions.
BSON regular expressions are decoded as :class:`~bson.regex.Regex`
@ -300,10 +284,10 @@ def _get_object_size(data: Any, position: int, obj_end: int) -> Tuple[int, int]:
except struct.error as exc:
raise InvalidBSON(str(exc)) from None
end = position + obj_size - 1
if end >= obj_end:
raise InvalidBSON("invalid object length")
if data[end] != 0:
raise InvalidBSON("bad eoo")
if end >= obj_end:
raise InvalidBSON("invalid object length")
# If this is the top-level document, validate the total size too.
if position == 0 and obj_size != obj_end:
raise InvalidBSON("invalid object length")
@ -1006,10 +990,7 @@ def _dict_to_bson(
elements.append(_name_value_to_bson(b"_id\x00", doc["_id"], check_keys, opts))
for key, value in doc.items():
if not top_level or key != "_id":
try:
elements.append(_element_to_bson(key, value, check_keys, opts))
except InvalidDocument as err:
raise InvalidDocument(f"Invalid document: {err}", doc) from err
elements.append(_element_to_bson(key, value, check_keys, opts))
except AttributeError:
raise TypeError(f"encoder expected a mapping type but got: {doc!r}") from None
@ -1109,9 +1090,7 @@ def _decode_all(data: _ReadableBuffer, opts: CodecOptions[_DocumentType]) -> lis
while position < end:
obj_size = _UNPACK_INT_FROM(data, position)[0]
if data_len - position < obj_size:
raise InvalidBSON(
f"invalid object size: expected {obj_size}, got {data_len - position}"
)
raise InvalidBSON("invalid object size")
obj_end = position + obj_size - 1
if data[obj_end] != 0:
raise InvalidBSON("bad eoo")
@ -1201,10 +1180,9 @@ def _decode_selective(
return doc
def _array_of_documents_to_buffer(data: Union[memoryview, bytes]) -> bytes:
def _array_of_documents_to_buffer(view: memoryview) -> bytes:
# Extract the raw bytes of each document.
position = 0
view = memoryview(data)
_, end = _get_object_size(view, position, len(view))
position += 4
buffers: list[memoryview] = []
@ -1329,7 +1307,7 @@ def decode_iter(
elements = data[position : position + obj_size]
position += obj_size
yield _bson_to_dict(elements, opts)
yield _bson_to_dict(elements, opts) # type:ignore[misc, type-var]
@overload
@ -1375,7 +1353,7 @@ def decode_file_iter(
raise InvalidBSON("cut off in middle of objsize")
obj_size = _UNPACK_INT_FROM(size_data, 0)[0] - 4
elements = size_data + file_obj.read(max(0, obj_size))
yield _bson_to_dict(elements, opts) # type:ignore[misc]
yield _bson_to_dict(elements, opts) # type:ignore[type-var, arg-type, misc]
def is_valid(bson: bytes) -> bool:
@ -1388,7 +1366,7 @@ def is_valid(bson: bytes) -> bool:
:param bson: the data to be validated
"""
if not isinstance(bson, bytes):
raise TypeError(f"BSON data must be an instance of a subclass of bytes, not {type(bson)}")
raise TypeError("BSON data must be an instance of a subclass of bytes")
try:
_bson_to_dict(bson, DEFAULT_CODEC_OPTIONS)

File diff suppressed because it is too large Load Diff

View File

@ -72,7 +72,6 @@ typedef struct codec_options_t {
unsigned char datetime_conversion;
PyObject* options_obj;
unsigned char is_raw_bson;
unsigned char is_dict_class;
} codec_options_t;
/* C API functions */

View File

@ -13,10 +13,7 @@
# limitations under the License.
from __future__ import annotations
import struct
import warnings
from enum import Enum
from typing import TYPE_CHECKING, Any, Optional, Sequence, Tuple, Type, Union, overload
from typing import TYPE_CHECKING, Any, Tuple, Type, Union
from uuid import UUID
"""Tools for representing BSON binary data.
@ -65,9 +62,6 @@ if TYPE_CHECKING:
from array import array as _array
from mmap import mmap as _mmap
import numpy as np
import numpy.typing as npt
class UuidRepresentation:
UNSPECIFIED = 0
@ -82,7 +76,7 @@ class UuidRepresentation:
:class:`~bson.binary.Binary` instance will be returned instead of a
:class:`uuid.UUID` instance.
See `unspecified representation details <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#unspecified>`_ for details.
See :ref:`unspecified-representation-details` for details.
.. versionadded:: 3.11
"""
@ -94,7 +88,7 @@ class UuidRepresentation:
and decoded from BSON binary, using RFC-4122 byte order with
binary subtype :data:`UUID_SUBTYPE`.
See `standard representation details <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#standard>`_ for details.
See :ref:`standard-representation-details` for details.
.. versionadded:: 3.11
"""
@ -106,7 +100,7 @@ class UuidRepresentation:
and decoded from BSON binary, using RFC-4122 byte order with
binary subtype :data:`OLD_UUID_SUBTYPE`.
See `python legacy representation details <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#python_legacy>`_ for details.
See :ref:`python-legacy-representation-details` for details.
.. versionadded:: 3.11
"""
@ -118,7 +112,7 @@ class UuidRepresentation:
and decoded from BSON binary subtype :data:`OLD_UUID_SUBTYPE`,
using the Java driver's legacy byte order.
See `Java Legacy UUID <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#java_legacy>`_ for details.
See :ref:`java-legacy-representation-details` for details.
.. versionadded:: 3.11
"""
@ -130,7 +124,7 @@ class UuidRepresentation:
and decoded from BSON binary subtype :data:`OLD_UUID_SUBTYPE`,
using the C# driver's legacy byte order.
See `C# Legacy UUID <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#csharp_legacy>`_ for details.
See :ref:`csharp-legacy-representation-details` for details.
.. versionadded:: 3.11
"""
@ -197,94 +191,21 @@ SENSITIVE_SUBTYPE = 8
"""
VECTOR_SUBTYPE = 9
"""BSON binary subtype for densely packed vector data.
.. versionadded:: 4.10
"""
USER_DEFINED_SUBTYPE = 128
"""BSON binary subtype for any user defined structure.
"""
class BinaryVectorDtype(Enum):
"""Datatypes of vector subtype.
:param FLOAT32: (0x27) Pack list of :class:`float` as float32
:param INT8: (0x03) Pack list of :class:`int` in [-128, 127] as signed int8
:param PACKED_BIT: (0x10) Pack list of :class:`int` in [0, 255] as unsigned uint8
The `PACKED_BIT` value represents a special case where vector values themselves
can only be of two values (0 or 1) but these are packed together into groups of 8,
a byte. In Python, these are displayed as ints in range [0, 255]
Each value is of type bytes with a length of one.
.. versionadded:: 4.10
"""
INT8 = b"\x03"
FLOAT32 = b"\x27"
PACKED_BIT = b"\x10"
class BinaryVector:
"""Vector of numbers along with metadata for binary interoperability.
.. versionadded:: 4.10
"""
__slots__ = ("data", "dtype", "padding")
def __init__(
self,
data: Union[Sequence[float | int], npt.NDArray[np.number]],
dtype: BinaryVectorDtype,
padding: int = 0,
):
"""
:param data: Sequence of numbers representing the mathematical vector.
:param dtype: The data type stored in binary
:param padding: The number of bits in the final byte that are to be ignored
when a vector element's size is less than a byte
and the length of the vector is not a multiple of 8.
(Padding is equivalent to a negative value of `count` in
`numpy.unpackbits <https://numpy.org/doc/stable/reference/generated/numpy.unpackbits.html>`_)
"""
self.data = data
self.dtype = dtype
self.padding = padding
def __repr__(self) -> str:
return f"BinaryVector(dtype={self.dtype}, padding={self.padding}, data={self.data})"
def __eq__(self, other: Any) -> bool:
if not isinstance(other, BinaryVector):
return False
return (
self.dtype == other.dtype and self.padding == other.padding and self.data == other.data
)
def __len__(self) -> int:
return len(self.data)
class Binary(bytes):
"""Representation of BSON binary data.
We want to represent Python strings as the BSON string type.
We need to wrap binary data so that we can tell
This is necessary because we want to represent Python strings as
the BSON string type. We need to wrap binary data so we can tell
the difference between what should be considered binary data and
what should be considered a string when we encode to BSON.
Subtype 9 provides a space-efficient representation of 1-dimensional vector data.
Its data is prepended with two bytes of metadata.
The first (dtype) describes its data type, such as float32 or int8.
The second (padding) prescribes the number of bits to ignore in the final byte.
This is relevant when the element size of the dtype is not a multiple of 8.
Raises TypeError if `subtype` is not an instance of :class:`int`.
Raises TypeError if `data` is not an instance of :class:`bytes`
or `subtype` is not an instance of :class:`int`.
Raises ValueError if `subtype` is not in [0, 256).
.. note::
@ -297,10 +218,7 @@ class Binary(bytes):
to use
.. versionchanged:: 3.9
Support any bytes-like type that implements the buffer protocol.
.. versionchanged:: 4.10
Addition of vector subtype.
Support any bytes-like type that implements the buffer protocol.
"""
_type_marker = 5
@ -308,11 +226,11 @@ class Binary(bytes):
def __new__(
cls: Type[Binary],
data: Union[memoryview, bytes, bytearray, _mmap, _array[Any]],
data: Union[memoryview, bytes, _mmap, _array[Any]],
subtype: int = BINARY_SUBTYPE,
) -> Binary:
if not isinstance(subtype, int):
raise TypeError(f"subtype must be an instance of int, not {type(subtype)}")
raise TypeError("subtype must be an instance of int")
if subtype >= 256 or subtype < 0:
raise ValueError("subtype must be contained in [0, 256)")
# Support any type that implements the buffer protocol.
@ -338,12 +256,12 @@ class Binary(bytes):
:param uuid_representation: A member of
:class:`~bson.binary.UuidRepresentation`. Default:
:const:`~bson.binary.UuidRepresentation.STANDARD`.
See `UUID representations <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#universally-unique-ids--uuids->`_ for details.
See :ref:`handling-uuid-data-example` for details.
.. versionadded:: 3.11
"""
if not isinstance(uuid, UUID):
raise TypeError(f"uuid must be an instance of uuid.UUID, not {type(uuid)}")
raise TypeError("uuid must be an instance of uuid.UUID")
if uuid_representation not in ALL_UUID_REPRESENTATIONS:
raise ValueError(
@ -387,7 +305,7 @@ class Binary(bytes):
:param uuid_representation: A member of
:class:`~bson.binary.UuidRepresentation`. Default:
:const:`~bson.binary.UuidRepresentation.STANDARD`.
See `UUID representations <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#universally-unique-ids--uuids->`_ for details.
See :ref:`handling-uuid-data-example` for details.
.. versionadded:: 3.11
"""
@ -419,219 +337,6 @@ class Binary(bytes):
f"cannot decode subtype {self.subtype} to {UUID_REPRESENTATION_NAMES[uuid_representation]}"
)
@classmethod
@overload
def from_vector(cls: Type[Binary], vector: BinaryVector) -> Binary:
...
@classmethod
@overload
def from_vector(
cls: Type[Binary],
vector: Union[list[int], list[float]],
dtype: BinaryVectorDtype,
padding: int = 0,
) -> Binary:
...
@classmethod
@overload
def from_vector(
cls: Type[Binary],
vector: npt.NDArray[np.number],
dtype: BinaryVectorDtype,
padding: int = 0,
) -> Binary:
...
@classmethod
def from_vector(
cls: Type[Binary],
vector: Union[BinaryVector, list[int], list[float], npt.NDArray[np.number]],
dtype: Optional[BinaryVectorDtype] = None,
padding: Optional[int] = None,
) -> Binary:
"""Create a BSON :class:`~bson.binary.Binary` of Vector subtype.
To interpret the representation of the numbers, a data type must be included.
See :class:`~bson.binary.BinaryVectorDtype` for available types and descriptions.
The dtype and padding are prepended to the binary data's value.
:param vector: Either a List of values, or a :class:`~bson.binary.BinaryVector` dataclass.
:param dtype: Data type of the values
:param padding: For fractional bytes, number of bits to ignore at end of vector.
:return: Binary packed data identified by dtype and padding.
.. versionchanged:: 4.14
When padding is non-zero, ignored bits should be zero. Raise exception on encoding, warn on decoding.
.. versionadded:: 4.10
"""
if isinstance(vector, BinaryVector):
if dtype or padding:
raise ValueError(
"The first argument, vector, has type BinaryVector. "
"dtype or padding cannot be separately defined, but were."
)
dtype = vector.dtype
padding = vector.padding
vector = vector.data # type: ignore
padding = 0 if padding is None else padding
if not isinstance(dtype, BinaryVectorDtype):
raise TypeError(
"dtype must be a bson.BinaryVectorDtype of BinaryVectorDType.INT8, PACKED_BIT, FLOAT32"
)
metadata = struct.pack("<sB", dtype.value, padding)
if isinstance(vector, list):
if dtype == BinaryVectorDtype.INT8: # pack ints in [-128, 127] as signed int8
format_str = "b"
if padding:
raise ValueError(f"padding does not apply to {dtype=}")
elif dtype == BinaryVectorDtype.PACKED_BIT: # pack ints in [0, 255] as unsigned uint8
format_str = "B"
if 0 <= padding > 7:
raise ValueError(f"{padding=}. It must be in [0,1, ..7].")
if padding and not vector:
raise ValueError("Empty vector with non-zero padding.")
elif dtype == BinaryVectorDtype.FLOAT32: # pack floats as float32
format_str = "f"
if padding:
raise ValueError(f"padding does not apply to {dtype=}")
else:
raise NotImplementedError("%s not yet supported" % dtype)
data = struct.pack(f"<{len(vector)}{format_str}", *vector)
else: # vector is numpy array or incorrect type.
try:
import numpy as np
except ImportError as exc:
raise ImportError(
"Failed to create binary from vector. Check type. If numpy array, numpy must be installed."
) from exc
if not isinstance(vector, np.ndarray):
raise TypeError(
"Could not create Binary. Vector must be a BinaryVector, list[int], list[float] or numpy ndarray."
)
if vector.ndim != 1:
raise ValueError(
"from_numpy_vector only supports 1D arrays as it creates a single vector."
)
if dtype == BinaryVectorDtype.FLOAT32:
vector = vector.astype(np.dtype("float32"), copy=False)
elif dtype == BinaryVectorDtype.INT8:
if vector.min() >= -128 and vector.max() <= 127:
vector = vector.astype(np.dtype("int8"), copy=False)
else:
raise ValueError("Values found outside INT8 range.")
elif dtype == BinaryVectorDtype.PACKED_BIT:
if vector.min() >= 0 and vector.max() <= 127:
vector = vector.astype(np.dtype("uint8"), copy=False)
else:
raise ValueError("Values found outside UINT8 range.")
else:
raise NotImplementedError("%s not yet supported" % dtype)
data = vector.tobytes()
if padding and len(vector) and not (data[-1] & ((1 << padding) - 1)) == 0:
raise ValueError(
"Vector has a padding P, but bits in the final byte lower than P are non-zero. They must be zero."
)
return cls(metadata + data, subtype=VECTOR_SUBTYPE)
def as_vector(self, return_numpy: bool = False) -> BinaryVector:
"""From the Binary, create a list or 1-d numpy array of numbers, along with dtype and padding.
:param return_numpy: If True, BinaryVector.data will be a one-dimensional numpy array. By default, it is a list.
:return: BinaryVector
.. versionadded:: 4.10
"""
if self.subtype != VECTOR_SUBTYPE:
raise ValueError(f"Cannot decode subtype {self.subtype} as a vector")
dtype, padding = struct.unpack_from("<sB", self)
dtype = BinaryVectorDtype(dtype)
offset = 2
n_bytes = len(self) - offset
if padding and dtype != BinaryVectorDtype.PACKED_BIT:
raise ValueError(
f"Corrupt data. Padding ({padding}) must be 0 for all but PACKED_BIT dtypes. ({dtype=})"
)
if not return_numpy:
if dtype == BinaryVectorDtype.INT8:
dtype_format = "b"
format_string = f"<{n_bytes}{dtype_format}"
vector = list(struct.unpack_from(format_string, self, offset))
return BinaryVector(vector, dtype, padding)
elif dtype == BinaryVectorDtype.FLOAT32:
n_values = n_bytes // 4
if n_bytes % 4:
raise ValueError(
"Corrupt data. N bytes for a float32 vector must be a multiple of 4."
)
dtype_format = "f"
format_string = f"<{n_values}{dtype_format}"
vector = list(struct.unpack_from(format_string, self, offset))
return BinaryVector(vector, dtype, padding)
elif dtype == BinaryVectorDtype.PACKED_BIT:
# data packed as uint8
if padding and not n_bytes:
raise ValueError("Corrupt data. Vector has a padding P, but no data.")
if padding > 7 or padding < 0:
raise ValueError(f"Corrupt data. Padding ({padding}) must be between 0 and 7.")
dtype_format = "B"
format_string = f"<{n_bytes}{dtype_format}"
unpacked_uint8s = list(struct.unpack_from(format_string, self, offset))
if padding and n_bytes and unpacked_uint8s[-1] & (1 << padding) - 1 != 0:
warnings.warn(
"Vector has a padding P, but bits in the final byte lower than P are non-zero. For pymongo>=5.0, they must be zero.",
DeprecationWarning,
stacklevel=2,
)
return BinaryVector(unpacked_uint8s, dtype, padding)
else:
raise NotImplementedError("Binary Vector dtype %s not yet supported" % dtype.name)
else: # create a numpy array
try:
import numpy as np
except ImportError as exc:
raise ImportError(
"Converting binary to numpy.ndarray requires numpy to be installed."
) from exc
if dtype == BinaryVectorDtype.INT8:
data = np.frombuffer(self[offset:], dtype="int8")
elif dtype == BinaryVectorDtype.FLOAT32:
if n_bytes % 4:
raise ValueError(
"Corrupt data. N bytes for a float32 vector must be a multiple of 4."
)
data = np.frombuffer(self[offset:], dtype="float32")
elif dtype == BinaryVectorDtype.PACKED_BIT:
# data packed as uint8
if padding and not n_bytes:
raise ValueError("Corrupt data. Vector has a padding P, but no data.")
if padding > 7 or padding < 0:
raise ValueError(f"Corrupt data. Padding ({padding}) must be between 0 and 7.")
data = np.frombuffer(self[offset:], dtype="uint8")
if padding and np.unpackbits(data[-1])[-padding:].sum() > 0:
warnings.warn(
"Vector has a padding P, but bits in the final byte lower than P are non-zero. For pymongo>=5.0, they must be zero.",
DeprecationWarning,
stacklevel=2,
)
else:
raise NotImplementedError("Binary Vector dtype %s not yet supported" % dtype.name)
return BinaryVector(data, dtype, padding)
@property
def subtype(self) -> int:
"""Subtype of this binary data."""

View File

@ -56,7 +56,7 @@ class Code(str):
**kwargs: Any,
) -> Code:
if not isinstance(code, str):
raise TypeError(f"code must be an instance of str, not {type(code)}")
raise TypeError("code must be an instance of str")
self = str.__new__(cls, code)
@ -67,7 +67,7 @@ class Code(str):
if scope is not None:
if not isinstance(scope, _Mapping):
raise TypeError(f"scope must be an instance of dict, not {type(scope)}")
raise TypeError("scope must be an instance of dict")
if self.__scope is not None:
self.__scope.update(scope) # type: ignore
else:

View File

@ -57,7 +57,7 @@ class TypeEncoder(abc.ABC):
Codec classes must implement the ``python_type`` attribute, and the
``transform_python`` method to support encoding.
See `encode data with type codecs <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/custom-types/type-codecs/#encode-data-with-type-codecs>`_ documentation for an example.
See :ref:`custom-type-type-codec` documentation for an example.
"""
@abc.abstractproperty
@ -76,7 +76,7 @@ class TypeDecoder(abc.ABC):
Codec classes must implement the ``bson_type`` attribute, and the
``transform_bson`` method to support decoding.
See `encode data with type codecs <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/custom-types/type-codecs/#encode-data-with-type-codecs>`_ documentation for an example.
See :ref:`custom-type-type-codec` documentation for an example.
"""
@abc.abstractproperty
@ -98,7 +98,7 @@ class TypeCodec(TypeEncoder, TypeDecoder):
``bson_type`` attribute, and the ``transform_bson`` method to support
decoding.
See `encode data with type codecs <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/custom-types/type-codecs/#encode-data-with-type-codecs>`_ documentation for an example.
See :ref:`custom-type-type-codec` documentation for an example.
"""
@ -118,7 +118,7 @@ class TypeRegistry:
>>> type_registry = TypeRegistry([Codec1, Codec2, Codec3, ...],
... fallback_encoder)
See `add codec to the type registry <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/custom-types/type-codecs/#add-codec-to-the-type-registry>`_ documentation for an example.
See :ref:`custom-type-type-registry` documentation for an example.
:param type_codecs: iterable of type codec instances. If
``type_codecs`` contains multiple codecs that transform a single
@ -128,7 +128,7 @@ class TypeRegistry:
type.
:param fallback_encoder: callable that accepts a single,
unencodable python value and transforms it into a type that
:mod:`bson` can encode. See `define a fallback encoder <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/custom-types/type-codecs/#define-a-fallback-encoder>`_
:mod:`bson` can encode. See :ref:`fallback-encoder-callable`
documentation for an example.
"""
@ -160,16 +160,6 @@ class TypeRegistry:
f"Expected an instance of {TypeEncoder.__name__}, {TypeDecoder.__name__}, or {TypeCodec.__name__}, got {codec!r} instead"
)
@property
def codecs(self) -> list[TypeEncoder | TypeDecoder | TypeCodec]:
"""The list of type codecs in this registry."""
return self.__type_codecs
@property
def fallback_encoder(self) -> Optional[_Fallback]:
"""The fallback encoder in this registry."""
return self._fallback_encoder
def _validate_type_encoder(self, codec: _Codec) -> None:
from bson import _BUILT_IN_TYPES
@ -273,6 +263,9 @@ if TYPE_CHECKING:
def _arguments_repr(self) -> str:
...
def _options_dict(self) -> dict[Any, Any]:
...
# NamedTuple API
@classmethod
def _make(cls, obj: Iterable[Any]) -> CodecOptions[_DocumentType]:
@ -324,10 +317,10 @@ else:
>>> doc._id
ObjectId('5b3016359110ea14e8c58b93')
See `Dates and Times <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#dates-and-times>`_ for examples using the `tz_aware` and
See :doc:`/examples/datetimes` for examples using the `tz_aware` and
`tzinfo` options.
See `UUID <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#universally-unique-ids--uuids->`_ for examples using the `uuid_representation`
See :doc:`/examples/uuid` for examples using the `uuid_representation`
option.
:param document_class: BSON documents returned in queries will be decoded
@ -341,7 +334,7 @@ else:
:data:`~bson.binary.UuidRepresentation.UNSPECIFIED`. New
applications should consider setting this to
:data:`~bson.binary.UuidRepresentation.STANDARD` for cross language
compatibility. See `UUID representations <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/uuid/#universally-unique-ids--uuids->`_ for details.
compatibility. See :ref:`handling-uuid-data-example` for details.
:param unicode_decode_error_handler: The error handler to apply when
a Unicode-related error occurs during BSON decoding that would
otherwise raise :exc:`UnicodeDecodeError`. Valid options include
@ -408,23 +401,17 @@ else:
"uuid_representation must be a value from bson.binary.UuidRepresentation"
)
if not isinstance(unicode_decode_error_handler, str):
raise ValueError(
f"unicode_decode_error_handler must be a string, not {type(unicode_decode_error_handler)}"
)
raise ValueError("unicode_decode_error_handler must be a string")
if tzinfo is not None:
if not isinstance(tzinfo, datetime.tzinfo):
raise TypeError(
f"tzinfo must be an instance of datetime.tzinfo, not {type(tzinfo)}"
)
raise TypeError("tzinfo must be an instance of datetime.tzinfo")
if not tz_aware:
raise ValueError("cannot specify tzinfo without also setting tz_aware=True")
type_registry = type_registry or TypeRegistry()
if not isinstance(type_registry, TypeRegistry):
raise TypeError(
f"type_registry must be an instance of TypeRegistry, not {type(type_registry)}"
)
raise TypeError("type_registry must be an instance of TypeRegistry")
return tuple.__new__(
cls,
@ -463,6 +450,19 @@ else:
)
)
def _options_dict(self) -> dict[str, Any]:
"""Dictionary of the arguments used to create this object."""
# TODO: PYTHON-2442 use _asdict() instead
return {
"document_class": self.document_class,
"tz_aware": self.tz_aware,
"uuid_representation": self.uuid_representation,
"unicode_decode_error_handler": self.unicode_decode_error_handler,
"tzinfo": self.tzinfo,
"type_registry": self.type_registry,
"datetime_conversion": self.datetime_conversion,
}
def __repr__(self) -> str:
return f"{self.__class__.__name__}({self._arguments_repr()})"
@ -478,7 +478,7 @@ else:
.. versionadded:: 3.5
"""
opts = self._asdict()
opts = self._options_dict()
opts.update(kwargs)
return CodecOptions(**opts)

View File

@ -20,6 +20,7 @@ from __future__ import annotations
import calendar
import datetime
import functools
from typing import Any, Union, cast
from bson.codec_options import DEFAULT_CODEC_OPTIONS, CodecOptions, DatetimeConversion
@ -31,7 +32,7 @@ EPOCH_NAIVE = EPOCH_AWARE.replace(tzinfo=None)
_DATETIME_ERROR_SUGGESTION = (
"(Consider Using CodecOptions(datetime_conversion=DATETIME_AUTO)"
" or MongoClient(datetime_conversion='DATETIME_AUTO'))."
" See: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes"
" See: https://pymongo.readthedocs.io/en/stable/examples/datetimes.html#handling-out-of-range-datetimes"
)
@ -51,7 +52,7 @@ class DatetimeMS:
To decode UTC datetimes as a ``DatetimeMS``, `datetime_conversion` in
:class:`~bson.codec_options.CodecOptions` must be set to 'datetime_ms' or
'datetime_auto'. See `handling out of range datetimes <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes>`_ for
'datetime_auto'. See :ref:`handling-out-of-range-datetimes` for
details.
:param value: An instance of :class:`datetime.datetime` to be
@ -113,36 +114,17 @@ class DatetimeMS:
return self._value
def _datetime_to_millis(dtm: datetime.datetime) -> int:
"""Convert datetime to milliseconds since epoch UTC."""
if dtm.utcoffset() is not None:
dtm = dtm - dtm.utcoffset() # type: ignore
return int(calendar.timegm(dtm.timetuple()) * 1000 + dtm.microsecond // 1000)
# Inclusive and exclusive min and max for timezones.
# Timezones are hashed by their offset, which is a timedelta
# and therefore there are more than 24 possible timezones.
@functools.lru_cache(maxsize=None)
def _min_datetime_ms(tz: datetime.timezone = datetime.timezone.utc) -> int:
return _datetime_to_millis(datetime.datetime.min.replace(tzinfo=tz))
_MIN_UTC = datetime.datetime.min.replace(tzinfo=utc)
_MAX_UTC = datetime.datetime.max.replace(tzinfo=utc)
_MIN_UTC_MS = _datetime_to_millis(_MIN_UTC)
_MAX_UTC_MS = _datetime_to_millis(_MAX_UTC)
# Inclusive min and max for timezones.
def _min_datetime_ms(tz: datetime.tzinfo = utc) -> int:
delta = tz.utcoffset(_MIN_UTC)
if delta is not None:
offset_millis = (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000
else:
offset_millis = 0
return max(_MIN_UTC_MS, _MIN_UTC_MS - offset_millis)
def _max_datetime_ms(tz: datetime.tzinfo = utc) -> int:
delta = tz.utcoffset(_MAX_UTC)
if delta is not None:
offset_millis = (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000
else:
offset_millis = 0
return min(_MAX_UTC_MS, _MAX_UTC_MS - offset_millis)
@functools.lru_cache(maxsize=None)
def _max_datetime_ms(tz: datetime.timezone = datetime.timezone.utc) -> int:
return _datetime_to_millis(datetime.datetime.max.replace(tzinfo=tz))
def _millis_to_datetime(
@ -154,7 +136,7 @@ def _millis_to_datetime(
or opts.datetime_conversion == DatetimeConversion.DATETIME_CLAMP
or opts.datetime_conversion == DatetimeConversion.DATETIME_AUTO
):
tz = opts.tzinfo or utc
tz = opts.tzinfo or datetime.timezone.utc
if opts.datetime_conversion == DatetimeConversion.DATETIME_CLAMP:
millis = max(_min_datetime_ms(tz), min(millis, _max_datetime_ms(tz)))
elif opts.datetime_conversion == DatetimeConversion.DATETIME_AUTO:
@ -180,3 +162,10 @@ def _millis_to_datetime(
return DatetimeMS(millis)
else:
raise ValueError("datetime_conversion must be an element of DatetimeConversion")
def _datetime_to_millis(dtm: datetime.datetime) -> int:
"""Convert datetime to milliseconds since epoch UTC."""
if dtm.utcoffset() is not None:
dtm = dtm - dtm.utcoffset() # type: ignore
return int(calendar.timegm(dtm.timetuple()) * 1000 + dtm.microsecond // 1000)

View File

@ -56,9 +56,9 @@ class DBRef:
.. seealso:: The MongoDB documentation on `dbrefs <https://dochub.mongodb.org/core/dbrefs>`_.
"""
if not isinstance(collection, str):
raise TypeError(f"collection must be an instance of str, not {type(collection)}")
raise TypeError("collection must be an instance of str")
if database is not None and not isinstance(database, str):
raise TypeError(f"database must be an instance of str, not {type(database)}")
raise TypeError("database must be an instance of str")
self.__collection = collection
self.__id = id

View File

@ -20,11 +20,8 @@ from __future__ import annotations
import decimal
import struct
from decimal import Decimal
from typing import Any, Sequence, Tuple, Type, Union
from bson.codec_options import TypeDecoder, TypeEncoder
_PACK_64 = struct.Struct("<Q").pack
_UNPACK_64 = struct.Struct("<Q").unpack
@ -61,42 +58,6 @@ _DEC128_CTX = decimal.Context(**_CTX_OPTIONS.copy()) # type: ignore
_VALUE_OPTIONS = Union[decimal.Decimal, float, str, Tuple[int, Sequence[int], int]]
class DecimalEncoder(TypeEncoder):
"""Converts Python :class:`decimal.Decimal` to BSON :class:`Decimal128`.
For example::
opts = CodecOptions(type_registry=TypeRegistry([DecimalEncoder()]))
bson.encode({"d": decimal.Decimal('1.0')}, codec_options=opts)
.. versionadded:: 4.15
"""
@property
def python_type(self) -> Type[Decimal]:
return Decimal
def transform_python(self, value: Any) -> Decimal128:
return Decimal128(value)
class DecimalDecoder(TypeDecoder):
"""Converts BSON :class:`Decimal128` to Python :class:`decimal.Decimal`.
For example::
opts = CodecOptions(type_registry=TypeRegistry([DecimalDecoder()]))
bson.decode(data, codec_options=opts)
.. versionadded:: 4.15
"""
@property
def bson_type(self) -> Type[Decimal128]:
return Decimal128
def transform_bson(self, value: Any) -> decimal.Decimal:
return value.to_decimal()
def create_decimal128_context() -> decimal.Context:
"""Returns an instance of :class:`decimal.Context` appropriate
for working with IEEE-754 128-bit decimal floating point values.
@ -262,7 +223,7 @@ class Decimal128:
"from list or tuple. Must have exactly 2 "
"elements."
)
self.__high, self.__low = value
self.__high, self.__low = value # type: ignore
else:
raise TypeError(f"Cannot convert {value!r} to Decimal128")
@ -316,7 +277,7 @@ class Decimal128:
point in Binary Integer Decimal (BID) format).
"""
if not isinstance(value, bytes):
raise TypeError(f"value must be an instance of bytes, not {type(value)}")
raise TypeError("value must be an instance of bytes")
if len(value) != 16:
raise ValueError("value must be exactly 16 bytes")
return cls((_UNPACK_64(value[8:])[0], _UNPACK_64(value[:8])[0])) # type: ignore

View File

@ -15,8 +15,6 @@
"""Exceptions raised by the BSON package."""
from __future__ import annotations
from typing import Any, Optional
class BSONError(Exception):
"""Base class for all BSON exceptions."""
@ -33,17 +31,6 @@ class InvalidStringData(BSONError):
class InvalidDocument(BSONError):
"""Raised when trying to create a BSON object from an invalid document."""
def __init__(self, message: str, document: Optional[Any] = None) -> None:
super().__init__(message)
self._document = document
@property
def document(self) -> Any:
"""The invalid document that caused the error.
..versionadded:: 4.16"""
return self._document
class InvalidId(BSONError):
"""Raised when trying to create an ObjectId from invalid data."""

View File

@ -22,7 +22,7 @@ is emitted and parsed, with the default being the Relaxed Extended JSON format.
when :const:`CANONICAL_JSON_OPTIONS` or :const:`LEGACY_JSON_OPTIONS` is
provided, respectively.
.. _Extended JSON: https://github.com/mongodb/specifications/blob/master/source/extended-json/extended-json.md
.. _Extended JSON: https://github.com/mongodb/specifications/blob/master/source/extended-json.rst
Example usage (deserialization):
@ -125,10 +125,10 @@ from bson.binary import ALL_UUID_SUBTYPES, UUID_SUBTYPE, Binary, UuidRepresentat
from bson.code import Code
from bson.codec_options import CodecOptions, DatetimeConversion
from bson.datetime_ms import (
_MAX_UTC_MS,
EPOCH_AWARE,
DatetimeMS,
_datetime_to_millis,
_max_datetime_ms,
_millis_to_datetime,
)
from bson.dbref import DBRef
@ -281,7 +281,7 @@ class JSONOptions(_BASE_CLASS):
return DatetimeMS objects when the underlying datetime is
out-of-range and 'datetime_clamp' to clamp to the minimum and
maximum possible datetimes. Defaults to 'datetime'. See
`handling out of range datetimes <https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes>`_ for details.
:ref:`handling-out-of-range-datetimes` for details.
:param args: arguments to :class:`~bson.codec_options.CodecOptions`
:param kwargs: arguments to :class:`~bson.codec_options.CodecOptions`
@ -324,7 +324,7 @@ class JSONOptions(_BASE_CLASS):
"JSONOptions.datetime_representation must be one of LEGACY, "
"NUMBERLONG, or ISO8601 from DatetimeRepresentation."
)
self = cast(JSONOptions, super().__new__(cls, *args, **kwargs))
self = cast(JSONOptions, super().__new__(cls, *args, **kwargs)) # type:ignore[arg-type]
if json_mode not in (JSONMode.LEGACY, JSONMode.RELAXED, JSONMode.CANONICAL):
raise ValueError(
"JSONOptions.json_mode must be one of LEGACY, RELAXED, "
@ -382,6 +382,19 @@ class JSONOptions(_BASE_CLASS):
)
)
def _options_dict(self) -> dict[Any, Any]:
# TODO: PYTHON-2442 use _asdict() instead
options_dict = super()._options_dict()
options_dict.update(
{
"strict_number_long": self.strict_number_long,
"datetime_representation": self.datetime_representation,
"strict_uuid": self.strict_uuid,
"json_mode": self.json_mode,
}
)
return options_dict
def with_options(self, **kwargs: Any) -> JSONOptions:
"""
Make a copy of this JSONOptions, overriding some options::
@ -395,7 +408,7 @@ class JSONOptions(_BASE_CLASS):
.. versionadded:: 3.12
"""
opts = self._asdict()
opts = self._options_dict()
for opt in ("strict_number_long", "datetime_representation", "strict_uuid", "json_mode"):
opts[opt] = kwargs.get(opt, getattr(self, opt))
opts.update(kwargs)
@ -604,28 +617,25 @@ def _parse_canonical_datetime(
raise TypeError(f"Bad $date, extra field(s): {doc}")
# mongoexport 2.6 and newer
if isinstance(dtm, str):
try:
# Parse offset
if dtm[-1] == "Z":
dt = dtm[:-1]
offset = "Z"
elif dtm[-6] in ("+", "-") and dtm[-3] == ":":
# (+|-)HH:MM
dt = dtm[:-6]
offset = dtm[-6:]
elif dtm[-5] in ("+", "-"):
# (+|-)HHMM
dt = dtm[:-5]
offset = dtm[-5:]
elif dtm[-3] in ("+", "-"):
# (+|-)HH
dt = dtm[:-3]
offset = dtm[-3:]
else:
dt = dtm
offset = ""
except IndexError as exc:
raise ValueError(f"time data {dtm!r} does not match ISO-8601 datetime format") from exc
# Parse offset
if dtm[-1] == "Z":
dt = dtm[:-1]
offset = "Z"
elif dtm[-6] in ("+", "-") and dtm[-3] == ":":
# (+|-)HH:MM
dt = dtm[:-6]
offset = dtm[-6:]
elif dtm[-5] in ("+", "-"):
# (+|-)HHMM
dt = dtm[:-5]
offset = dtm[-5:]
elif dtm[-3] in ("+", "-"):
# (+|-)HH
dt = dtm[:-3]
offset = dtm[-3:]
else:
dt = dtm
offset = ""
# Parse the optional factional seconds portion.
dot_index = dt.rfind(".")
@ -831,18 +841,18 @@ def _encode_binary(data: bytes, subtype: int, json_options: JSONOptions) -> Any:
return {"$binary": {"base64": base64.b64encode(data).decode(), "subType": "%02x" % subtype}}
def _encode_datetimems(obj: Any, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_datetimems(obj: Any, json_options: JSONOptions) -> dict:
if (
json_options.datetime_representation == DatetimeRepresentation.ISO8601
and 0 <= int(obj) <= _MAX_UTC_MS
and 0 <= int(obj) <= _max_datetime_ms()
):
return _encode_datetime(obj.as_datetime(), json_options)
elif json_options.datetime_representation == DatetimeRepresentation.LEGACY:
return {"$date": int(obj)}
return {"$date": str(int(obj))}
return {"$date": {"$numberLong": str(int(obj))}}
def _encode_code(obj: Code, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_code(obj: Code, json_options: JSONOptions) -> dict:
if obj.scope is None:
return {"$code": str(obj)}
else:
@ -860,7 +870,7 @@ def _encode_noop(obj: Any, dummy0: Any) -> Any:
return obj
def _encode_regex(obj: Any, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_regex(obj: Any, json_options: JSONOptions) -> dict:
flags = ""
if obj.flags & re.IGNORECASE:
flags += "i"
@ -905,7 +915,7 @@ def _encode_float(obj: float, json_options: JSONOptions) -> Any:
return obj
def _encode_datetime(obj: datetime.datetime, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_datetime(obj: datetime.datetime, json_options: JSONOptions) -> dict:
if json_options.datetime_representation == DatetimeRepresentation.ISO8601:
if not obj.tzinfo:
obj = obj.replace(tzinfo=utc)
@ -928,15 +938,15 @@ def _encode_datetime(obj: datetime.datetime, json_options: JSONOptions) -> dict:
return {"$date": {"$numberLong": str(millis)}}
def _encode_bytes(obj: bytes, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_bytes(obj: bytes, json_options: JSONOptions) -> dict:
return _encode_binary(obj, 0, json_options)
def _encode_binary_obj(obj: Binary, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_binary_obj(obj: Binary, json_options: JSONOptions) -> dict:
return _encode_binary(obj, obj.subtype, json_options)
def _encode_uuid(obj: uuid.UUID, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_uuid(obj: uuid.UUID, json_options: JSONOptions) -> dict:
if json_options.strict_uuid:
binval = Binary.from_uuid(obj, uuid_representation=json_options.uuid_representation)
return _encode_binary(binval, binval.subtype, json_options)
@ -944,27 +954,27 @@ def _encode_uuid(obj: uuid.UUID, json_options: JSONOptions) -> dict: # type: ig
return {"$uuid": obj.hex}
def _encode_objectid(obj: ObjectId, dummy0: Any) -> dict: # type: ignore[type-arg]
def _encode_objectid(obj: ObjectId, dummy0: Any) -> dict:
return {"$oid": str(obj)}
def _encode_timestamp(obj: Timestamp, dummy0: Any) -> dict: # type: ignore[type-arg]
def _encode_timestamp(obj: Timestamp, dummy0: Any) -> dict:
return {"$timestamp": {"t": obj.time, "i": obj.inc}}
def _encode_decimal128(obj: Timestamp, dummy0: Any) -> dict: # type: ignore[type-arg]
def _encode_decimal128(obj: Timestamp, dummy0: Any) -> dict:
return {"$numberDecimal": str(obj)}
def _encode_dbref(obj: DBRef, json_options: JSONOptions) -> dict: # type: ignore[type-arg]
def _encode_dbref(obj: DBRef, json_options: JSONOptions) -> dict:
return _json_convert(obj.as_doc(), json_options=json_options)
def _encode_minkey(dummy0: Any, dummy1: Any) -> dict: # type: ignore[type-arg]
def _encode_minkey(dummy0: Any, dummy1: Any) -> dict:
return {"$minKey": 1}
def _encode_maxkey(dummy0: Any, dummy1: Any) -> dict: # type: ignore[type-arg]
def _encode_maxkey(dummy0: Any, dummy1: Any) -> dict:
return {"$maxKey": 1}
@ -972,7 +982,7 @@ def _encode_maxkey(dummy0: Any, dummy1: Any) -> dict: # type: ignore[type-arg]
# Each encoder function's signature is:
# - obj: a Python data type, e.g. a Python int for _encode_int
# - json_options: a JSONOptions
_ENCODERS: dict[Type, Callable[[Any, JSONOptions], Any]] = { # type: ignore[type-arg]
_ENCODERS: dict[Type, Callable[[Any, JSONOptions], Any]] = {
bool: _encode_noop,
bytes: _encode_bytes,
datetime.datetime: _encode_datetime,
@ -1043,7 +1053,7 @@ def _get_datetime_size(obj: datetime.datetime) -> int:
return 5 + len(str(obj.time()))
def _get_regex_size(obj: Regex) -> int: # type: ignore[type-arg]
def _get_regex_size(obj: Regex) -> int:
return 18 + len(obj.pattern)

Some files were not shown because too many files have changed in this diff Show More