PYTHON-4976 Replace hatch with uv as our python environment and workfow tool (#2068)

This commit is contained in:
Steven Silvester 2025-01-22 08:48:17 -06:00 committed by GitHub
parent f1af917894
commit cfe7784db9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 2233 additions and 174 deletions

View File

@ -37,8 +37,7 @@ export PIP_QUIET=1 # Quiet by default
export PIP_PREFER_BINARY=1 # Prefer binary dists by default
set +x
python -c "import sys; sys.exit(sys.prefix == sys.base_prefix)" || (echo "Not inside a virtual env!"; exit 1)
PYTHON_IMPL=$(python -c "import platform; print(platform.python_implementation())")
PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())")
# Try to source local Drivers Secrets
if [ -f ./secrets-export.sh ]; then
@ -48,9 +47,13 @@ else
echo "Not sourcing secrets"
fi
# Ensure C extensions have compiled.
# Start compiling the args we'll pass to uv.
# Run in an isolated environment so as not to pollute the base venv.
UV_ARGS=("--isolated --extra test")
# Ensure C extensions if applicable.
if [ -z "${NO_EXT:-}" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
python tools/fail_if_no_c.py
uv run tools/fail_if_no_c.py
fi
if [ "$AUTH" != "noauth" ]; then
@ -77,7 +80,7 @@ if [ "$AUTH" != "noauth" ]; then
fi
if [ -n "$TEST_ENTERPRISE_AUTH" ]; then
python -m pip install '.[gssapi]'
UV_ARGS+=("--extra gssapi")
if [ "Windows_NT" = "$OS" ]; then
echo "Setting GSSAPI_PASS"
export GSSAPI_PASS=${SASL_PASS}
@ -118,24 +121,26 @@ if [ "$SSL" != "nossl" ]; then
fi
if [ "$COMPRESSORS" = "snappy" ]; then
python -m pip install '.[snappy]'
UV_ARGS+=("--extra snappy")
elif [ "$COMPRESSORS" = "zstd" ]; then
python -m pip install zstandard
UV_ARGS+=("--extra zstandard")
fi
# PyOpenSSL test setup.
if [ -n "$TEST_PYOPENSSL" ]; then
python -m pip install '.[ocsp]'
UV_ARGS+=("--extra ocsp")
fi
if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
# Check for libmongocrypt checkout.
# Check for libmongocrypt download.
if [ ! -d "libmongocrypt" ]; then
echo "Run encryption setup first!"
exit 1
fi
python -m pip install '.[encryption]'
UV_ARGS+=("--extra encryption")
# TODO: Test with 'pip install pymongocrypt'
UV_ARGS+=("--group pymongocrypt_source")
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
BASE=$(pwd)/libmongocrypt/nocrypto
@ -155,21 +160,17 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE
exit 1
fi
export PYMONGOCRYPT_LIB
# TODO: Test with 'pip install pymongocrypt'
if [ ! -d "libmongocrypt_git" ]; then
git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
fi
python -m pip install -U setuptools
python -m pip install ./libmongocrypt_git/bindings/python
python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
# PATH is updated by PREPARE_SHELL for access to mongocryptd.
# Ensure pymongocrypt is working properly.
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
# PATH is updated by configure-env.sh for access to mongocryptd.
fi
if [ -n "$TEST_ENCRYPTION" ]; then
if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then
python -m pip install '.[ocsp]'
UV_ARGS+=("--extra ocsp")
fi
if [ -n "$TEST_CRYPT_SHARED" ]; then
@ -214,22 +215,22 @@ if [ -n "$TEST_ATLAS" ]; then
fi
if [ -n "$TEST_OCSP" ]; then
python -m pip install ".[ocsp]"
UV_ARGS+=("--extra ocsp")
TEST_SUITES="ocsp"
fi
if [ -n "$TEST_AUTH_AWS" ]; then
python -m pip install ".[aws]"
UV_ARGS+=("--extra aws")
TEST_SUITES="auth_aws"
fi
if [ -n "$TEST_AUTH_OIDC" ]; then
python -m pip install ".[aws]"
UV_ARGS+=("--extra aws")
TEST_SUITES="auth_oidc"
fi
if [ -n "$PERF_TEST" ]; then
python -m pip install simplejson
UV_ARGS+=("--group perf")
start_time=$(date +%s)
TEST_SUITES="perf"
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
@ -237,8 +238,8 @@ if [ -n "$PERF_TEST" ]; then
TEST_ARGS="test/performance/perf_test.py $TEST_ARGS"
fi
echo "Running $AUTH tests over $SSL with python $(which python)"
python -c 'import sys; print(sys.version)'
echo "Running $AUTH tests over $SSL with python $(uv python find)"
uv run python -c 'import sys; print(sys.version)'
# Run the tests, and store the results in Evergreen compatible XUnit XML
@ -249,27 +250,30 @@ python -c 'import sys; print(sys.version)'
if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
# Keep in sync with combine-coverage.sh.
# coverage >=5 is needed for relative_files=true.
python -m pip install pytest-cov "coverage>=5,<=7.5"
UV_ARGS+=("--group coverage")
TEST_ARGS="$TEST_ARGS --cov"
fi
if [ -n "$GREEN_FRAMEWORK" ]; then
python -m pip install $GREEN_FRAMEWORK
UV_ARGS+=("--group $GREEN_FRAMEWORK")
fi
# Show the installed packages
PIP_QUIET=0 python -m pip list
# shellcheck disable=SC2048
PIP_QUIET=0 uv run ${UV_ARGS[*]} --with pip pip list
if [ -z "$GREEN_FRAMEWORK" ]; then
# Use --capture=tee-sys so pytest prints test output inline:
# https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
if [ -z "$TEST_SUITES" ]; then
python -m pytest -v --capture=tee-sys --durations=5 $TEST_ARGS
else
python -m pytest -v --capture=tee-sys --durations=5 -m $TEST_SUITES $TEST_ARGS
PYTEST_ARGS="-v --capture=tee-sys --durations=5 $TEST_ARGS"
if [ -n "$TEST_SUITES" ]; then
PYTEST_ARGS="-m $TEST_SUITES $PYTEST_ARGS"
fi
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} pytest $PYTEST_ARGS
else
python green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
# shellcheck disable=SC2048
uv run ${UV_ARGS[*]} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
fi
# Handle perf test post actions.

View File

@ -14,14 +14,16 @@ fi
PROJECT_DIRECTORY="$(pwd)"
DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
HATCH_CONFIG=$PROJECT_DIRECTORY/hatch_config.toml
UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
CARGO_HOME=$(cygpath -m $CARGO_HOME)
HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
fi
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
@ -62,7 +64,9 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
export CARGO_HOME="$CARGO_HOME"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export HATCH_CONFIG="$HATCH_CONFIG"
export UV_TOOL_DIR="$UV_TOOL_DIR"
export UV_CACHE_DIR="$UV_CACHE_DIR"
export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH"
# shellcheck disable=SC2154
export PROJECT="${project:-mongo-python-driver}"

View File

@ -6,7 +6,7 @@
# ]
# ///
# Note: Run this file with `hatch run`, `pipx run`, or `uv run`.
# Note: Run this file with `pipx run`, or `uv run`.
from __future__ import annotations
import sys

View File

@ -40,3 +40,16 @@ if ! command -v just 2>/dev/null; then
fi
echo "Installing just... done."
fi
# Install uv.
if ! command -v uv 2>/dev/null; then
echo "Installing uv..."
# On most systems we can install directly.
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {
_pip_install uv uv
}
if ! command -v uv 2>/dev/null; then
export PATH="$PATH:$_BIN_DIR"
fi
echo "Installing uv... done."
fi

View File

@ -6,10 +6,14 @@ HERE=$(dirname ${BASH_SOURCE:-$0})
pushd "$(dirname "$(dirname $HERE)")" > /dev/null
# Source the env file to pick up common variables.
if [ -f $HERE/scripts/env.sh ]; then
source $HERE/scripts/env.sh
if [ -f $HERE/env.sh ]; then
source $HERE/env.sh
fi
# Ensure dependencies are installed.
. $HERE/install-dependencies.sh
# Set the location of the python bin dir.
if [ "Windows_NT" = "${OS:-}" ]; then
BIN_DIR=.venv/Scripts
@ -17,8 +21,6 @@ else
BIN_DIR=.venv/bin
fi
. $HERE/install-dependencies.sh
# Ensure there is a python venv.
if [ ! -d $BIN_DIR ]; then
. .evergreen/utils.sh
@ -26,49 +28,15 @@ if [ ! -d $BIN_DIR ]; then
if [ -z "${PYTHON_BINARY:-}" ]; then
PYTHON_BINARY=$(find_python3)
fi
echo "Creating virtual environment..."
createvirtualenv "$PYTHON_BINARY" .venv
echo "Creating virtual environment... done."
export UV_PYTHON=${PYTHON_BINARY}
echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
fi
echo "Using python $UV_PYTHON"
uv sync
uv run --with pip pip install -e .
echo "Setting up python environment... done."
# Activate the virtual env.
. $BIN_DIR/activate
# Ensure there is a local hatch.
if [ ! -f $BIN_DIR/hatch ]; then
echo "Installing hatch..."
python -m pip install hatch || {
# CARGO_HOME is defined in configure-env.sh
export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/}
export RUSTUP_HOME="${CARGO_HOME}/.rustup"
${DRIVERS_TOOLS}/.evergreen/install-rust.sh
source "${CARGO_HOME}/env"
python -m pip install hatch
}
echo "Installing hatch... done."
# Ensure there is a pre-commit hook if there is a git checkout.
if [ -d .git ] && [ ! -f .git/hooks/pre-commit ]; then
uv run pre-commit install
fi
# Ensure hatch does not write to user or global locations.
HATCH_CONFIG=${HATCH_CONFIG:-hatch_config.toml}
if [ ! -f ${HATCH_CONFIG} ]; then
touch hatch_config.toml
hatch config restore
hatch config set dirs.data "$(pwd)/.hatch/data"
hatch config set dirs.cache "$(pwd)/.hatch/cache"
fi
# Ensure there is a local pre-commit if there is a git checkout.
if [ -d .git ]; then
if [ ! -f $BIN_DIR/pre-commit ]; then
python -m pip install pre-commit
fi
# Ensure the pre-commit hook is installed.
if [ ! -f .git/hooks/pre-commit ]; then
pre-commit install
fi
fi
# Install pymongo and its test deps.
python -m pip install ".[test]"

View File

@ -7,4 +7,4 @@ if [ -z "${DRIVERS_TOOLS}" ]; then
fi
bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz mongocryptd.pid
rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid

3
.gitignore vendored
View File

@ -22,10 +22,9 @@ venv/
secrets-export.sh
libmongocrypt.tar.gz
libmongocrypt/
libmongocrypt_git/
hatch_config.toml
.venv
expansion.yml
*expansions.yml
.evergreen/scripts/env.sh
# Lambda temp files

View File

@ -16,7 +16,7 @@ be of interest or that has already been addressed.
## Supported Interpreters
PyMongo supports CPython 3.9+ and PyPy3.9+. Language features not
PyMongo supports CPython 3.9+ and PyPy3.10+. Language features not
supported by all interpreters can not be used.
## Style Guide
@ -28,7 +28,7 @@ including 4 space indents and 79 character line limits.
- Avoid backward breaking changes if at all possible.
- Write inline documentation for new classes and methods.
- We use [hatch](https://hatch.pypa.io/dev/) for python environment management and packaging.
- We use [uv](https://docs.astral.sh/uv/) for python environment management and packaging.
- We use [just](https://just.systems/man/en/) as our task runner.
- Write tests and make sure they pass (make sure you have a mongod
running on the default port, then execute `just test` from the cmd
@ -194,7 +194,7 @@ the pages will re-render and the browser will automatically refresh.
## Running Tests Locally
- Ensure you have started the appropriate Mongo Server(s).
- Run `just install` to set up `hatch` in a local virtual environment, or you can manually
- Run `just install` to set a local virtual environment, or you can manually
create a virtual environment and run `pytest` directly. If you want to use a specific
version of Python, remove the `.venv` folder and set `PYTHON_BINARY` before running `just install`.
- Run `just test` or `pytest` to run all of the tests.

View File

@ -152,11 +152,6 @@ command:
python -m pip install "pymongo[gssapi,aws,ocsp,snappy,zstd,encryption]"
```
Additional dependencies are:
- (to generate documentation or run tests)
[hatch](https://hatch.pypa.io/dev/)
## Examples
Here's a basic example (for more see the *examples* section of the
@ -201,8 +196,7 @@ ObjectId('4aba160ee23f6b543e000002')
Documentation is available at
[pymongo.readthedocs.io](https://pymongo.readthedocs.io/en/stable/).
Documentation can be generated by running **pip install hatch; hatch run doc:build**. Generated
documentation can be found in the `doc/build/html/` directory.
See the [contributing guide](./CONTRIBUTING.md#documentation) for how to build the documentation.
## Learning Resources
@ -213,10 +207,11 @@ Center](https://www.mongodb.com/developer/languages/python/).
## Testing
The easiest way to run the tests is to run *hatch run test:test** in the root
of the distribution. For example,
The easiest way to run the tests is to run the following from the repository root.
```bash
pip install hatch
hatch run test:test
pip install -e ".[test]"
pytest
```
For more advanced testing scenarios, see the [contributing guide](./CONTRIBUTING.md#running-tests-locally).

View File

@ -106,13 +106,8 @@ About This Documentation
This documentation is generated using the `Sphinx
<https://www.sphinx-doc.org/en/master/>`_ documentation generator. The source files
for the documentation are located in the *doc/* directory of the
**PyMongo** distribution. To generate the docs locally run the
following command from the root directory of the **PyMongo** source:
.. code-block:: bash
$ pip install hatch
$ hatch run doc:build
**PyMongo** distribution. See the PyMongo `contributing guide <https://github.com/mongodb/mongo-python-driver/blob/master/CONTRIBUTING.md>`_
for instructions on the building the docs from source.
Indices and tables
------------------

View File

@ -1,39 +0,0 @@
# See https://hatch.pypa.io/dev/config/environment/overview/
[envs.doc]
features = ["docs"]
[envs.doc.scripts]
build = "sphinx-build -W -b html doc ./doc/_build/html"
serve = "sphinx-autobuild -W -b html doc --watch ./pymongo --watch ./bson --watch ./gridfs ./doc/_build/serve"
linkcheck = "sphinx-build -E -b linkcheck doc ./doc/_build/linkcheck"
[envs.doctest]
features = ["docs","test"]
[envs.doctest.scripts]
test = "sphinx-build -E -b doctest doc ./doc/_build/doctest"
[envs.typing]
pre-install-commands = [
"pip install -q -r requirements/typing.txt",
]
[envs.typing.scripts]
check-mypy = [
"mypy --install-types --non-interactive bson gridfs tools pymongo",
"mypy --install-types --non-interactive --config-file mypy_test.ini test",
"mypy --install-types --non-interactive test/test_typing.py test/test_typing_strict.py"
]
check-pyright = ["rm -f pyrightconfig.json", "pyright test/test_typing.py test/test_typing_strict.py"]
check-strict-pyright = [
"echo '{{\"strict\": [\"tests/test_typing_strict.py\"]}}' > pyrightconfig.json",
"pyright test/test_typing_strict.py",
"rm -f pyrightconfig.json"
]
check = ["check-mypy", "check-pyright", "check-strict-pyright"]
[envs.test]
features = ["test"]
[envs.test.scripts]
test = "pytest -v --durations=5 --maxfail=10 {args}"
test-eg = "bash ./.evergreen/run-tests.sh {args}"
test-async = "pytest -v --durations=5 --maxfail=10 -m default_async {args}"
test-mockupdb = ["pip install -U git+https://github.com/mongodb-labs/mongo-mockup-db@master", "test -m mockupdb"]

View File

@ -3,10 +3,12 @@ set shell := ["bash", "-c"]
set dotenv-load
set dotenv-filename := "./.evergreen/scripts/env.sh"
# Handle cross-platform paths to local python cli tools.
python_bin_dir := if os_family() == "windows" { "./.venv/Scripts" } else { "./.venv/bin" }
hatch_bin := python_bin_dir + "/hatch"
pre_commit_bin := python_bin_dir + "/pre-commit"
# Commonly used command segments.
uv_run := "uv run --isolated "
typing_run := uv_run + "--group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd"
docs_run := uv_run + "--extra docs"
doc_build := "./doc/_build"
mypy_args := "--install-types --non-interactive"
# Make the default recipe private so it doesn't show up in the list.
[private]
@ -18,47 +20,55 @@ install:
[group('docs')]
docs:
{{hatch_bin}} run doc:build
{{docs_run}} sphinx-build -W -b html doc {{doc_build}}/html
[group('docs')]
docs-serve:
{{hatch_bin}} run doc:serve
{{docs_run}} sphinx-autobuild -W -b html doc --watch ./pymongo --watch ./bson --watch ./gridfs {{doc_build}}/serve
[group('docs')]
docs-linkcheck:
{{hatch_bin}} run doc:linkcheck
{{docs_run}} sphinx-build -E -b linkcheck doc {{doc_build}}/linkcheck
[group('docs')]
docs-test:
{{hatch_bin}} run doctest:test
{{docs_run}} --extra test sphinx-build -E -b doctest doc {{doc_build}}/doctest
[group('typing')]
typing:
{{hatch_bin}} run typing:check
just typing-mypy
just typing-pyright
[group('typing')]
typing-mypy:
{{hatch_bin}} run typing:mypy
{{typing_run}} mypy {{mypy_args}} bson gridfs tools pymongo
{{typing_run}} mypy {{mypy_args}} --config-file mypy_test.ini test
{{typing_run}} mypy {{mypy_args}} test/test_typing.py test/test_typing_strict.py
[group('typing')]
typing-pyright:
{{typing_run}} pyright test/test_typing.py test/test_typing_strict.py
{{typing_run}} pyright -p strict_pyrightconfig.json test/test_typing_strict.py
[group('lint')]
lint:
{{pre_commit_bin}} run --all-files
{{uv_run}} pre-commit run --all-files
[group('lint')]
lint-manual:
{{pre_commit_bin}} run --all-files --hook-stage manual
{{uv_run}} pre-commit run --all-files --hook-stage manual
[group('test')]
test *args:
{{hatch_bin}} run test:test {{args}}
test *args="-v --durations=5 --maxfail=10":
{{uv_run}} --extra test pytest {{args}}
[group('test')]
test-mockupdb:
{{hatch_bin}} run test:test-mockupdb
test-mockupdb *args:
{{uv_run}} -v --extra test --group mockupdb pytest -m mockupdb {{args}}
[group('test')]
test-eg *args:
{{hatch_bin}} run test:test-eg {{args}}
bash ./.evergreen/run-tests.sh {{args}}
[group('encryption')]
setup-encryption:

View File

@ -26,7 +26,7 @@ _NO_COMPRESSION.update(_SENSITIVE_COMMANDS)
def _have_snappy() -> bool:
try:
import snappy # type:ignore[import-not-found] # noqa: F401
import snappy # type:ignore[import-untyped] # noqa: F401
return True
except ImportError:

View File

@ -45,6 +45,30 @@ Documentation = "https://www.mongodb.com/docs/languages/python/pymongo-driver/cu
Source = "https://github.com/mongodb/mongo-python-driver"
Tracker = "https://jira.mongodb.org/projects/PYTHON/issues"
[dependency-groups]
dev = [
"pre-commit>=4.0"
]
gevent = ["gevent"]
eventlet = ["eventlet"]
coverage = [
"pytest-cov",
"coverage>=5,<=7.5"
]
mockupdb = [
"mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master"
]
pymongocrypt_source = [
"pymongocrypt@git+https://github.com/mongodb/libmongocrypt@master#subdirectory=bindings/python"
]
perf = ["simplejson"]
typing = [
"mypy==1.14.1",
"pyright==1.1.392.post0",
"typing_extensions",
"pip"
]
# Used to call hatch_build.py
[tool.hatch.build.hooks.custom]

View File

@ -1,7 +0,0 @@
mypy==1.14.1
pyright==1.1.392.post0
typing_extensions
-r ./encryption.txt
-r ./ocsp.txt
-r ./zstd.txt
-r ./aws.txt

View File

@ -0,0 +1 @@
{"strict": ["tests/test_typing_strict.py"]}

2092
uv.lock generated Normal file

File diff suppressed because it is too large Load Diff