PYTHON-5615 Use uv python when python toolchain is not available (#2597)
This commit is contained in:
Steven Silvester
GitHub
parent
ad1167d01e
commit
a5f6d638b9
@ -5,19 +5,12 @@
|
||||
|
||||
set -eu
|
||||
|
||||
. .evergreen/utils.sh
|
||||
# Set up the virtual env.
|
||||
. .evergreen/scripts/setup-dev-env.sh
|
||||
uv sync --group coverage
|
||||
source .venv/bin/activate
|
||||
|
||||
if [ -z "${PYTHON_BINARY:-}" ]; then
|
||||
PYTHON_BINARY=$(find_python3)
|
||||
fi
|
||||
|
||||
createvirtualenv "$PYTHON_BINARY" covenv
|
||||
# Keep in sync with run-tests.sh
|
||||
# coverage >=5 is needed for relative_files=true.
|
||||
pip install -q "coverage[toml]>=5,<=7.5"
|
||||
|
||||
pip list
|
||||
ls -la coverage/
|
||||
|
||||
python -m coverage combine coverage/coverage.*
|
||||
python -m coverage html -d htmlcov
|
||||
coverage combine coverage/coverage.*
|
||||
coverage html -d htmlcov
|
||||
|
||||
@ -101,8 +101,8 @@ functions:
|
||||
- AUTH
|
||||
- SSL
|
||||
- ORCHESTRATION_FILE
|
||||
- PYTHON_BINARY
|
||||
- PYTHON_VERSION
|
||||
- UV_PYTHON
|
||||
- TOOLCHAIN_VERSION
|
||||
- STORAGE_ENGINE
|
||||
- REQUIRE_API_VERSION
|
||||
- DRIVERS_TOOLS
|
||||
@ -134,10 +134,10 @@ functions:
|
||||
- AWS_SECRET_ACCESS_KEY
|
||||
- AWS_SESSION_TOKEN
|
||||
- COVERAGE
|
||||
- PYTHON_BINARY
|
||||
- UV_PYTHON
|
||||
- LIBMONGOCRYPT_URL
|
||||
- MONGODB_URI
|
||||
- PYTHON_VERSION
|
||||
- TOOLCHAIN_VERSION
|
||||
- DISABLE_TEST_COMMANDS
|
||||
- GREEN_FRAMEWORK
|
||||
- NO_EXT
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -11,7 +11,7 @@ buildvariants:
|
||||
VERSION: latest
|
||||
NO_EXT: "1"
|
||||
REQUIRE_FIPS: "1"
|
||||
PYTHON_BINARY: /usr/bin/python3.11
|
||||
UV_PYTHON: /usr/bin/python3.11
|
||||
tags: []
|
||||
- name: other-hosts-rhel8-zseries-latest
|
||||
tasks:
|
||||
@ -72,7 +72,7 @@ buildvariants:
|
||||
# Aws auth tests
|
||||
- name: auth-aws-ubuntu-20
|
||||
tasks:
|
||||
- name: .auth-aws !.auth-aws-ecs
|
||||
- name: .auth-aws
|
||||
display_name: Auth AWS Ubuntu-20
|
||||
run_on:
|
||||
- ubuntu2004-small
|
||||
@ -453,14 +453,12 @@ buildvariants:
|
||||
SUB_TEST_NAME: pyopenssl
|
||||
|
||||
# Search index tests
|
||||
- name: search-index-helpers-rhel8-python3.10
|
||||
- name: search-index-helpers-rhel8
|
||||
tasks:
|
||||
- name: .search_index
|
||||
display_name: Search Index Helpers RHEL8 Python3.10
|
||||
display_name: Search Index Helpers RHEL8
|
||||
run_on:
|
||||
- rhel87-small
|
||||
expansions:
|
||||
PYTHON_BINARY: /opt/python/3.10/bin/python3
|
||||
|
||||
# Server version tests
|
||||
- name: mongodb-v4.2
|
||||
|
||||
@ -19,20 +19,14 @@ fi
|
||||
# Now we can safely enable xtrace
|
||||
set -o xtrace
|
||||
|
||||
# Install python with pip.
|
||||
PYTHON_VER="python3.10"
|
||||
# Install a c compiler.
|
||||
apt-get -qq update < /dev/null > /dev/null
|
||||
apt-get -q install -y software-properties-common
|
||||
# Use openpgp to avoid gpg key timeout.
|
||||
mkdir -p $HOME/.gnupg
|
||||
echo "keyserver keys.openpgp.org" >> $HOME/.gnupg/gpg.conf
|
||||
add-apt-repository -y 'ppa:deadsnakes/ppa'
|
||||
apt-get -qq install $PYTHON_VER $PYTHON_VER-venv build-essential $PYTHON_VER-dev -y < /dev/null > /dev/null
|
||||
apt-get -q install -y build-essential
|
||||
|
||||
export PYTHON_BINARY=$PYTHON_VER
|
||||
export SET_XTRACE_ON=1
|
||||
cd src
|
||||
rm -rf .venv
|
||||
rm -f .evergreen/scripts/test-env.sh || true
|
||||
rm -f .evergreen/scripts/env.sh || true
|
||||
bash ./.evergreen/just.sh setup-tests auth_aws ecs-remote
|
||||
bash .evergreen/just.sh run-tests
|
||||
|
||||
@ -37,6 +37,8 @@ cleanup_tests() {
|
||||
trap "cleanup_tests" SIGINT ERR
|
||||
|
||||
# Start the test runner.
|
||||
echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..."
|
||||
uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@"
|
||||
echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done."
|
||||
|
||||
cleanup_tests
|
||||
|
||||
@ -11,11 +11,10 @@ pushd $HERE/../.. >/dev/null
|
||||
BASE_SHA="$1"
|
||||
HEAD_SHA="$2"
|
||||
|
||||
. .evergreen/utils.sh
|
||||
|
||||
if [ -z "${PYTHON_BINARY:-}" ]; then
|
||||
PYTHON_BINARY=$(find_python3)
|
||||
fi
|
||||
# Set up the virtual env.
|
||||
. $HERE/setup-dev-env.sh
|
||||
uv venv --seed
|
||||
source .venv/bin/activate
|
||||
|
||||
# Use the previous commit if this was not a PR run.
|
||||
if [ "$BASE_SHA" == "$HEAD_SHA" ]; then
|
||||
@ -24,7 +23,6 @@ fi
|
||||
|
||||
function get_import_time() {
|
||||
local log_file
|
||||
createvirtualenv "$PYTHON_BINARY" import-venv
|
||||
python -m pip install -q ".[aws,encryption,gssapi,ocsp,snappy,zstd]"
|
||||
# Import once to cache modules
|
||||
python -c "import pymongo"
|
||||
|
||||
@ -356,12 +356,10 @@ def create_oidc_auth_variants():
|
||||
|
||||
def create_search_index_variants():
|
||||
host = DEFAULT_HOST
|
||||
python = CPYTHONS[0]
|
||||
return [
|
||||
create_variant(
|
||||
[".search_index"],
|
||||
get_variant_name("Search Index Helpers", host, python=python),
|
||||
python=python,
|
||||
get_variant_name("Search Index Helpers", host),
|
||||
host=host,
|
||||
)
|
||||
]
|
||||
@ -438,8 +436,7 @@ def create_aws_auth_variants():
|
||||
|
||||
for host_name in ["ubuntu20", "win64", "macos"]:
|
||||
expansions = dict()
|
||||
# PYTHON-5604 - we need to skip ECS tests for now.
|
||||
tasks = [".auth-aws !.auth-aws-ecs"]
|
||||
tasks = [".auth-aws"]
|
||||
tags = []
|
||||
if host_name == "macos":
|
||||
tasks = [".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2"]
|
||||
@ -477,7 +474,7 @@ def create_alternative_hosts_variants():
|
||||
if "fips" in host_name.lower():
|
||||
expansions["REQUIRE_FIPS"] = "1"
|
||||
# Use explicit Python 3.11 binary on the host since the default python3 is 3.9.
|
||||
expansions["PYTHON_BINARY"] = "/usr/bin/python3.11"
|
||||
expansions["UV_PYTHON"] = "/usr/bin/python3.11"
|
||||
if "amazon" in host_name.lower():
|
||||
tags.append("pr")
|
||||
variants.append(
|
||||
@ -543,7 +540,7 @@ def create_server_version_tasks():
|
||||
)
|
||||
server_func = FunctionCall(func="run server", vars=expansions)
|
||||
test_vars = expansions.copy()
|
||||
test_vars["PYTHON_VERSION"] = python
|
||||
test_vars["TOOLCHAIN_VERSION"] = python
|
||||
test_vars["TEST_NAME"] = f"default_{sync}"
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func]))
|
||||
@ -599,7 +596,7 @@ def create_test_non_standard_tasks():
|
||||
name = get_task_name("test-non-standard", python=python, **expansions)
|
||||
server_func = FunctionCall(func="run server", vars=expansions)
|
||||
test_vars = expansions.copy()
|
||||
test_vars["PYTHON_VERSION"] = python
|
||||
test_vars["TOOLCHAIN_VERSION"] = python
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func]))
|
||||
return tasks
|
||||
@ -639,7 +636,7 @@ def create_test_standard_auth_tasks():
|
||||
name = get_task_name("test-standard-auth", python=python, **expansions)
|
||||
server_func = FunctionCall(func="run server", vars=expansions)
|
||||
test_vars = expansions.copy()
|
||||
test_vars["PYTHON_VERSION"] = python
|
||||
test_vars["TOOLCHAIN_VERSION"] = python
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func]))
|
||||
return tasks
|
||||
@ -691,7 +688,7 @@ def create_standard_tasks():
|
||||
name = get_task_name("test-standard", python=python, sync=sync, **expansions)
|
||||
server_func = FunctionCall(func="run server", vars=expansions)
|
||||
test_vars = expansions.copy()
|
||||
test_vars["PYTHON_VERSION"] = python
|
||||
test_vars["TOOLCHAIN_VERSION"] = python
|
||||
test_vars["TEST_NAME"] = f"default_{sync}"
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func]))
|
||||
@ -707,7 +704,7 @@ def create_no_orchestration_tasks():
|
||||
]
|
||||
name = get_task_name("test-no-orchestration", python=python)
|
||||
assume_func = FunctionCall(func="assume ec2 role")
|
||||
test_vars = dict(PYTHON_VERSION=python)
|
||||
test_vars = dict(TOOLCHAIN_VERSION=python)
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
commands = [assume_func, test_func]
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=commands))
|
||||
@ -756,7 +753,7 @@ def create_aws_tasks():
|
||||
if "t" in python:
|
||||
tags.append("free-threaded")
|
||||
name = get_task_name(f"{base_name}-{test_type}", python=python)
|
||||
test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, PYTHON_VERSION=python)
|
||||
test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, TOOLCHAIN_VERSION=python)
|
||||
test_func = FunctionCall(func="run tests", vars=test_vars)
|
||||
funcs = [server_func, assume_func, test_func]
|
||||
tasks.append(EvgTask(name=name, tags=tags, commands=funcs))
|
||||
@ -768,7 +765,7 @@ def create_aws_tasks():
|
||||
TEST_NAME="auth_aws",
|
||||
SUB_TEST_NAME="web-identity",
|
||||
AWS_ROLE_SESSION_NAME="test",
|
||||
PYTHON_VERSION=python,
|
||||
TOOLCHAIN_VERSION=python,
|
||||
)
|
||||
if "t" in python:
|
||||
tags.append("free-threaded")
|
||||
@ -806,9 +803,11 @@ def create_mod_wsgi_tasks():
|
||||
task_name = "mod-wsgi-embedded-mode-"
|
||||
task_name += topology.replace("_", "-")
|
||||
task_name = get_task_name(task_name, python=python)
|
||||
server_vars = dict(TOPOLOGY=topology, PYTHON_VERSION=python)
|
||||
server_vars = dict(TOPOLOGY=topology, TOOLCHAIN_VERSION=python)
|
||||
server_func = FunctionCall(func="run server", vars=server_vars)
|
||||
vars = dict(TEST_NAME="mod_wsgi", SUB_TEST_NAME=test.split("-")[0], PYTHON_VERSION=python)
|
||||
vars = dict(
|
||||
TEST_NAME="mod_wsgi", SUB_TEST_NAME=test.split("-")[0], TOOLCHAIN_VERSION=python
|
||||
)
|
||||
test_func = FunctionCall(func="run tests", vars=vars)
|
||||
tags = ["mod_wsgi", "pr"]
|
||||
commands = [server_func, test_func]
|
||||
@ -830,7 +829,7 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name):
|
||||
ORCHESTRATION_FILE=file_name,
|
||||
OCSP_SERVER_TYPE=server_type,
|
||||
TEST_NAME="ocsp",
|
||||
PYTHON_VERSION=python,
|
||||
TOOLCHAIN_VERSION=python,
|
||||
VERSION=version,
|
||||
)
|
||||
test_func = FunctionCall(func="run tests", vars=vars)
|
||||
@ -864,7 +863,7 @@ def create_aws_lambda_tasks():
|
||||
def create_search_index_tasks():
|
||||
assume_func = FunctionCall(func="assume ec2 role")
|
||||
server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index"))
|
||||
vars = dict(TEST_NAME="search_index")
|
||||
vars = dict(TEST_NAME="search_index", TOOLCHAIN_VERSION=CPYTHONS[0])
|
||||
test_func = FunctionCall(func="run tests", vars=vars)
|
||||
task_name = "test-search-index-helpers"
|
||||
tags = ["search_index"]
|
||||
@ -1075,8 +1074,8 @@ def create_run_server_func():
|
||||
"AUTH",
|
||||
"SSL",
|
||||
"ORCHESTRATION_FILE",
|
||||
"PYTHON_BINARY",
|
||||
"PYTHON_VERSION",
|
||||
"UV_PYTHON",
|
||||
"TOOLCHAIN_VERSION",
|
||||
"STORAGE_ENGINE",
|
||||
"REQUIRE_API_VERSION",
|
||||
"DRIVERS_TOOLS",
|
||||
@ -1100,10 +1099,10 @@ def create_run_tests_func():
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"COVERAGE",
|
||||
"PYTHON_BINARY",
|
||||
"UV_PYTHON",
|
||||
"LIBMONGOCRYPT_URL",
|
||||
"MONGODB_URI",
|
||||
"PYTHON_VERSION",
|
||||
"TOOLCHAIN_VERSION",
|
||||
"DISABLE_TEST_COMMANDS",
|
||||
"GREEN_FRAMEWORK",
|
||||
"NO_EXT",
|
||||
|
||||
@ -133,43 +133,17 @@ def create_variant(
|
||||
*,
|
||||
version: str | None = None,
|
||||
host: Host | str | None = None,
|
||||
python: str | None = None,
|
||||
expansions: dict | None = None,
|
||||
**kwargs: Any,
|
||||
) -> BuildVariant:
|
||||
expansions = expansions and expansions.copy() or dict()
|
||||
if version:
|
||||
expansions["VERSION"] = version
|
||||
if python:
|
||||
expansions["PYTHON_BINARY"] = get_python_binary(python, host)
|
||||
return create_variant_generic(
|
||||
tasks, display_name, version=version, host=host, expansions=expansions, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def get_python_binary(python: str, host: Host) -> str:
|
||||
"""Get the appropriate python binary given a python version and host."""
|
||||
name = host.name
|
||||
if name in ["win64", "win32"]:
|
||||
if name == "win32":
|
||||
base = "C:/python/32"
|
||||
else:
|
||||
base = "C:/python"
|
||||
python_dir = python.replace(".", "").replace("t", "")
|
||||
return f"{base}/Python{python_dir}/python{python}.exe"
|
||||
|
||||
if name in ["rhel8", "ubuntu22", "ubuntu20", "rhel7"]:
|
||||
return f"/opt/python/{python}/bin/python3"
|
||||
|
||||
if name in ["macos", "macos-arm64"]:
|
||||
bin_name = "python3t" if "t" in python else "python3"
|
||||
python_dir = python.replace("t", "")
|
||||
framework_dir = "PythonT" if "t" in python else "Python"
|
||||
return f"/Library/Frameworks/{framework_dir}.Framework/Versions/{python_dir}/bin/{bin_name}"
|
||||
|
||||
raise ValueError(f"no match found for python {python} on {name}")
|
||||
|
||||
|
||||
def get_versions_from(min_version: str) -> list[str]:
|
||||
"""Get all server versions starting from a minimum version."""
|
||||
min_version_float = float(min_version)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
# Install the dependencies needed for an evergreen run.
|
||||
# Install the necessary dependencies.
|
||||
set -eu
|
||||
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
@ -13,50 +13,6 @@ fi
|
||||
# Set up the default bin directory.
|
||||
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
|
||||
PYMONGO_BIN_DIR="$HOME/.local/bin"
|
||||
export PATH="$PYMONGO_BIN_DIR:$PATH"
|
||||
fi
|
||||
|
||||
# Helper function to pip install a dependency using a temporary python env.
|
||||
function _pip_install() {
|
||||
_HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
. $_HERE/../utils.sh
|
||||
_VENV_PATH=$(mktemp -d)
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
_VENV_PATH=$(cygpath -m $_VENV_PATH)
|
||||
fi
|
||||
echo "Installing $2 using pip..."
|
||||
createvirtualenv "$(find_python3)" $_VENV_PATH
|
||||
python -m pip install $1
|
||||
_suffix=""
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
_suffix=".exe"
|
||||
fi
|
||||
ln -s "$(which $2)" $PYMONGO_BIN_DIR/${2}${_suffix}
|
||||
# uv also comes with a uvx binary.
|
||||
if [ $2 == "uv" ]; then
|
||||
ln -s "$(which uvx)" $PYMONGO_BIN_DIR/uvx${_suffix}
|
||||
fi
|
||||
echo "Installed to ${PYMONGO_BIN_DIR}"
|
||||
echo "Installing $2 using pip... done."
|
||||
}
|
||||
|
||||
# Ensure just is installed.
|
||||
if ! command -v just &>/dev/null; then
|
||||
# On most systems we can install directly.
|
||||
_TARGET=""
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
_TARGET="--target x86_64-pc-windows-msvc"
|
||||
fi
|
||||
_BIN_DIR=$PYMONGO_BIN_DIR
|
||||
mkdir -p ${_BIN_DIR}
|
||||
echo "Installing just..."
|
||||
mkdir -p "$_BIN_DIR" 2>/dev/null || true
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $_TARGET --to "$_BIN_DIR" || {
|
||||
# Remove just file if it exists (can be created if there was an install error).
|
||||
rm -f ${_BIN_DIR}/just
|
||||
_pip_install rust-just just
|
||||
}
|
||||
echo "Installing just... done."
|
||||
fi
|
||||
|
||||
# Ensure uv is installed.
|
||||
@ -64,14 +20,17 @@ if ! command -v uv &>/dev/null; then
|
||||
_BIN_DIR=$PYMONGO_BIN_DIR
|
||||
mkdir -p ${_BIN_DIR}
|
||||
echo "Installing uv..."
|
||||
# On most systems we can install directly.
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {
|
||||
_pip_install uv uv
|
||||
}
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
|
||||
fi
|
||||
export PATH="$PYMONGO_BIN_DIR:$PATH"
|
||||
echo "Installing uv... done."
|
||||
fi
|
||||
|
||||
# Ensure just is installed.
|
||||
if ! command -v just &>/dev/null; then
|
||||
uv tool install rust-just
|
||||
fi
|
||||
|
||||
popd > /dev/null
|
||||
|
||||
@ -33,7 +33,7 @@ def _setup_azure_vm(base_env: dict[str, str]) -> None:
|
||||
env["AZUREKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
|
||||
run_command(f"{azure_dir}/run-command.sh", env=env)
|
||||
|
||||
env["AZUREKMS_CMD"] = "NO_EXT=1 bash .evergreen/just.sh setup-tests kms azure-remote"
|
||||
env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote"
|
||||
run_command(f"{azure_dir}/run-command.sh", env=env)
|
||||
LOGGER.info("Setting up Azure VM... done.")
|
||||
|
||||
@ -53,7 +53,7 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None:
|
||||
env["GCPKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
|
||||
run_command(f"{gcp_dir}/run-command.sh", env=env)
|
||||
|
||||
env["GCPKMS_CMD"] = "NO_EXT=1 bash ./.evergreen/just.sh setup-tests kms gcp-remote"
|
||||
env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote"
|
||||
run_command(f"{gcp_dir}/run-command.sh", env=env)
|
||||
LOGGER.info("Setting up GCP VM...")
|
||||
|
||||
|
||||
@ -89,7 +89,7 @@ def test_oidc_send_to_remote(sub_test_name: str) -> None:
|
||||
env[f"{upper_name}OIDC_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
|
||||
env[
|
||||
f"{upper_name}OIDC_TEST_CMD"
|
||||
] = f"NO_EXT=1 OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
|
||||
] = f"OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
|
||||
elif sub_test_name in K8S_NAMES:
|
||||
env["K8S_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
|
||||
env["K8S_TEST_CMD"] = "OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh"
|
||||
|
||||
@ -106,10 +106,11 @@ def handle_aws_lambda() -> None:
|
||||
env["TEST_LAMBDA_DIRECTORY"] = str(target_dir)
|
||||
env.setdefault("AWS_REGION", "us-east-1")
|
||||
dirs = ["pymongo", "gridfs", "bson"]
|
||||
# Store the original .so files.
|
||||
before_sos = []
|
||||
# Remove the original .so files.
|
||||
for dname in dirs:
|
||||
before_sos.extend(f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so"))
|
||||
so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")]
|
||||
for so_path in list(so_paths):
|
||||
Path(so_path).unlink()
|
||||
# Build the c extensions.
|
||||
docker = which("docker") or which("podman")
|
||||
if not docker:
|
||||
@ -122,15 +123,11 @@ def handle_aws_lambda() -> None:
|
||||
target = ROOT / "test/lambda/mongodb" / dname
|
||||
shutil.rmtree(target, ignore_errors=True)
|
||||
shutil.copytree(ROOT / dname, target)
|
||||
# Remove the original so files from the lambda directory.
|
||||
for so_path in before_sos:
|
||||
(ROOT / "test/lambda/mongodb" / so_path).unlink()
|
||||
# Remove the new so files from the ROOT directory.
|
||||
for dname in dirs:
|
||||
so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")]
|
||||
for so_path in list(so_paths):
|
||||
if so_path not in before_sos:
|
||||
Path(so_path).unlink()
|
||||
Path(so_path).unlink()
|
||||
|
||||
script_name = "run-deployed-lambda-aws-tests.sh"
|
||||
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/aws_lambda/{script_name}", env=env)
|
||||
|
||||
@ -1,22 +1,17 @@
|
||||
#!/bin/bash
|
||||
# Set up a development environment on an evergreen host.
|
||||
# Set up development environment.
|
||||
set -eu
|
||||
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
|
||||
ROOT=$(dirname "$(dirname $HERE)")
|
||||
pushd $ROOT > /dev/null
|
||||
|
||||
# Bail early if running on GitHub Actions.
|
||||
if [ -n "${GITHUB_ACTION:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Source the env files to pick up common variables.
|
||||
if [ -f $HERE/env.sh ]; then
|
||||
. $HERE/env.sh
|
||||
fi
|
||||
# PYTHON_BINARY or PYTHON_VERSION may be defined in test-env.sh.
|
||||
|
||||
# Get variables defined in test-env.sh.
|
||||
if [ -f $HERE/test-env.sh ]; then
|
||||
. $HERE/test-env.sh
|
||||
fi
|
||||
@ -24,36 +19,40 @@ fi
|
||||
# Ensure dependencies are installed.
|
||||
bash $HERE/install-dependencies.sh
|
||||
|
||||
# Get the appropriate UV_PYTHON.
|
||||
. $ROOT/.evergreen/utils.sh
|
||||
# Handle the value for UV_PYTHON.
|
||||
. $HERE/setup-uv-python.sh
|
||||
|
||||
if [ -z "${PYTHON_BINARY:-}" ]; then
|
||||
if [ -n "${PYTHON_VERSION:-}" ]; then
|
||||
PYTHON_BINARY=$(get_python_binary $PYTHON_VERSION)
|
||||
else
|
||||
PYTHON_BINARY=$(find_python3)
|
||||
# Only run the next part if not running on CI.
|
||||
if [ -z "${CI:-}" ]; then
|
||||
# Add the default install path to the path if needed.
|
||||
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
|
||||
export PATH="$PATH:$HOME/.local/bin"
|
||||
fi
|
||||
|
||||
# Set up venv, making sure c extensions build unless disabled.
|
||||
if [ -z "${NO_EXT:-}" ]; then
|
||||
export PYMONGO_C_EXT_MUST_BUILD=1
|
||||
fi
|
||||
|
||||
(
|
||||
cd $ROOT && uv sync
|
||||
)
|
||||
|
||||
# Set up build utilities on Windows spawn hosts.
|
||||
if [ -f $HOME/.visualStudioEnv.sh ]; then
|
||||
set +u
|
||||
SSH_TTY=1 source $HOME/.visualStudioEnv.sh
|
||||
set -u
|
||||
fi
|
||||
|
||||
# Only set up pre-commit if we are in a git checkout.
|
||||
if [ -f $HERE/.git ]; then
|
||||
if ! command -v pre-commit &>/dev/null; then
|
||||
uv tool install pre-commit
|
||||
fi
|
||||
fi
|
||||
export UV_PYTHON=${PYTHON_BINARY}
|
||||
echo "Using python $UV_PYTHON"
|
||||
|
||||
# Add the default install path to the path if needed.
|
||||
if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
|
||||
export PATH="$PATH:$HOME/.local/bin"
|
||||
if [ ! -f .git/hooks/pre-commit ]; then
|
||||
uvx pre-commit install
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Set up venv, making sure c extensions build unless disabled.
|
||||
if [ -z "${NO_EXT:-}" ]; then
|
||||
export PYMONGO_C_EXT_MUST_BUILD=1
|
||||
fi
|
||||
# Set up visual studio env on Windows spawn hosts.
|
||||
if [ -f $HOME/.visualStudioEnv.sh ]; then
|
||||
set +u
|
||||
SSH_TTY=1 source $HOME/.visualStudioEnv.sh
|
||||
set -u
|
||||
fi
|
||||
uv sync
|
||||
|
||||
echo "Setting up python environment... done."
|
||||
|
||||
popd > /dev/null
|
||||
|
||||
@ -8,9 +8,13 @@ echo "Setting up system..."
|
||||
bash .evergreen/scripts/configure-env.sh
|
||||
source .evergreen/scripts/env.sh
|
||||
bash $DRIVERS_TOOLS/.evergreen/setup.sh
|
||||
bash .evergreen/scripts/install-dependencies.sh
|
||||
popd
|
||||
|
||||
# Run spawn host-specific tasks.
|
||||
if [ -z "${CI:-}" ]; then
|
||||
bash $HERE/setup-dev-env.sh
|
||||
fi
|
||||
|
||||
# Enable core dumps if enabled on the machine
|
||||
# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
|
||||
if [ -f /proc/self/coredump_filter ]; then
|
||||
|
||||
53
.evergreen/scripts/setup-uv-python.sh
Executable file
53
.evergreen/scripts/setup-uv-python.sh
Executable file
@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
# Set up the UV_PYTHON variable.
|
||||
set -eu
|
||||
|
||||
HERE=$(dirname ${BASH_SOURCE:-$0})
|
||||
HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
|
||||
|
||||
# Use min supported version by default.
|
||||
_python="3.10"
|
||||
|
||||
# Source the env files to pick up common variables.
|
||||
if [ -f $HERE/env.sh ]; then
|
||||
. $HERE/env.sh
|
||||
fi
|
||||
|
||||
# Get variables defined in test-env.sh.
|
||||
if [ -f $HERE/test-env.sh ]; then
|
||||
. $HERE/test-env.sh
|
||||
fi
|
||||
|
||||
if [ -z "${UV_PYTHON:-}" ]; then
|
||||
set -x
|
||||
# Translate a TOOLCHAIN_VERSION to UV_PYTHON.
|
||||
if [ -n "${TOOLCHAIN_VERSION:-}" ]; then
|
||||
_python=$TOOLCHAIN_VERSION
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
if [[ "$_python" == *"t"* ]]; then
|
||||
binary_name="python3t"
|
||||
framework_dir="PythonT"
|
||||
else
|
||||
binary_name="python3"
|
||||
framework_dir="Python"
|
||||
fi
|
||||
_python=$(echo "$_python" | sed 's/t//g')
|
||||
_python="/Library/Frameworks/$framework_dir.Framework/Versions/$_python/bin/$binary_name"
|
||||
elif [ "Windows_NT" = "${OS:-}" ]; then
|
||||
_python=$(echo $_python | cut -d. -f1,2 | sed 's/\.//g; s/t//g')
|
||||
if [[ "$TOOLCHAIN_VERSION" == *"t"* ]]; then
|
||||
_exe="python${TOOLCHAIN_VERSION}.exe"
|
||||
else
|
||||
_exe="python.exe"
|
||||
fi
|
||||
if [ -n "${IS_WIN32:-}" ]; then
|
||||
_python="C:/python/32/Python${_python}/${_exe}"
|
||||
else
|
||||
_python="C:/python/Python${_python}/${_exe}"
|
||||
fi
|
||||
elif [ -d "/opt/python/$_python/bin" ]; then
|
||||
_python="/opt/python/$_python/bin/python3"
|
||||
fi
|
||||
fi
|
||||
export UV_PYTHON="$_python"
|
||||
fi
|
||||
@ -31,8 +31,7 @@ PASS_THROUGH_ENV = [
|
||||
"NO_EXT",
|
||||
"MONGODB_API_VERSION",
|
||||
"DEBUG_LOG",
|
||||
"PYTHON_BINARY",
|
||||
"PYTHON_VERSION",
|
||||
"UV_PYTHON",
|
||||
"REQUIRE_FIPS",
|
||||
"IS_WIN32",
|
||||
]
|
||||
|
||||
@ -15,5 +15,4 @@ echo "Copying files to $target..."
|
||||
rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_dir
|
||||
echo "Copying files to $target... done"
|
||||
|
||||
ssh $target $remote_dir/.evergreen/scripts/setup-system.sh
|
||||
ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} .evergreen/scripts/setup-dev-env.sh"
|
||||
ssh $target "$remote_dir/.evergreen/scripts/setup-system.sh"
|
||||
|
||||
@ -1,148 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Utility functions used by pymongo evergreen scripts.
|
||||
set -eu
|
||||
|
||||
find_python3() {
|
||||
PYTHON=""
|
||||
# Find a suitable toolchain version, if available.
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3"
|
||||
elif [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
|
||||
PYTHON="C:/python/Python310/python.exe"
|
||||
else
|
||||
# Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.10+.
|
||||
if [ -f "/opt/python/3.10/bin/python3" ]; then
|
||||
PYTHON="/opt/python/Current/bin/python3"
|
||||
elif is_python_310 "$(command -v /opt/mongodbtoolchain/v5/bin/python3)"; then
|
||||
PYTHON="/opt/mongodbtoolchain/v5/bin/python3"
|
||||
elif is_python_310 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then
|
||||
PYTHON="/opt/mongodbtoolchain/v4/bin/python3"
|
||||
elif is_python_310 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then
|
||||
PYTHON="/opt/mongodbtoolchain/v3/bin/python3"
|
||||
fi
|
||||
fi
|
||||
# Add a fallback system python3 if it is available and Python 3.10+.
|
||||
if [ -z "$PYTHON" ]; then
|
||||
if is_python_310 "$(command -v python3)"; then
|
||||
PYTHON="$(command -v python3)"
|
||||
fi
|
||||
fi
|
||||
if [ -z "$PYTHON" ]; then
|
||||
echo "Cannot test without python3.10+ installed!"
|
||||
exit 1
|
||||
fi
|
||||
echo "$PYTHON"
|
||||
}
|
||||
|
||||
# Usage:
|
||||
# createvirtualenv /path/to/python /output/path/for/venv
|
||||
# * param1: Python binary to use for the virtualenv
|
||||
# * param2: Path to the virtualenv to create
|
||||
createvirtualenv () {
|
||||
PYTHON=$1
|
||||
VENVPATH=$2
|
||||
|
||||
# Prefer venv
|
||||
VENV="$PYTHON -m venv"
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
VIRTUALENV="$PYTHON -m virtualenv"
|
||||
else
|
||||
VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv")
|
||||
VIRTUALENV="$VIRTUALENV -p $PYTHON"
|
||||
fi
|
||||
if ! $VENV $VENVPATH 2>/dev/null; then
|
||||
# Workaround for bug in older versions of virtualenv.
|
||||
$VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH
|
||||
fi
|
||||
if [ "Windows_NT" = "${OS:-}" ]; then
|
||||
# Workaround https://bugs.python.org/issue32451:
|
||||
# mongovenv/Scripts/activate: line 3: $'\r': command not found
|
||||
dos2unix $VENVPATH/Scripts/activate || true
|
||||
. $VENVPATH/Scripts/activate
|
||||
else
|
||||
. $VENVPATH/bin/activate
|
||||
fi
|
||||
|
||||
export PIP_QUIET=1
|
||||
python -m pip install --upgrade pip
|
||||
}
|
||||
|
||||
# Usage:
|
||||
# testinstall /path/to/python /path/to/.whl ["no-virtualenv"]
|
||||
# * param1: Python binary to test
|
||||
# * param2: Path to the wheel to install
|
||||
# * param3 (optional): If set to a non-empty string, don't create a virtualenv. Used in manylinux containers.
|
||||
testinstall () {
|
||||
PYTHON=$1
|
||||
RELEASE=$2
|
||||
NO_VIRTUALENV=$3
|
||||
PYTHON_IMPL=$(python -c "import platform; print(platform.python_implementation())")
|
||||
|
||||
if [ -z "$NO_VIRTUALENV" ]; then
|
||||
createvirtualenv $PYTHON venvtestinstall
|
||||
PYTHON=python
|
||||
fi
|
||||
|
||||
$PYTHON -m pip install --upgrade $RELEASE
|
||||
cd tools
|
||||
|
||||
if [ "$PYTHON_IMPL" = "CPython" ]; then
|
||||
$PYTHON fail_if_no_c.py
|
||||
fi
|
||||
|
||||
$PYTHON -m pip uninstall -y pymongo
|
||||
cd ..
|
||||
|
||||
if [ -z "$NO_VIRTUALENV" ]; then
|
||||
deactivate
|
||||
rm -rf venvtestinstall
|
||||
fi
|
||||
}
|
||||
|
||||
# Function that returns success if the provided Python binary is version 3.10 or later
|
||||
# Usage:
|
||||
# is_python_310 /path/to/python
|
||||
# * param1: Python binary
|
||||
is_python_310() {
|
||||
if [ -z "$1" ]; then
|
||||
return 1
|
||||
elif $1 -c "import sys; exit(sys.version_info[:2] < (3, 10))"; then
|
||||
# runs when sys.version_info[:2] >= (3, 10)
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# Function that gets a python binary given a python version string.
|
||||
# Versions can be of the form 3.xx or pypy3.xx.
|
||||
get_python_binary() {
|
||||
version=$1
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
if [[ "$version" == *"t"* ]]; then
|
||||
binary_name="python3t"
|
||||
framework_dir="PythonT"
|
||||
else
|
||||
binary_name="python3"
|
||||
framework_dir="Python"
|
||||
fi
|
||||
version=$(echo "$version" | sed 's/t//g')
|
||||
PYTHON="/Library/Frameworks/$framework_dir.Framework/Versions/$version/bin/$binary_name"
|
||||
elif [ "Windows_NT" = "${OS:-}" ]; then
|
||||
version=$(echo $version | cut -d. -f1,2 | sed 's/\.//g; s/t//g')
|
||||
if [ -n "${IS_WIN32:-}" ]; then
|
||||
PYTHON="C:/python/32/Python$version/python.exe"
|
||||
else
|
||||
PYTHON="C:/python/Python$version/python.exe"
|
||||
fi
|
||||
else
|
||||
PYTHON="/opt/python/$version/bin/python3"
|
||||
fi
|
||||
if is_python_310 "$(command -v $PYTHON)"; then
|
||||
echo "$PYTHON"
|
||||
else
|
||||
echo "Could not find suitable python binary for '$version'" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@ -194,7 +194,7 @@ the pages will re-render and the browser will automatically refresh.
|
||||
|
||||
- Run `just install` to set a local virtual environment, or you can manually
|
||||
create a virtual environment and run `pytest` directly. If you want to use a specific
|
||||
version of Python, remove the `.venv` folder and set `PYTHON_BINARY` before running `just install`.
|
||||
version of Python, set `UV_PYTHON` before running `just install`.
|
||||
- Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args
|
||||
to set up the server. All given options will be passed to
|
||||
[`run-orchestration.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh). Run `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h`
|
||||
@ -335,7 +335,7 @@ Locally you can run:
|
||||
|
||||
- Run `just run-server`.
|
||||
- Run `just setup-tests`.
|
||||
- Run `UV_PYTHON=3.13t just run-tests`.
|
||||
- Run `UV_PYTHON=3.14t just run-tests`.
|
||||
|
||||
### AWS Lambda tests
|
||||
|
||||
@ -399,6 +399,16 @@ To run any of the test suites with minimum supported dependencies, pass `--test-
|
||||
- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`.
|
||||
- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`.
|
||||
- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`.
|
||||
- There are some considerations about the Python version used in the test:
|
||||
- If a specific version of Python is needed in a task that is running on variants with a toolchain, use
|
||||
``TOOLCHAIN_VERSION`` (e.g. `TOOLCHAIN_VERSION=3.10`). The actual path lookup needs to be done on the host, since
|
||||
tasks are host-agnostic.
|
||||
- If a specific Python binary is needed (for example on the FIPS host), set `UV_PYTHON=/path/to/python`.
|
||||
- If a specific Python version is needed and the toolchain will not be available, use `UV_PYTHON` (e.g. `UV_PYTHON=3.11`).
|
||||
- The default if neither ``TOOLCHAIN_VERSION`` or ``UV_PYTHON`` is set is to use UV to install the minimum
|
||||
supported version of Python and use that. This ensures a consistent behavior across host types that do not
|
||||
have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`)
|
||||
needed to build the C extensions.
|
||||
- Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`.
|
||||
- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`.
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user