From 5782ce8c7e5697281de96dd67a06eb3bb2676757 Mon Sep 17 00:00:00 2001 From: Billy Donahue Date: Fri, 14 Jun 2024 22:50:56 -0400 Subject: [PATCH] SERVER-91438 buildscripts/poetry_sync.sh (#23413) GitOrigin-RevId: 2e5b28b0dac9355b7a2c80584f28dcc2a03e6ae0 --- buildscripts/poetry_sync.sh | 84 +++++++++++++++++++ docs/building.md | 16 ++-- site_scons/mongo/pip_requirements.py | 6 +- src/third_party/wiredtiger/test/evergreen.yml | 5 +- 4 files changed, 99 insertions(+), 12 deletions(-) create mode 100755 buildscripts/poetry_sync.sh diff --git a/buildscripts/poetry_sync.sh b/buildscripts/poetry_sync.sh new file mode 100755 index 00000000000..3740b79a898 --- /dev/null +++ b/buildscripts/poetry_sync.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Run this in a mongo git repo to setup the Python dependencies +# needed to build that repo, as specified in `poetry.lock`. +# +# If your virtual env has been activated, you can usually just +# execute this script with no arguments. +# +# Optionally override the Python interpreter with `$PYTHON3`, or +# with the `-p ` option. It defaults to +# `python3`, which is usually correct. +# +# Normally, this script refuses to install unless a virtual env is +# active. This avoids making accidental system-wide changes. A user +# can override this safety measure with `-f`. +# +# Our workspace setup requires a specific version of poetry to be +# installed, this script automates the pip install of that version. + +poetry_version='1.5.1' + +allow_no_venv=0 +python_optarg="" +dry_run=0 + +while getopts p:fin opt; do + case "${opt}" in + p) + python_optarg="${OPTARG}" + ;; + f) + allow_no_venv=1 + ;; + n) + dry_run=1 + ;; + ?) + echo "invalid option: ${opt}" >&2 + exit 2 + ;; + esac +done + +run () { + echo "$@" + if [[ "${dry_run}" == 1 ]]; then + return + fi + "$@" +} + +if [[ "${allow_no_venv}" != 1 && -z "${VIRTUAL_ENV}" ]]; then + cat <&2 +Refusing to run without a python virtual env. +See https://github.com/10gen/mongo/blob/master/docs/building.md#python-prerequisites. +Provide the '-f' option to run anyway. +EOF + exit 1 +fi + +# Make a structured and user-evident choice of python interpreter. +if [[ -n "${python_optarg}" ]]; then + py3="${python_optarg}" + echo "Using '${py3}' from explicit \`-p\` option" >&2 +elif [[ -n "${VIRTUAL_ENV}" ]]; then + py3="${VIRTUAL_ENV}/bin/python3" + echo "Using '${py3}' based on VIRTUAL_ENV=${VIRTUAL_ENV}" >&2 +elif [[ -n "${PYTHON3}" ]]; then + py3="${PYTHON3}" + echo "Using '${py3}' from PYTHON3 variable" >&2 +else + py3="python3" + echo "Using '${py3}' as hardcoded fallback value" >&2 +fi + +pip_opts=() +if [[ "${allow_no_venv}" != 1 ]]; then + # Exploit pip's own enforcement of virtualenv. + pip_opts+=('--require-virtualenv') +fi +run "${py3}" -m pip install "${pip_opts[@]}" "poetry==${poetry_version}" + +run env \ + PYTHON_KEYRING_BACKEND="keyring.backends.null.Keyring" \ + "${py3}" -m poetry install --no-root --sync diff --git a/docs/building.md b/docs/building.md index 4a8867c3334..ec79c0840e3 100644 --- a/docs/building.md +++ b/docs/building.md @@ -37,8 +37,13 @@ python3 --version # Should be >=3.10. If it is not, follow [online instructions] python3 -m venv python3-venv --prompt mongo # Create a virtual environment. "python3-venv" is non standard but it is kept since it is assumed elsewhere in our code. source python3-venv/bin/activate # You should see a (mongo) appear in your terminal which python3 # This should point to the python in python3-venv -python3 -m pip install 'poetry==1.5.1' # It is also non standard to install poetry into its own virtual environment. However, the idea is to make even fewer unpinned dependencies. -PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring python3 -m poetry install --no-root --sync # install all required python dependencies to build and test. + +# It is also non standard to install poetry into its own virtual environment. +# However, the idea is to make even fewer unpinned dependencies. +# Install poetry 1.5.1 into the virtual env, then install all +# required python dependencies to build and test. +buildscripts/poetry_sync.sh + python3 buildscripts/scons.py --build-profile=opt ninja -f opt.ninja -j 200 install-devcore ``` @@ -58,8 +63,7 @@ dedicated to building MongoDB is optional but recommended. $ python3 -m venv --prompt mongo # Optional (venv_path can be a path of your choice) $ source /bin/activate # Optional (might be slightly different based on the your shell) - $ python3 -m pip install 'poetry==1.5.1' - $ python3 -m poetry install --no-root --sync + $ buildscripts/poetry_sync.sh -i Note: In order to compile C-based Python modules, you'll also need the Python and OpenSSL C headers. Run: @@ -68,7 +72,9 @@ Python and OpenSSL C headers. Run: - Ubuntu (20.04 and newer)/Debian (Bullseye and newer) - `apt install python-dev-is-python3 libssl-dev` - Ubuntu (18.04 and older)/Debian (Buster and older) - `apt install python3.7-dev libssl-dev` -Note: If you are seeing errors involving "Prompt dismissed.." you might need to run the following command before poetry install. +Note: If you are running poetry manually and seeing errors involving "Prompt +dismissed.." you might need to run the following command before poetry install +(`buildscripts/poetry-sync.sh` does this internally). $ export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring diff --git a/site_scons/mongo/pip_requirements.py b/site_scons/mongo/pip_requirements.py index 3fda5f12bd4..b60e0ee6928 100644 --- a/site_scons/mongo/pip_requirements.py +++ b/site_scons/mongo/pip_requirements.py @@ -59,8 +59,7 @@ def verify_requirements(silent: bool = False, executable=sys.executable): raise MissingRequirements( f"Detected one or more packages are out of date. " f"Try running:\n" - f" export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring\n" - f" python3 -m poetry install --no-root --sync" + f" buildscripts/poetry_sync.sh -p '{executable}'\n" ) # String match should look like the following @@ -88,6 +87,5 @@ def verify_requirements(silent: bool = False, executable=sys.executable): raise MissingRequirements( f"Detected one or more packages are out of date. " f"Try running:\n" - f" export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring\n" - f" {executable} -m poetry install --no-root --sync" + f" buildscripts/poetry_sync.sh -p '{executable}'\n" ) diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index a2ea0f6a215..aa35ccb7417 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -207,12 +207,11 @@ functions: virtualenv -p python3 venv source venv/bin/activate python3 -m pip install 'poetry==1.5.1' - export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring # FIXME-WT-12911 - There's a dependency ordering issue in the mongo poetry.lock file. Running poetry install twice resolves it. set +o errexit - python3 -m poetry install --no-root --sync + buildscripts/poetry_sync.sh set -o errexit - python3 -m poetry install --no-root --sync + buildscripts/poetry_sync.sh ./buildscripts/scons.py --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --link-model=dynamic --ninja generate-ninja ICECC=icecc CCACHE=ccache ninja -j$(nproc --all) install-mongod