SERVER-91438 buildscripts/poetry_sync.sh (#23413)

GitOrigin-RevId: 2e5b28b0dac9355b7a2c80584f28dcc2a03e6ae0

buildscripts/poetry_sync.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Run this in a mongo git repo to setup the Python dependencies
+# needed to build that repo, as specified in `poetry.lock`.
+#
+# If your virtual env has been activated, you can usually just
+# execute this script with no arguments.
+#
+# Optionally override the Python interpreter with `$PYTHON3`, or
+# with the `-p <path_to_python3>` option.  It defaults to
+# `python3`, which is usually correct.
+#
+# Normally, this script refuses to install unless a virtual env is
+# active. This avoids making accidental system-wide changes. A user
+# can override this safety measure with `-f`.
+#
+# Our workspace setup requires a specific version of poetry to be
+# installed, this script automates the pip install of that version.
+
+poetry_version='1.5.1'
+
+allow_no_venv=0
+python_optarg=""
+dry_run=0
+
+while getopts p:fin opt; do
+    case "${opt}" in
+    p)
+        python_optarg="${OPTARG}"
+        ;;
+    f)
+        allow_no_venv=1
+        ;;
+    n)
+        dry_run=1
+        ;;
+    ?)
+        echo "invalid option: ${opt}" >&2
+        exit 2
+        ;;
+    esac
+done
+
+run () {
+    echo "$@"
+    if [[ "${dry_run}" == 1 ]]; then
+        return
+    fi
+    "$@"
+}
+
+if [[ "${allow_no_venv}" != 1 && -z "${VIRTUAL_ENV}" ]]; then
+    cat <<EOF >&2
+Refusing to run without a python virtual env.
+See https://github.com/10gen/mongo/blob/master/docs/building.md#python-prerequisites.
+Provide the '-f' option to run anyway.
+EOF
+    exit 1
+fi
+
+# Make a structured and user-evident choice of python interpreter.
+if [[ -n "${python_optarg}" ]]; then
+    py3="${python_optarg}"
+    echo "Using '${py3}' from explicit \`-p\` option" >&2
+elif [[ -n "${VIRTUAL_ENV}" ]]; then
+    py3="${VIRTUAL_ENV}/bin/python3"
+    echo "Using '${py3}' based on VIRTUAL_ENV=${VIRTUAL_ENV}" >&2
+elif [[ -n "${PYTHON3}" ]]; then
+    py3="${PYTHON3}"
+    echo "Using '${py3}' from PYTHON3 variable" >&2
+else
+    py3="python3"
+    echo "Using '${py3}' as hardcoded fallback value" >&2
+fi
+
+pip_opts=()
+if [[ "${allow_no_venv}" != 1 ]]; then
+    # Exploit pip's own enforcement of virtualenv.
+    pip_opts+=('--require-virtualenv')
+fi
+run "${py3}" -m pip install "${pip_opts[@]}" "poetry==${poetry_version}"
+
+run env \
+    PYTHON_KEYRING_BACKEND="keyring.backends.null.Keyring" \
+    "${py3}" -m poetry install --no-root --sync

docs/building.md

@@ -37,8 +37,13 @@ python3 --version # Should be >=3.10. If it is not, follow [online instructions]
 python3 -m venv python3-venv --prompt mongo # Create a virtual environment. "python3-venv" is non standard but it is kept since it is assumed elsewhere in our code.
 source python3-venv/bin/activate # You should see a (mongo) appear in your terminal
 which python3 # This should point to the python in python3-venv
-python3 -m pip install 'poetry==1.5.1' # It is also non standard to install poetry into its own virtual environment. However, the idea is to make even fewer unpinned dependencies.
-PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring python3 -m poetry install --no-root --sync # install all required python dependencies to build and test.
+
+# It is also non standard to install poetry into its own virtual environment.
+# However, the idea is to make even fewer unpinned dependencies.
+# Install poetry 1.5.1 into the virtual env, then install all
+# required python dependencies to build and test.
+buildscripts/poetry_sync.sh
+
 python3 buildscripts/scons.py --build-profile=opt
 ninja -f opt.ninja -j 200 install-devcore
 ```
@@ -58,8 +63,7 @@ dedicated to building MongoDB is optional but recommended.
 
     $ python3 -m venv <venv_path> --prompt mongo # Optional (venv_path can be a path of your choice)
     $ source <venv_path>/bin/activate # Optional (might be slightly different based on the your shell)
-    $ python3 -m pip install 'poetry==1.5.1'
-    $ python3 -m poetry install --no-root --sync
+    $ buildscripts/poetry_sync.sh -i
 
 Note: In order to compile C-based Python modules, you'll also need the
 Python and OpenSSL C headers. Run:
@@ -68,7 +72,9 @@ Python and OpenSSL C headers. Run:
 - Ubuntu (20.04 and newer)/Debian (Bullseye and newer) - `apt install python-dev-is-python3 libssl-dev`
 - Ubuntu (18.04 and older)/Debian (Buster and older) - `apt install python3.7-dev libssl-dev`
 
-Note: If you are seeing errors involving "Prompt dismissed.." you might need to run the following command before poetry install.
+Note: If you are running poetry manually and seeing errors involving "Prompt
+dismissed.." you might need to run the following command before poetry install
+(`buildscripts/poetry-sync.sh` does this internally).
 
     $ export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
 

site_scons/mongo/pip_requirements.py

@@ -59,8 +59,7 @@ def verify_requirements(silent: bool = False, executable=sys.executable):
         raise MissingRequirements(
             f"Detected one or more packages are out of date. "
             f"Try running:\n"
-            f"    export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring\n"
-            f"    python3 -m poetry install --no-root --sync"
+            f"    buildscripts/poetry_sync.sh -p '{executable}'\n"
         )
 
     # String match should look like the following
@@ -88,6 +87,5 @@ def verify_requirements(silent: bool = False, executable=sys.executable):
         raise MissingRequirements(
             f"Detected one or more packages are out of date. "
             f"Try running:\n"
-            f"    export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring\n"
-            f"    {executable} -m poetry install --no-root --sync"
+            f"    buildscripts/poetry_sync.sh -p '{executable}'\n"
         )

src/third_party/wiredtiger/test/evergreen.yml

@@ -207,12 +207,11 @@ functions:
         virtualenv -p python3 venv
         source venv/bin/activate
         python3 -m pip install 'poetry==1.5.1'
-        export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
         # FIXME-WT-12911 - There's a dependency ordering issue in the mongo poetry.lock file. Running poetry install twice resolves it.
         set +o errexit
-        python3 -m poetry install --no-root --sync
+        buildscripts/poetry_sync.sh
         set -o errexit
-        python3 -m poetry install --no-root --sync
+        buildscripts/poetry_sync.sh
 
         ./buildscripts/scons.py --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --link-model=dynamic --ninja generate-ninja ICECC=icecc CCACHE=ccache
         ninja -j$(nproc --all) install-mongod