PYTHON-3983 Clean up handling of PYTHON_BINARY (#1382)

2023-10-06 19:11:52 -05:00 · 2023-10-06 19:11:52 -05:00 · ed3ae41407
commit ed3ae41407
parent 556557255a
6 changed files with 57 additions and 60 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -79,17 +79,6 @@ functions:
              export PATH="$MONGODB_BINARIES:$PATH"
              export PROJECT="${project}"
              export PIP_QUIET=1
-           ENSURE_TOOLCHAIN_PYTHON_BINARY: |
-              # Make sure PYTHON_BINARY is set to a suitable toolchain python.
-              if [ -z "$PYTHON_BINARY" ]; then
-                  if [ "$(uname -s)" = "Darwin" ]; then
-                      export PYTHON_BINARY=/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
-                  elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
-                      export PYTHON_BINARY=/cygdrive/c/python/Python39/python
-                  else
-                      export PYTHON_BINARY=/opt/python/3.9/bin/python3
-                  fi
-              fi
           EOT

    # Load the expansion file to make an evergreen variable with the current unique version
@ -502,12 +491,8 @@ functions:
            export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
          fi

-          export PYTHON_BINARY=${PYTHON_BINARY}
-          if [ -z "$PYTHON_BINARY" ]; then
-            export PYTHON_BINARY=${python3_binary}
-          fi
-
          GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
+            PYTHON_BINARY=${PYTHON_BINARY} \
            C_EXTENSIONS=${C_EXTENSIONS} \
            COVERAGE=${COVERAGE} \
            COMPRESSORS=${COMPRESSORS} \
@ -678,7 +663,6 @@ functions:
        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
        script: |
          ${PREPARE_SHELL}
-          ${ENSURE_TOOLCHAIN_PYTHON_BINARY}
          bash .evergreen/run-mongodb-oidc-test.sh

  "run aws auth test with aws credentials as environment variables":
@ -1630,8 +1614,6 @@ tasks:
      tags: ["no-server"]
      commands:
        - func: "run tests"
-          vars:
-            PYTHON_BINARY: /opt/python/3.7/bin/python3

    - name: "atlas-connect"
      tags: ["atlas-connect"]
@ -2108,7 +2090,6 @@ tasks:
          vars:
            VERSION: "latest"
            TOPOLOGY: "server"
-            PYTHON_BINARY: "/opt/mongodbtoolchain/v3/bin/python3"
        - func: "run tests"
 # }}}
    - name: "coverage-report"
@ -2260,7 +2241,6 @@ axes:
          skip_EC2_auth_test: true
          skip_ECS_auth_test: true
          skip_web_identity_auth_test: true
-          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
      - id: macos-1100
        display_name: "macOS 11.00"
@ -2269,7 +2249,6 @@ axes:
          skip_EC2_auth_test: true
          skip_ECS_auth_test: true
          skip_web_identity_auth_test: true
-          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
      - id: macos-1100-arm64
        display_name: "macOS 11.00 Arm64"
@ -2278,21 +2257,18 @@ axes:
          skip_EC2_auth_test: true
          skip_ECS_auth_test: true
          skip_web_identity_auth_test: true
-          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
      - id: rhel7
        display_name: "RHEL 7.x"
        run_on: rhel79-small
        batchtime: 10080  # 7 days
        variables:
-          python3_binary: "/opt/python/3.8/bin/python3"
          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz
      - id: rhel8
        display_name: "RHEL 8.x"
        run_on: rhel87-small
        batchtime: 10080  # 7 days
        variables:
-          python3_binary: "/opt/python/3.8/bin/python3"
          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz
      - id: rhel80-fips
        display_name: "RHEL 8.0 FIPS"
@ -2304,14 +2280,10 @@ axes:
        display_name: "Ubuntu 22.04"
        run_on: ubuntu2204-small
        batchtime: 10080  # 7 days
-        variables:
-          python3_binary: python3
      - id: ubuntu-18.04
        display_name: "Ubuntu 18.04"
        run_on: ubuntu1804-small
        batchtime: 10080  # 7 days
-        variables:
-          python3_binary: python3
      - id: rhel83-zseries
        display_name: "RHEL 8.3 (zSeries)"
        run_on: rhel83-zseries-small
@ -2334,7 +2306,6 @@ axes:
          skip_ECS_auth_test: true
          skip_EC2_auth_test: true
          skip_web_identity_auth_test: true
-          python3_binary: "C:/python/Python38/python.exe"
          venv_bin_dir: "Scripts"
          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz

@ -2476,14 +2447,6 @@ axes:
        variables:
           PYTHON_BINARY: "/opt/python/pypy3.10/bin/pypy3"

-  - id: python-version-mac
-    display_name: "Python"
-    values:
-      - id: "system-python3"
-        display_name: "Python3"
-        variables:
-          PYTHON_BINARY: "python3"
-
  - id: python-version-windows
    display_name: "Python"
    values:
@ -3166,7 +3129,6 @@ buildvariants:
 - matrix_name: "aws-auth-test-mac"
  matrix_spec:
    platform: [macos-1014]
-    python-version-mac: ["system-python3"]
  display_name: "MONGODB-AWS Auth ${platform} ${python-version-mac}"
  tasks:
    - name: "aws-auth-test-4.4"
--- a/.evergreen/run-mongodb-aws-test.sh
+++ b/.evergreen/run-mongodb-aws-test.sh
@ -18,6 +18,9 @@ echo "Running MONGODB-AWS authentication tests"
 # ensure no secrets are printed in log files
 set +x

+. ./.evergreen/utils.sh
+PYTHON_BINARY=$(find_python3)
+
 # Try to source exported AWS Secrets
 if [ -f ./secrets-export.sh ]; then
  source ./secrets-export.sh
--- a/.evergreen/run-mongodb-oidc-test.sh
+++ b/.evergreen/run-mongodb-oidc-test.sh
@ -13,13 +13,13 @@ fi

 # Get the drivers secrets.  Use an existing secrets file first.
 if [ ! -f "./secrets-export.sh" ]; then
-    bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc
 fi
 source ./secrets-export.sh

 # # If the file did not have our creds, get them from the vault.
 if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then
-    bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc
    source ./secrets-export.sh
 fi

--- a/.evergreen/tox.sh
+++ b/.evergreen/tox.sh
@ -1,18 +1,11 @@
 #!/bin/bash
 set -o errexit  # Exit the script with error if any of the commands fail
+set -x

 . .evergreen/utils.sh

 if [ -z "$PYTHON_BINARY" ]; then
-    # Use Python 3 from the server toolchain to test on ARM, POWER or zSeries if a
-    # system python3 doesn't exist or exists but is older than 3.7.
-    if is_python_37 "$(command -v python3)"; then
-        PYTHON_BINARY=$(command -v python3)
-    elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then
-        PYTHON_BINARY=$(command -v /opt/mongodbtoolchain/v3/bin/python3)
-    else
-        echo "Cannot test without python3.7+ installed!"
-    fi
+    PYTHON_BINARY=$(find_python3)
 fi

 if $PYTHON_BINARY -m tox --version; then
--- a/.evergreen/utils.sh
+++ b/.evergreen/utils.sh
@ -2,6 +2,40 @@

 set -o xtrace

+find_python3() {
+    PYTHON=""
+    # Add a fallback system python3 if it is available and Python 3.7+.
+    if is_python_37 "$(command -v python3)"; then
+        PYTHON="$(command -v python3)"
+    fi
+    # Find a suitable toolchain version, if available.
+    if [ "$(uname -s)" = "Darwin" ]; then
+        # macos 11.00
+        if [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then
+            PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3"
+        # macos 10.14
+        elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then
+            PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3"
+        fi
+    elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+        PYTHON="C:/python/Python37/python.exe"
+    else
+        # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.7+.
+        if [ -f "/opt/python/3.7/bin/python3" ]; then
+            PYTHON="/opt/python/3.7/bin/python3"
+        elif is_python_37 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then
+            PYTHON="/opt/mongodbtoolchain/v4/bin/python3"
+        elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then
+            PYTHON="/opt/mongodbtoolchain/v3/bin/python3"
+        fi
+    fi
+    if [ -z "$PYTHON" ]; then
+        echo "Cannot test without python3.7+ installed!"
+        exit 1
+    fi
+    echo "$PYTHON"
+}
+
 # Usage:
 # createvirtualenv /path/to/python /output/path/for/venv
 # * param1: Python binary to use for the virtualenv
@ -9,17 +43,18 @@ set -o xtrace
 createvirtualenv () {
    PYTHON=$1
    VENVPATH=$2
-    if $PYTHON -m virtualenv --version; then
+    # Prefer venv
+    VENV="$PYTHON -m venv"
+    if [ "$(uname -s)" = "Darwin" ]; then
        VIRTUALENV="$PYTHON -m virtualenv"
-    elif $PYTHON -m venv -h > /dev/null; then
-        # System virtualenv might not be compatible with the python3 on our path
-        VIRTUALENV="$PYTHON -m venv"
    else
-        echo "Cannot test without virtualenv"
-        exit 1
+        VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv")
+        VIRTUALENV="$VIRTUALENV -p $PYTHON"
+    fi
+    if ! $VENV $VENVPATH 2>/dev/null; then
+        # Workaround for bug in older versions of virtualenv.
+        $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH
    fi
-    # Workaround for bug in older versions of virtualenv.
-    $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH
    if [ "Windows_NT" = "$OS" ]; then
        # Workaround https://bugs.python.org/issue32451:
        # mongovenv/Scripts/activate: line 3: $'\r': command not found
@ -31,7 +66,7 @@ createvirtualenv () {

    export PIP_QUIET=1
    python -m pip install --upgrade pip
-    python -m pip install --upgrade setuptools wheel tox
+    python -m pip install --upgrade setuptools tox
 }

 # Usage:
--- a/test/test_index_management.py
+++ b/test/test_index_management.py
@ -67,9 +67,13 @@ class TestSearchIndexProse(unittest.TestCase):
        password = os.environ["DB_PASSWORD"]
        cls.client = MongoClient(url, username=username, password=password)
        cls.client.drop_database(_NAME)
-        cls.addClassCleanup(cls.client.close)
        cls.db = cls.client.test_search_index_prose

+    @classmethod
+    def tearDownClass(cls):
+        cls.client.drop_database(_NAME)
+        cls.client.close()
+
    def wait_for_ready(self, coll, name=_NAME, predicate=None):
        """Wait for a search index to be ready."""
        indices: list[Mapping[str, Any]] = []