PYTHON-4540 Convert libmongocrypt download to python (#2148)

.evergreen/run-azurekms-fail-test.sh

@@ -2,10 +2,8 @@
 set -o errexit  # Exit the script with error if any of the commands fail
 HERE=$(dirname ${BASH_SOURCE:-$0})
 . $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
-export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh
-PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
-    KEY_NAME="${AZUREKMS_KEYNAME}" \
+KEY_NAME="${AZUREKMS_KEYNAME}" \
     KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
     $HERE/just.sh test-eg
 bash $HERE/scripts/teardown-tests.sh

.evergreen/run-azurekms-test.sh

@@ -6,7 +6,6 @@ echo "Copying files ... begin"
 export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
 export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
 export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
-LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 # Set up the remote files to test.
 git add .
 git commit -m "add files" || true
@@ -20,7 +19,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \
+AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash .evergreen/just.sh setup-test" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh

.evergreen/run-gcpkms-test.sh

@@ -8,7 +8,6 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
 export GCPKMS_PROJECT=${GCPKMS_PROJECT}
 export GCPKMS_ZONE=${GCPKMS_ZONE}
 export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
-LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 # Set up the remote files to test.
 git add .
 git commit -m "add files" || true
@@ -19,7 +18,7 @@ echo "Untarring file ... begin"
 GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Running test ... end"
 bash $HERE/scripts/teardown-tests.sh

.evergreen/scripts/run-gcpkms-fail-test.sh

@@ -2,7 +2,5 @@
 set -eu
 HERE=$(dirname ${BASH_SOURCE:-$0})
 . $HERE/env.sh
-export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
-export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh
 bash ./.evergreen/just.sh test-eg

.evergreen/scripts/setup-libmongocrypt.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-set -o errexit  # Exit the script with error if any of the commands fail
-set -o xtrace
-
-TARGET=""
-
-if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
-    # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
-    powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
-    TARGET="windows-test"
-fi
-
-if [ "$(uname -s)" = "Darwin" ]; then
-    TARGET="macos"
-fi
-
-if [ "$(uname -s)" = "Linux" ]; then
-    rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release)
-    arch=$(uname -m)
-    echo "RHEL $rhel_ver $arch"
-    if [[ $rhel_ver =~ 7 ]]; then
-        TARGET="rhel-70-64-bit"
-    elif [[ $rhel_ver =~ 8 ]]; then
-        if [ "$arch" = "x86_64" ]; then
-            TARGET="rhel-80-64-bit"
-        elif [ "$arch" = "arm" ]; then
-            TARGET="rhel-82-arm64"
-        fi
-    fi
-fi
-
-if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then
-    LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz"
-fi
-
-if [ -z "$LIBMONGOCRYPT_URL" ]; then
-    echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
-    exit 1
-fi
-rm -rf libmongocrypt libmongocrypt.tar.gz
-echo "Fetching $LIBMONGOCRYPT_URL..."
-curl -O "$LIBMONGOCRYPT_URL"
-echo "Fetching $LIBMONGOCRYPT_URL...done"
-mkdir libmongocrypt
-tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
-ls -la libmongocrypt
-ls -la libmongocrypt/nocrypto
-
-if [ "Windows_NT" = "${OS:-''}" ]; then
-    # libmongocrypt's windows dll is not marked executable.
-    chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll
-fi

.evergreen/scripts/setup-tests.sh

@@ -58,4 +58,4 @@ fi
 
 . $ROOT_DIR/.evergreen/utils.sh
 PYTHON=${PYTHON_BINARY:-$(find_python3)}
-$PYTHON $SCRIPT_DIR/setup-tests.py
+$PYTHON $SCRIPT_DIR/setup_tests.py

.evergreen/scripts/setup_tests.py

@@ -1,20 +1,26 @@
 from __future__ import annotations
 
 import base64
+import dataclasses
+import io
 import logging
 import os
 import platform
 import shlex
+import shutil
 import stat
 import subprocess
 import sys
+import tarfile
 from pathlib import Path
 from typing import Any
+from urllib import request
 
 HERE = Path(__file__).absolute().parent
 ROOT = HERE.parent.parent
 ENV_FILE = HERE / "test-env.sh"
 DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
+PLATFORM = "windows" if os.name == "nt" else sys.platform
 
 logging.basicConfig(level=logging.INFO)
 LOGGER = logging.getLogger(__name__)
@@ -74,6 +80,13 @@ EXTRAS_MAP = dict(
 )
 
 
+@dataclasses.dataclass
+class Distro:
+    name: str
+    version_id: str
+    arch: str
+
+
 def write_env(name: str, value: Any) -> None:
     with ENV_FILE.open("a", newline="\n") as fid:
         # Remove any existing quote chars.
@@ -92,6 +105,69 @@ def run_command(cmd: str) -> None:
     LOGGER.info("Running command %s... done.", cmd)
 
 
+def get_distro() -> Distro:
+    name = ""
+    version_id = ""
+    arch = platform.machine()
+    with open("/etc/os-release") as fid:
+        for line in fid.readlines():
+            line = line.replace('"', "")  # noqa: PLW2901
+            if line.startswith("NAME="):
+                _, _, name = line.strip().partition("=")
+            if line.startswith("VERSION_ID="):
+                _, _, version_id = line.strip().partition("=")
+    return Distro(name=name, version_id=version_id, arch=arch)
+
+
+def setup_libmongocrypt():
+    target = ""
+    if PLATFORM == "windows":
+        # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
+        if is_set("TEST_FLE_GCP_AUTO"):
+            run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"')
+        target = "windows-test"
+
+    elif PLATFORM == "darwin":
+        target = "macos"
+
+    else:
+        distro = get_distro()
+        if distro.name.startswith("Debian"):
+            target = f"debian{distro.version_id}"
+        elif distro.name.startswith("Red Hat"):
+            if distro.version_id.startswith("7"):
+                target = "rhel-70-64-bit"
+            elif distro.version_id.startswith("8"):
+                if distro.arch == "aarch64":
+                    target = "rhel-82-arm64"
+                else:
+                    target = "rhel-80-64-bit"
+
+    if not is_set("LIBMONGOCRYPT_URL"):
+        if not target:
+            raise ValueError("Cannot find libmongocrypt target for current platform!")
+        url = f"https://s3.amazonaws.com/mciuploads/libmongocrypt/{target}/master/latest/libmongocrypt.tar.gz"
+    else:
+        url = os.environ["LIBMONGOCRYPT_URL"]
+
+    shutil.rmtree(HERE / "libmongocrypt", ignore_errors=True)
+
+    LOGGER.info(f"Fetching {url}...")
+    with request.urlopen(request.Request(url), timeout=15.0) as response:  # noqa: S310
+        if response.status == 200:
+            fileobj = io.BytesIO(response.read())
+        with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid:
+            fid.extractall(Path.cwd() / "libmongocrypt")
+    LOGGER.info(f"Fetching {url}... done.")
+
+    run_command("ls -la libmongocrypt")
+    run_command("ls -la libmongocrypt/nocrypto")
+
+    if PLATFORM == "windows":
+        # libmongocrypt's windows dll is not marked executable.
+        run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
+
+
 def handle_test_env() -> None:
     AUTH = os.environ.get("AUTH", "noauth")
     SSL = os.environ.get("SSL", "nossl")
@@ -156,7 +232,7 @@ def handle_test_env() -> None:
         write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
 
     if is_set("TEST_ENTERPRISE_AUTH"):
-        if os.name == "nt":
+        if PLATFORM == "windows":
             LOGGER.info("Setting GSSAPI_PASS")
             write_env("GSSAPI_PASS", os.environ["SASL_PASS"])
             write_env("GSSAPI_CANONICALIZE", "true")
@@ -214,19 +290,19 @@ def handle_test_env() -> None:
     if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
         # Check for libmongocrypt download.
         if not (ROOT / "libmongocrypt").exists():
-            run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh")
+            setup_libmongocrypt()
 
         # TODO: Test with 'pip install pymongocrypt'
         UV_ARGS.append("--group pymongocrypt_source")
 
         # Use the nocrypto build to avoid dependency issues with older windows/python versions.
         BASE = ROOT / "libmongocrypt/nocrypto"
-        if sys.platform == "linux":
+        if PLATFORM == "linux":
             if (BASE / "lib/libmongocrypt.so").exists():
                 PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so"
             else:
                 PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so"
-        elif sys.platform == "darwin":
+        elif PLATFORM == "darwin":
             PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib"
         else:
             PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll"
@@ -244,7 +320,7 @@ def handle_test_env() -> None:
     if is_set("TEST_CRYPT_SHARED"):
         CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
         LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
-        if os.name == "nt":
+        if PLATFORM == "windows":
             write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH")
         else:
             write_env(