From 63dc749d8f76f0629dce11db39c5d78473ade314 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 8 Apr 2026 10:24:40 -0500 Subject: [PATCH] SERVER-109005 add rapidyaml wheels (#51312) GitOrigin-RevId: 3719dab73eb82e78d41608717557725acc4cb001 --- buildscripts/mongo_rapidyaml_builds/README.md | 103 ++++++++ .../build_rapidyaml_macos.sh | 189 ++++++++++++++ .../build_rapidyaml_manylinux2014.sh | 122 +++++++++ .../build_rapidyaml_windows_x64.ps1 | 236 ++++++++++++++++++ poetry.lock | 27 +- pyproject.toml | 9 +- 6 files changed, 675 insertions(+), 11 deletions(-) create mode 100644 buildscripts/mongo_rapidyaml_builds/README.md create mode 100755 buildscripts/mongo_rapidyaml_builds/build_rapidyaml_macos.sh create mode 100755 buildscripts/mongo_rapidyaml_builds/build_rapidyaml_manylinux2014.sh create mode 100644 buildscripts/mongo_rapidyaml_builds/build_rapidyaml_windows_x64.ps1 diff --git a/buildscripts/mongo_rapidyaml_builds/README.md b/buildscripts/mongo_rapidyaml_builds/README.md new file mode 100644 index 00000000000..7e3394e5e15 --- /dev/null +++ b/buildscripts/mongo_rapidyaml_builds/README.md @@ -0,0 +1,103 @@ +# mongo rapidyaml wheel builds + +This directory contains scripts to produce versioned `rapidyaml` wheels that can be uploaded to S3 and consumed directly instead of building from the git dependency in `pyproject.toml`. + +The scripts default to the `rapidyaml` commit currently pinned in `pyproject.toml`: + +- `a5d485fd44719e1c03e059177fc1f695fc462b66` + +They also require `RAPIDYAML_VERSION` to be set explicitly. The MongoDB fork does not currently publish git tags, so `setuptools-scm` cannot infer a stable release version on its own. + +All artifacts are written to `dist/`. + +## Contents + +| Script | Platform | Output | +| :--------------------------------- | :---------------------------------------------- | :--------------------- | +| `build_rapidyaml_manylinux2014.sh` | Linux (`x86_64`, `aarch64`, `s390x`, `ppc64le`) | `dist/rapidyaml-*.whl` | +| `build_rapidyaml_macos.sh` | macOS host arch (`x86_64` or `arm64`) | `dist/rapidyaml-*.whl` | +| `build_rapidyaml_windows_x64.ps1` | Windows x86_64 | `dist/rapidyaml-*.whl` | + +## Quick Start + +### Linux (manylinux2014) + +Requirements: Docker. + +To cross-build non-native Linux architectures via QEMU, enable `binfmt` once: + +```bash +docker run --privileged --rm tonistiigi/binfmt --install all +``` + +Build for the host Linux arch: + +```bash +RAPIDYAML_VERSION=0.9.0.post0 ./build_rapidyaml_manylinux2014.sh +``` + +Cross-build specific Linux wheels: + +```bash +RAPIDYAML_VERSION=0.9.0.post0 ARCH=x86_64 PLATFORM=linux/amd64 ./build_rapidyaml_manylinux2014.sh +RAPIDYAML_VERSION=0.9.0.post0 ARCH=aarch64 PLATFORM=linux/arm64 ./build_rapidyaml_manylinux2014.sh +RAPIDYAML_VERSION=0.9.0.post0 ARCH=s390x PLATFORM=linux/s390x ./build_rapidyaml_manylinux2014.sh +RAPIDYAML_VERSION=0.9.0.post0 ARCH=ppc64le PLATFORM=linux/ppc64le ./build_rapidyaml_manylinux2014.sh +``` + +### macOS + +Run the script on each target macOS architecture you want to publish. The script intentionally builds for the host arch only, which keeps wheel tags and interpreter usage straightforward. + +The script creates and uses a temporary virtualenv, so it works with Homebrew-managed Python installations that reject direct `pip install` into the system environment. + +It also leaves `Python.framework` external during delocation, so the wheel should be built with the same Python distribution family you expect consumers to use. + +```bash +RAPIDYAML_VERSION=0.9.0.post0 PYTHON_BIN=python3.13 ./build_rapidyaml_macos.sh +``` + +### Windows x86_64 + +Run in Developer PowerShell for Visual Studio so `cl.exe` is available: + +```powershell +$env:RAPIDYAML_VERSION = "0.9.0.post0" +$env:PYTHON_BIN = "C:\Python313\python.exe" +.\build_rapidyaml_windows_x64.ps1 +``` + +Note: `pyproject.toml` currently excludes `rapidyaml` on Windows, so a Windows wheel is only needed if that marker changes later. + +## Build Behavior + +- The Linux script builds inside the appropriate `manylinux2014` image and runs `auditwheel repair`. +- The macOS script creates a temporary virtualenv, installs its build tooling there, and runs `delocate-wheel` while excluding `Python.framework` from bundling. +- The Windows script runs `delvewheel repair` after building. +- Every script clones the `mongodb-forks/rapidyaml` repo, checks out the requested ref, initializes submodules, builds a wheel, and performs a simple `import ryml` smoke test. +- Linux defaults to `cp313-cp313`, which matches the repo's current Python version. Override that when you need a wheel for a different interpreter. + +## Environment Variables + +| Variable | Purpose | Default | +| :------------------------- | :------------------------------------------------ | :----------------------------------------------- | +| `RAPIDYAML_REPO` | Git repo to clone | `https://github.com/mongodb-forks/rapidyaml.git` | +| `RAPIDYAML_REF` | Branch, tag, or commit to build | `a5d485fd44719e1c03e059177fc1f695fc462b66` | +| `RAPIDYAML_VERSION` | Explicit wheel version passed to `setuptools-scm` | required | +| `OUT_DIR` | Output directory | `./dist` | +| `PYTHON_TAG` | manylinux Python interpreter tag | `cp313-cp313` | +| `PYTHON_BIN` | Host Python executable for macOS or Windows | `python3` on macOS, `python` on Windows | +| `ARCH` | Target architecture | host arch | +| `PLATFORM` | Docker platform override for Linux | auto | +| `CPU_FLAGS` | Extra C/C++ CPU portability flags | platform-specific defaults | +| `MACOSX_DEPLOYMENT_TARGET` | macOS minimum OS version | `10.13` on x86_64, `11.0` on arm64 | + +## Consuming the Wheels + +Once the wheels are uploaded, you can replace the current git dependency in `pyproject.toml` with URL-based entries scoped by platform markers. + +For example: + +```toml +rapidyaml = { url = "https://your-bucket/rapidyaml/0.9.0.post0/rapidyaml-0.9.0.post0-cp313-cp313-manylinux2014_x86_64.whl", markers = "platform_system == 'Linux' and platform_machine == 'x86_64'" } +``` diff --git a/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_macos.sh b/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_macos.sh new file mode 100755 index 00000000000..cf0f7984af4 --- /dev/null +++ b/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_macos.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +RAPIDYAML_REPO="${RAPIDYAML_REPO:-https://github.com/mongodb-forks/rapidyaml.git}" +RAPIDYAML_REF="${RAPIDYAML_REF:-a5d485fd44719e1c03e059177fc1f695fc462b66}" +RAPIDYAML_VERSION="${RAPIDYAML_VERSION:-}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +OUT_DIR="${OUT_DIR:-$(pwd)/dist}" +ARCH="${ARCH:-$(uname -m)}" +HOST_ARCH="$(uname -m)" +CPU_FLAGS="${CPU_FLAGS:-}" + +if [[ -z "$RAPIDYAML_VERSION" ]]; then + echo "RAPIDYAML_VERSION must be set (for example: 0.9.0.post0)." >&2 + exit 1 +fi + +case "$HOST_ARCH" in +x86_64 | amd64) + HOST_ARCH=x86_64 + ;; +arm64 | aarch64) + HOST_ARCH=arm64 + ;; +*) + echo "Unsupported host arch '$HOST_ARCH'. Expected x86_64 or arm64." >&2 + exit 1 + ;; +esac + +case "$ARCH" in +x86_64 | amd64) + ARCH=x86_64 + MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-10.13}" + CPU_FLAGS="${CPU_FLAGS:--march=x86-64 -mtune=generic}" + ;; +arm64 | aarch64) + ARCH=arm64 + MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-11.0}" + ;; +*) + echo "Unsupported ARCH='$ARCH'. Expected x86_64 or arm64." >&2 + exit 1 + ;; +esac + +if [[ "$ARCH" != "$HOST_ARCH" ]]; then + echo "build_rapidyaml_macos.sh must run natively on the target macOS arch." >&2 + echo "Requested ARCH='$ARCH', but host is '$HOST_ARCH'." >&2 + exit 1 +fi + +mkdir -p "$OUT_DIR" + +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rapidyaml-build.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT +VENV_DIR="$TMP_DIR/venv" +ENV_PYTHON="$VENV_DIR/bin/python" +BUILD_REQUIREMENTS_FILE="$TMP_DIR/build-requirements.txt" + +SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" +COMMON_CFLAGS="-mmacosx-version-min=$MACOSX_DEPLOYMENT_TARGET" +COMMON_LDFLAGS="-mmacosx-version-min=$MACOSX_DEPLOYMENT_TARGET" +ARCHFLAGS_VALUE="" +CMAKE_OSX_ARCHITECTURES_VALUE="" +CMAKE_FLAGS_VALUE="${CMAKE_FLAGS:-}" + +if [[ "$ARCH" == "x86_64" ]]; then + COMMON_CFLAGS="-arch $ARCH $COMMON_CFLAGS" + COMMON_LDFLAGS="-arch $ARCH $COMMON_LDFLAGS" + ARCHFLAGS_VALUE="-arch $ARCH" + CMAKE_OSX_ARCHITECTURES_VALUE="$ARCH" +else + # c4core's older TargetArchitecture.cmake rejects explicit "arm64" on macOS. + # Build natively on Apple Silicon and tell the project logic to treat the CPU as aarch64. + CMAKE_FLAGS_VALUE="${CMAKE_FLAGS_VALUE:+$CMAKE_FLAGS_VALUE }-DCMAKE_SYSTEM_PROCESSOR=aarch64" +fi + +if [[ -n "$CPU_FLAGS" ]]; then + COMMON_CFLAGS="$COMMON_CFLAGS $CPU_FLAGS" +fi + +echo "==> Build rapidyaml wheel for macOS ($ARCH)" +echo " RAPIDYAML_REF=$RAPIDYAML_REF" +echo " RAPIDYAML_VERSION=$RAPIDYAML_VERSION" +echo " PYTHON_BIN=$PYTHON_BIN" +echo " MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET" +[ -n "$CPU_FLAGS" ] && echo " CPU_FLAGS=$CPU_FLAGS" +[ -n "$CMAKE_FLAGS_VALUE" ] && echo " CMAKE_FLAGS=$CMAKE_FLAGS_VALUE" + +git clone "$RAPIDYAML_REPO" "$TMP_DIR/rapidyaml" +cd "$TMP_DIR/rapidyaml" +git -c advice.detachedHead=false checkout "$RAPIDYAML_REF" +git submodule update --init --recursive + +"$PYTHON_BIN" -m venv "$VENV_DIR" +if [[ ! -x "$ENV_PYTHON" ]]; then + echo "Failed to create virtualenv at $VENV_DIR" >&2 + exit 1 +fi + +export SETUPTOOLS_SCM_PRETEND_VERSION="$RAPIDYAML_VERSION" +export SDKROOT +if [[ -n "$ARCHFLAGS_VALUE" ]]; then + export ARCHFLAGS="$ARCHFLAGS_VALUE" +else + unset ARCHFLAGS || true +fi +if [[ -n "$CMAKE_OSX_ARCHITECTURES_VALUE" ]]; then + export CMAKE_OSX_ARCHITECTURES="$CMAKE_OSX_ARCHITECTURES_VALUE" +else + unset CMAKE_OSX_ARCHITECTURES || true +fi +export CMAKE_FLAGS="$CMAKE_FLAGS_VALUE" +export MACOSX_DEPLOYMENT_TARGET +export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$(sysctl -n hw.logicalcpu)}" +export CFLAGS="${CFLAGS:+$CFLAGS }$COMMON_CFLAGS" +export CXXFLAGS="${CXXFLAGS:+$CXXFLAGS }$COMMON_CFLAGS" +export LDFLAGS="${LDFLAGS:+$LDFLAGS }$COMMON_LDFLAGS" +export PATH="$VENV_DIR/bin:$PATH" + +"$ENV_PYTHON" -m pip install --upgrade "pip<26" setuptools wheel build delocate "packaging<26" + +"$ENV_PYTHON" - "$BUILD_REQUIREMENTS_FILE" <<'PY' +import pathlib +import sys +import tomllib + + +def normalize_requirement(req_string: str) -> str: + requirement, sep, marker = req_string.partition(";") + requirement = requirement.strip() + + if "~=" in requirement: + name, version = requirement.split("~=", 1) + requirement = f"{name.strip()}=={version.strip()}" + + if sep: + return f"{requirement}; {marker.strip()}" + return requirement + + +data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8")) +pathlib.Path(sys.argv[1]).write_text( + "".join(f"{normalize_requirement(req)}\n" for req in data["build-system"]["requires"]), + encoding="utf-8", +) +PY + +while IFS= read -r requirement; do + [[ -n "$requirement" ]] || continue + echo "==> Installing build dependency $requirement" + "$ENV_PYTHON" -m pip install --upgrade "$requirement" +done <"$BUILD_REQUIREMENTS_FILE" + +SWIG_EXECUTABLE="$(command -v swig || true)" +if [[ -z "$SWIG_EXECUTABLE" ]]; then + echo "swig not found after installing build dependencies." >&2 + exit 1 +fi +export SWIG_EXECUTABLE +export SWIG_DIR="$("$SWIG_EXECUTABLE" -swiglib)" +if [[ -z "$SWIG_DIR" ]]; then + echo "Failed to resolve SWIG_DIR from $SWIG_EXECUTABLE" >&2 + exit 1 +fi + +"$ENV_PYTHON" -m build --wheel --no-isolation --outdir "$TMP_DIR/wheelhouse" + +wheel="$(ls -1 "$TMP_DIR"/wheelhouse/*.whl)" +echo "==> Built wheel: $(basename "$wheel")" +delocate-listdeps "$wheel" + +mkdir -p "$TMP_DIR/repaired" +# Do not vendor the interpreter's Python.framework into the wheel. The +# extension should load libpython from the target Python installation. +delocate-wheel --exclude "Python.framework" -w "$TMP_DIR/repaired" -v "$wheel" + +repaired_wheel="$(ls -1 "$TMP_DIR"/repaired/*.whl)" +cp "$repaired_wheel" "$OUT_DIR/" +echo "==> Wrote $OUT_DIR/$(basename "$repaired_wheel")" + +"$ENV_PYTHON" -m pip install --force-reinstall "$repaired_wheel" +"$ENV_PYTHON" - <<'PY' +import ryml +print("Imported ryml from", ryml.__file__) +PY + +shasum -a 256 "$repaired_wheel" diff --git a/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_manylinux2014.sh b/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_manylinux2014.sh new file mode 100755 index 00000000000..6036ca66a42 --- /dev/null +++ b/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_manylinux2014.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +# -------- config (overridable via env) -------------------------------------- +ARCH="${ARCH:-$(uname -m)}" # x86_64 | aarch64 | s390x | ppc64le +RAPIDYAML_REPO="${RAPIDYAML_REPO:-https://github.com/mongodb-forks/rapidyaml.git}" +RAPIDYAML_REF="${RAPIDYAML_REF:-a5d485fd44719e1c03e059177fc1f695fc462b66}" +RAPIDYAML_VERSION="${RAPIDYAML_VERSION:-}" +PYTHON_TAG="${PYTHON_TAG:-cp313-cp313}" +OUT_DIR="${OUT_DIR:-$(pwd)/dist}" +PLATFORM="${PLATFORM:-}" # e.g. linux/arm64 if you want to force Docker +CPU_FLAGS="${CPU_FLAGS:-}" +DOCKER_IMAGE="" +AUDITWHEEL_PLAT="" + +if [[ -z "$RAPIDYAML_VERSION" ]]; then + echo "RAPIDYAML_VERSION must be set (for example: 0.9.0.post0)." >&2 + exit 1 +fi + +case "$ARCH" in +x86_64 | amd64) + ARCH=x86_64 + DOCKER_IMAGE="quay.io/pypa/manylinux2014_x86_64" + AUDITWHEEL_PLAT="manylinux2014_x86_64" + CPU_FLAGS="${CPU_FLAGS:--march=x86-64 -mtune=generic}" + ;; +aarch64 | arm64) + ARCH=aarch64 + DOCKER_IMAGE="quay.io/pypa/manylinux2014_aarch64" + AUDITWHEEL_PLAT="manylinux2014_aarch64" + CPU_FLAGS="${CPU_FLAGS:--mcpu=generic}" + ;; +s390x | 390x) + ARCH=s390x + DOCKER_IMAGE="quay.io/pypa/manylinux2014_s390x" + AUDITWHEEL_PLAT="manylinux2014_s390x" + ;; +ppc64le | ppc) + ARCH=ppc64le + DOCKER_IMAGE="quay.io/pypa/manylinux2014_ppc64le" + AUDITWHEEL_PLAT="manylinux2014_ppc64le" + ;; +*) + echo "Unsupported ARCH='$ARCH'. Expected x86_64|aarch64|s390x|ppc64le." >&2 + exit 1 + ;; +esac + +mkdir -p "$OUT_DIR" + +echo "==> Build rapidyaml wheel for manylinux2014 ($ARCH)" +echo " RAPIDYAML_REF=$RAPIDYAML_REF" +echo " RAPIDYAML_VERSION=$RAPIDYAML_VERSION" +echo " PYTHON_TAG=$PYTHON_TAG" +echo " Image: $DOCKER_IMAGE" +[ -n "$PLATFORM" ] && echo " docker --platform: $PLATFORM" +[ -n "$CPU_FLAGS" ] && echo " CPU_FLAGS=$CPU_FLAGS" + +PLATFORM_ARGS=() +[ -n "$PLATFORM" ] && PLATFORM_ARGS=(--platform "$PLATFORM") + +docker run --rm -t "${PLATFORM_ARGS[@]}" \ + -e RAPIDYAML_REPO="$RAPIDYAML_REPO" \ + -e RAPIDYAML_REF="$RAPIDYAML_REF" \ + -e RAPIDYAML_VERSION="$RAPIDYAML_VERSION" \ + -e PYTHON_TAG="$PYTHON_TAG" \ + -e AUDITWHEEL_PLAT="$AUDITWHEEL_PLAT" \ + -e CPU_FLAGS="$CPU_FLAGS" \ + -v "$OUT_DIR":/out \ + "$DOCKER_IMAGE" \ + bash -lc ' + set -euo pipefail + + PYBIN="/opt/python/${PYTHON_TAG}/bin/python" + if [[ ! -x "$PYBIN" ]]; then + echo "Python interpreter not found: $PYBIN" >&2 + exit 1 + fi + + echo "==> glibc baseline:" + ldd --version > /tmp/lddv && sed -n "1p" /tmp/lddv + + yum -y install git >/dev/null 2>&1 || true + + rm -rf /tmp/rapidyaml /tmp/wheelhouse /tmp/repaired + git clone "$RAPIDYAML_REPO" /tmp/rapidyaml + cd /tmp/rapidyaml + git -c advice.detachedHead=false checkout "$RAPIDYAML_REF" + git submodule update --init --recursive + + export SETUPTOOLS_SCM_PRETEND_VERSION="$RAPIDYAML_VERSION" + export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)}" + if [[ -n "${CPU_FLAGS}" ]]; then + export CFLAGS="${CFLAGS:+$CFLAGS }${CPU_FLAGS}" + export CXXFLAGS="${CXXFLAGS:+$CXXFLAGS }${CPU_FLAGS}" + fi + + "$PYBIN" -m pip install --upgrade pip build auditwheel + "$PYBIN" -m build --wheel --outdir /tmp/wheelhouse + + wheel="$(ls -1 /tmp/wheelhouse/*.whl)" + echo "==> Built wheel: $(basename "$wheel")" + auditwheel show "$wheel" + + mkdir -p /tmp/repaired + auditwheel repair --plat "$AUDITWHEEL_PLAT" --wheel-dir /tmp/repaired "$wheel" + + repaired_wheel="$(ls -1 /tmp/repaired/*.whl)" + cp "$repaired_wheel" /out/ + echo "==> Wrote /out/$(basename "$repaired_wheel")" + + "$PYBIN" -m pip install --force-reinstall "$repaired_wheel" + "$PYBIN" - <<'"'"'PY'"'"' +import ryml +print("Imported ryml from", ryml.__file__) +PY + + sha256sum "$repaired_wheel" + ' + +echo "Built wheels in: $OUT_DIR" diff --git a/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_windows_x64.ps1 b/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_windows_x64.ps1 new file mode 100644 index 00000000000..34e4370ab9d --- /dev/null +++ b/buildscripts/mongo_rapidyaml_builds/build_rapidyaml_windows_x64.ps1 @@ -0,0 +1,236 @@ +<# build_rapidyaml_windows_x64.ps1 + Builds a rapidyaml wheel for x86_64-pc-windows-msvc. + Output: .\dist\rapidyaml-*.whl +#> + +$ErrorActionPreference = "Stop" + +# ---- Config (override via env before running) ------------------------------- +$RAPIDYAML_REPO = if ($env:RAPIDYAML_REPO) { $env:RAPIDYAML_REPO } else { "https://github.com/mongodb-forks/rapidyaml.git" } +$RAPIDYAML_REF = if ($env:RAPIDYAML_REF) { $env:RAPIDYAML_REF } else { "a5d485fd44719e1c03e059177fc1f695fc462b66" } +$RAPIDYAML_VERSION = if ($env:RAPIDYAML_VERSION) { $env:RAPIDYAML_VERSION } else { "" } +$OUT_DIR = if ($env:OUT_DIR) { $env:OUT_DIR } else { Join-Path (Get-Location) "dist" } +$PYTHON_BIN = if ($env:PYTHON_BIN) { $env:PYTHON_BIN } else { "python" } + +function Import-MsvcEnvironment { + if (Get-Command cl.exe -ErrorAction SilentlyContinue) { + return $true + } + + $CandidateBatches = @() + if ($env:VCVARS64_BAT) { + $CandidateBatches += @{ + Path = $env:VCVARS64_BAT + Args = "" + } + } + if ($env:VSDEVCMD_BAT) { + $CandidateBatches += @{ + Path = $env:VSDEVCMD_BAT + Args = "-arch=x64 -host_arch=x64" + } + } + + $VswhereCandidates = @( + (Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"), + (Join-Path $env:ProgramFiles "Microsoft Visual Studio\Installer\vswhere.exe") + ) + + foreach ($Vswhere in $VswhereCandidates) { + if (-not $Vswhere -or -not (Test-Path $Vswhere)) { + continue + } + + $InstallPath = & $Vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($InstallPath)) { + continue + } + + $CandidateBatches += @{ + Path = (Join-Path $InstallPath "VC\Auxiliary\Build\vcvars64.bat") + Args = "" + } + $CandidateBatches += @{ + Path = (Join-Path $InstallPath "Common7\Tools\VsDevCmd.bat") + Args = "-arch=x64 -host_arch=x64" + } + } + + foreach ($Edition in @("Enterprise", "Professional", "Community", "BuildTools")) { + foreach ($Year in @("2022", "2019")) { + $Root = Join-Path ${env:ProgramFiles} "Microsoft Visual Studio\$Year\$Edition" + $CandidateBatches += @{ + Path = (Join-Path $Root "VC\Auxiliary\Build\vcvars64.bat") + Args = "" + } + $CandidateBatches += @{ + Path = (Join-Path $Root "Common7\Tools\VsDevCmd.bat") + Args = "-arch=x64 -host_arch=x64" + } + } + } + + foreach ($Candidate in $CandidateBatches) { + if (-not $Candidate.Path -or -not (Test-Path $Candidate.Path)) { + continue + } + + Write-Host "==> Importing MSVC environment from $($Candidate.Path)" + $Command = "call `"$($Candidate.Path)`"" + if ($Candidate.Args) { + $Command += " $($Candidate.Args)" + } + $Command += " >nul && set" + + $EnvLines = & cmd.exe /d /s /c $Command + if ($LASTEXITCODE -ne 0) { + continue + } + + foreach ($Line in $EnvLines) { + if ($Line -match "^([^=]+)=(.*)$") { + [Environment]::SetEnvironmentVariable($matches[1], $matches[2], "Process") + } + } + + if (Get-Command cl.exe -ErrorAction SilentlyContinue) { + return $true + } + } + + return $false +} + +if ([string]::IsNullOrWhiteSpace($RAPIDYAML_VERSION)) { + throw "RAPIDYAML_VERSION must be set (for example: 0.9.0.post0)." +} + +New-Item -ItemType Directory -Force -Path $OUT_DIR | Out-Null + +Write-Host "==> rapidyaml Windows x86_64" +Write-Host " RAPIDYAML_REF=$RAPIDYAML_REF" +Write-Host " RAPIDYAML_VERSION=$RAPIDYAML_VERSION" +Write-Host " OUT_DIR=$OUT_DIR" +Write-Host " PYTHON_BIN=$PYTHON_BIN" + +# ---- Tooling sanity --------------------------------------------------------- +if (-not (Import-MsvcEnvironment)) { + throw "MSVC (cl.exe) not found in PATH. Run this in 'Developer PowerShell for VS' or install VS Build Tools." +} +if (-not (Get-Command git.exe -ErrorAction SilentlyContinue)) { + throw "git.exe not found in PATH." +} + +$BuildRootBase = if ($env:RAPIDYAML_BUILD_ROOT) { $env:RAPIDYAML_BUILD_ROOT } else { Join-Path $env:SystemDrive "tmp" } +New-Item -ItemType Directory -Force -Path $BuildRootBase | Out-Null + +$BuildRoot = Join-Path $BuildRootBase ("rapidyaml-build-" + [System.Guid]::NewGuid().ToString("N")) +$RepoDir = Join-Path $BuildRoot "rapidyaml" +$VenvDir = Join-Path $BuildRoot "venv" +$WheelhouseDir = Join-Path $BuildRoot "wheelhouse" +$RepairedDir = Join-Path $BuildRoot "repaired" +$PushedLocation = $false +$OriginalPath = $env:Path + +New-Item -ItemType Directory -Force -Path $BuildRoot, $WheelhouseDir, $RepairedDir | Out-Null + +try { + & $PYTHON_BIN -m venv $VenvDir + $EnvPython = Join-Path $VenvDir "Scripts\python.exe" + if (-not (Test-Path $EnvPython)) { + throw "Failed to create virtualenv at $VenvDir." + } + + git clone $RAPIDYAML_REPO $RepoDir | Out-Null + + Push-Location $RepoDir + $PushedLocation = $true + + git -c advice.detachedHead=false checkout $RAPIDYAML_REF | Out-Null + git submodule update --init --recursive | Out-Null + + $env:SETUPTOOLS_SCM_PRETEND_VERSION = $RAPIDYAML_VERSION + if (-not $env:CMAKE_BUILD_PARALLEL_LEVEL) { + $env:CMAKE_BUILD_PARALLEL_LEVEL = $env:NUMBER_OF_PROCESSORS + } + $env:CMAKE_GENERATOR = "Ninja" + + & $EnvPython -m pip install --upgrade "pip<26" setuptools wheel build delvewheel "packaging<26" + $env:Path = "$(Join-Path $VenvDir 'Scripts');$env:Path" + + $BuildRequirementsJson = @' +import json +import pathlib +import tomllib + + +def normalize_requirement(req_string: str) -> str: + requirement, sep, marker = req_string.partition(";") + requirement = requirement.strip() + + if "~=" in requirement: + name, version = requirement.split("~=", 1) + requirement = f"{name.strip()}=={version.strip()}" + + if sep: + return f"{requirement}; {marker.strip()}" + return requirement + + +data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8")) +print(json.dumps([normalize_requirement(req) for req in data["build-system"]["requires"]])) +'@ | & $EnvPython - + $BuildRequirements = $BuildRequirementsJson | ConvertFrom-Json + if (-not $BuildRequirements) { + throw "Unable to read build-system.requires from pyproject.toml." + } + + foreach ($Requirement in $BuildRequirements) { + Write-Host "==> Installing build dependency $Requirement" + & $EnvPython -m pip install --upgrade $Requirement + } + + $SwigExecutable = Get-Command swig.exe -ErrorAction SilentlyContinue + if (-not $SwigExecutable) { + throw "swig.exe not found after installing build dependencies." + } + $env:SWIG_EXECUTABLE = $SwigExecutable.Source + $env:SWIG_DIR = (& $env:SWIG_EXECUTABLE -swiglib).Trim() + if (-not $env:SWIG_DIR) { + throw "Failed to resolve SWIG_DIR from $($env:SWIG_EXECUTABLE)." + } + + & $EnvPython -m build --wheel --no-isolation --outdir $WheelhouseDir + + $Wheel = Get-ChildItem -Path $WheelhouseDir -Filter *.whl | Select-Object -First 1 + if (-not $Wheel) { + throw "No wheel produced in $WheelhouseDir." + } + + Write-Host "==> Built wheel: $($Wheel.Name)" + & $EnvPython -m delvewheel show $Wheel.FullName + & $EnvPython -m delvewheel repair --wheel-dir $RepairedDir $Wheel.FullName + + $RepairedWheel = Get-ChildItem -Path $RepairedDir -Filter *.whl | Select-Object -First 1 + if (-not $RepairedWheel) { + throw "No repaired wheel produced in $RepairedDir." + } + + Copy-Item $RepairedWheel.FullName $OUT_DIR -Force + Write-Host "==> Wrote $(Join-Path $OUT_DIR $RepairedWheel.Name)" + + & $EnvPython -m pip install --force-reinstall $RepairedWheel.FullName + & $EnvPython -c "import ryml; print('Imported ryml from', ryml.__file__)" + + try { + $Sha = (Get-FileHash -Algorithm SHA256 $RepairedWheel.FullName).Hash + Write-Host "SHA256 $Sha $($RepairedWheel.Name)" + } catch { } +} +finally { + $env:Path = $OriginalPath + if ($PushedLocation) { + Pop-Location + } + Remove-Item -LiteralPath $BuildRoot -Recurse -Force -ErrorAction SilentlyContinue +} diff --git a/poetry.lock b/poetry.lock index 26f3a202a88..d44bea98d6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -773,7 +773,7 @@ description = "A library to handle automated deprecations" optional = false python-versions = "*" groups = ["powercycle-incompatible"] -markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and platform_system != \"Windows\"" +markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and python_version == \"3.13\"" files = [ {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, @@ -3895,23 +3895,30 @@ all = ["numpy"] [[package]] name = "rapidyaml" -version = "0.0.post1671" +version = "0.9.0.post0" description = "Rapid YAML - a library to parse and emit YAML, and do it fast" optional = false python-versions = ">=3.6" groups = ["powercycle-incompatible"] -markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and platform_system != \"Windows\"" -files = [] -develop = false +markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and python_version == \"3.13\"" +files = [ + {file = "rapidyaml-0.9.0.post0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5276ca91c401d5a8c0a2a97b3a63d7d42af7ed79be0aaa223ab73cdeb1ec35f1"}, + {file = "rapidyaml-0.9.0.post0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6c0136ae1e3f06e9e5e123cbb8b3646507259c0292ed76ac152c302c40856cd6"}, + {file = "rapidyaml-0.9.0.post0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8334c27797f118770224a161578c8213575b209a5f25e730f489be6731033969"}, + {file = "rapidyaml-0.9.0.post0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f92c2c5ac8657d8a448cefc2ec1817ad938696aacb0521d12323118179b97ce2"}, + {file = "rapidyaml-0.9.0.post0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:e13c0dbf866f67dc496cef709e08fd844fb9ed34d16eff59ada8ef960981dc08"}, + {file = "rapidyaml-0.9.0.post0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87bef46509479e2f54f5177bd25579012244a2b67aa294963bf7ce08c2f67798"}, + {file = "rapidyaml-0.9.0.post0-cp313-cp313-win_amd64.whl", hash = "sha256:d03a9ea4d600a96277e2fc9d49aa0caf58ae93d063ba267cf5c97a0d11b75331"}, + {file = "rapidyaml-0.9.0.post0.tar.gz", hash = "sha256:4335e7036b31e5bc55c992f96af373e429e886c2e8f447e00a7da5563d5f8fbc"}, +] [package.dependencies] deprecation = "*" [package.source] -type = "git" -url = "https://github.com/mongodb-forks/rapidyaml.git" -reference = "a5d485fd44719e1c03e059177fc1f695fc462b66" -resolved_reference = "a5d485fd44719e1c03e059177fc1f695fc462b66" +type = "legacy" +url = "https://mdb-build-public.s3.amazonaws.com/rapidyaml_wheels/simple" +reference = "mdb-build-public" [[package]] name = "referencing" @@ -5820,4 +5827,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog [metadata] lock-version = "2.1" python-versions = ">=3.13,<4.0" -content-hash = "9bbe30cd9b6800652ce9fc61a7af198d335ea9ef1f8d44cd6dcadd8f2ebcffcf" +content-hash = "dcacf326ad3ce557fba66779efea9015eb7e09ec54987f7ff2f1919a16161da4" diff --git a/pyproject.toml b/pyproject.toml index 40cad6139cd..5b282d4731e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,11 @@ cxxfilt = { version = "*", optional = true } pympler = { version = "*", optional = true } pyright = "1.1.393" +[[tool.poetry.source]] +name = "mdb-build-public" +url = "https://mdb-build-public.s3.amazonaws.com/rapidyaml_wheels/simple/" +priority = "explicit" + [tool.poetry.group.aws.dependencies] boto3 = "^1.34.156" botocore = "^1.34.156" @@ -76,8 +81,10 @@ tenacity = "^9.0.0" # specifically rapidyaml is broken on atlas distros with powercycle. # current we exclude this when running poetry install in powercycle. +# Use the S3 simple index so Poetry can pick a matching wheel or fall back to +# the sdist when a wheel is unavailable for the current platform. [tool.poetry.group.powercycle-incompatible.dependencies] -rapidyaml = {git = "https://github.com/mongodb-forks/rapidyaml.git@master", rev = "a5d485fd44719e1c03e059177fc1f695fc462b66", markers = "platform_system != 'Windows'"} +rapidyaml = { version = "==0.9.0.post0", source = "mdb-build-public", markers = "python_version == '3.13'" } [tool.poetry.group.export.dependencies] pipx = "1.6.0"