From 8995a80ec980668be9f3e0fcfe4614139d34ee83 Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Thu, 23 Nov 2023 14:14:21 -0600 Subject: [PATCH] switch to GHA for all actions, including arm64 linux (#668) * switch to GHA for all actions, including arm64 linux * upload the sdist we download from GHA * oops --- .circleci/build-wheel.sh | 33 -------- .circleci/config.yml | 97 ----------------------- .github/workflows/ci.yml | 14 +++- .github/workflows/wheel-builder.yml | 117 ++++++++++++++++++++++------ release.py | 11 +-- 5 files changed, 109 insertions(+), 163 deletions(-) delete mode 100755 .circleci/build-wheel.sh delete mode 100644 .circleci/config.yml diff --git a/.circleci/build-wheel.sh b/.circleci/build-wheel.sh deleted file mode 100755 index 5e2ce83..0000000 --- a/.circleci/build-wheel.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -ex - -cd /test - -echo "Building for ${PLATFORM}" - -PYBIN="/opt/python/${PYTHON}/bin" - -mkdir -p /test/wheelhouse.final - -"${PYBIN}"/python -m venv .venv - -.venv/bin/pip install -U pip wheel setuptools-rust - -.venv/bin/python setup.py sdist -cd dist -tar zxf bcrypt*.tar.gz -rm -rf bcrypt*.tar.gz -cd bcrypt* - -REGEX="cp3([0-9])*" -if [[ "${PYBIN}" =~ $REGEX ]]; then - PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}" -fi - -../../.venv/bin/python setup.py bdist_wheel "$PY_LIMITED_API" - -auditwheel repair --plat "${PLATFORM}" -w wheelhouse/ dist/bcrypt*.whl - -../../.venv/bin/pip install bcrypt --no-index -f wheelhouse/ -../../.venv/bin/python -c "import bcrypt; password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)" - -mv wheelhouse/* /test/wheelhouse.final diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 3cb3c16..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,97 +0,0 @@ -version: 2.1 - -commands: - docker-pull: - parameters: - image: - type: string - steps: - - run: docker pull <> - docker-run: - parameters: - image: - type: string - command: - type: string - steps: - - run: docker run -e PLATFORM -e PYTHON -v $(pwd):/test <> /bin/bash -c 'cd /test;<>' - -jobs: - linux-arm64: - machine: - image: ubuntu-2004:current - resource_class: arm.medium - parameters: - image: - type: string - toxenv: - type: string - steps: - - checkout - - docker-pull: - image: <> - - docker-run: - image: <> - command: /venv/bin/pip install tox && /venv/bin/tox -e <> - linux-arm64-wheel: - machine: - image: ubuntu-2004:current - resource_class: arm.medium - parameters: - image: - type: string - platform: - type: string - python: - type: string - environment: - PLATFORM: <> - PYTHON: <> - steps: - - checkout - - docker-pull: - image: <> - - docker-run: - image: <> - command: /test/.circleci/build-wheel.sh - - store_artifacts: - path: wheelhouse.final - -workflows: - ci: - jobs: - - linux-arm64: - # Changing this name should only be done in conjunction with updating - # the required checks on GH - name: linux-arm64-ci - image: ghcr.io/pyca/cryptography-runner-ubuntu-jammy:aarch64 - toxenv: py310 - # This makes sure it runs on all tags in addition to PRs/branches. - # By default CircleCI ignores tags. - filters: - tags: - only: /.*/ - - linux-arm64-wheel: - name: manylinux2014_aarch64-wheel - image: ghcr.io/pyca/cryptography-manylinux2014_aarch64:latest - python: cp37-cp37m - platform: manylinux2014_aarch64 - filters: - tags: - only: /.*/ - - linux-arm64-wheel: - name: manylinux_2_28_aarch64-wheel - image: ghcr.io/pyca/cryptography-manylinux_2_28:aarch64 - python: cp37-cp37m - platform: manylinux_2_28_aarch64 - filters: - tags: - only: /.*/ - - linux-arm64-wheel: - name: musllinux_1_1_aarch64-wheel - image: ghcr.io/pyca/cryptography-musllinux_1_1:aarch64 - python: cp37-cp37m - platform: musllinux_1_1_aarch64 - filters: - tags: - only: /.*/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7b0f77..c1ba4cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,14 +126,24 @@ jobs: CARGO_TARGET_DIR: ${{ format('{0}/src/_bcrypt/target/', github.workspace) }} linux-distros: - runs-on: ubuntu-latest + runs-on: ${{ matrix.IMAGE.RUNNER }} container: ghcr.io/pyca/cryptography-runner-${{ matrix.IMAGE.IMAGE }} strategy: matrix: IMAGE: - - {IMAGE: "alpine", TOXENV: "py311"} + - {IMAGE: "alpine", TOXENV: "py311", RUNNER: "ubuntu-latest"} + - {IMAGE: "alpine:aarch64", TOXENV: "py311", RUNNER: [self-hosted, Linux, ARM64]} + - {IMAGE: "ubuntu-jammy:aarch64", TOXENV: "py310", RUNNER: [self-hosted, Linux, ARM64]} name: "${{ matrix.IMAGE.TOXENV }} on ${{ matrix.IMAGE.IMAGE }}" steps: + - name: Ridiculous alpine workaround for actions support on arm64 + run: | + # This modifies /etc/os-release so the JS actions + # from GH can't detect that it's on alpine:aarch64. It will + # then use a glibc nodejs, which works fine when gcompat + # is installed in the container (which it is) + sed -i "s:ID=alpine:ID=NotpineForGHA:" /etc/os-release + if: matrix.IMAGE.IMAGE == 'alpine:aarch64' - uses: actions/checkout@v3.6.0 with: persist-credentials: false diff --git a/.github/workflows/wheel-builder.yml b/.github/workflows/wheel-builder.yml index ce442d6..7dff2b3 100644 --- a/.github/workflows/wheel-builder.yml +++ b/.github/workflows/wheel-builder.yml @@ -16,8 +16,28 @@ on: jobs: - manylinux: + sdist: runs-on: ubuntu-latest + name: sdists + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + # The tag to build or the tag received by the tag event + ref: ${{ github.event.inputs.version || github.ref }} + persist-credentials: false + + - run: python -m venv .venv + - name: Install Python dependencies + run: .venv/bin/pip install -U pip build + - name: Make sdist + run: .venv/bin/python -m build --sdist + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: "bcrypt-sdist" + path: dist/bcrypt* + manylinux: + needs: [sdist] + runs-on: ${{ matrix.MANYLINUX.RUNNER }} strategy: matrix: PYTHON: @@ -25,18 +45,55 @@ jobs: - { VERSION: "pp39-pypy39_pp73" } - { VERSION: "pp310-pypy310_pp73" } MANYLINUX: - - { CONTAINER: "cryptography-manylinux2014:x86_64", NAME: "manylinux2014" } - - { CONTAINER: "cryptography-manylinux_2_28:x86_64", NAME: "manylinux_2_28" } - - { CONTAINER: "cryptography-musllinux_1_1:x86_64", NAME: "musllinux_1_1" } + - { NAME: "manylinux2014_x86_64", CONTAINER: "cryptography-manylinux2014:x86_64", RUNNER: "ubuntu-latest" } + - { NAME: "manylinux_2_28_x86_64", CONTAINER: "cryptography-manylinux_2_28:x86_64", RUNNER: "ubuntu-latest"} + - { NAME: "musllinux_1_1_x86_64", CONTAINER: "cryptography-musllinux_1_1:x86_64", RUNNER: "ubuntu-latest"} + - { NAME: "musllinux_1_2_x86_64", CONTAINER: "cryptography-musllinux_1_2:x86_64", RUNNER: "ubuntu-latest"} + + - { NAME: "manylinux2014_aarch64", CONTAINER: "cryptography-manylinux2014_aarch64", RUNNER: [self-hosted, Linux, ARM64] } + - { NAME: "manylinux_2_28_aarch64", CONTAINER: "cryptography-manylinux_2_28:aarch64", RUNNER: [self-hosted, Linux, ARM64]} + - { NAME: "musllinux_1_1_aarch64", CONTAINER: "cryptography-musllinux_1_1:aarch64", RUNNER: [self-hosted, Linux, ARM64]} + - { NAME: "musllinux_1_2_aarch64", CONTAINER: "cryptography-musllinux_1_2:aarch64", RUNNER: [self-hosted, Linux, ARM64]} exclude: # There are no readily available musllinux PyPy distributions - PYTHON: { VERSION: "pp39-pypy39_pp73" } - MANYLINUX: { NAME: "musllinux_1_1", CONTAINER: "cryptography-musllinux_1_1:x86_64"} + MANYLINUX: { NAME: "musllinux_1_1_x86_64", CONTAINER: "cryptography-musllinux_1_1:x86_64", RUNNER: "ubuntu-latest"} - PYTHON: { VERSION: "pp310-pypy310_pp73" } - MANYLINUX: { NAME: "musllinux_1_1", CONTAINER: "cryptography-musllinux_1_1:x86_64"} + MANYLINUX: { NAME: "musllinux_1_1_x86_64", CONTAINER: "cryptography-musllinux_1_1:x86_64", RUNNER: "ubuntu-latest"} + - PYTHON: { VERSION: "pp39-pypy39_pp73" } + MANYLINUX: { NAME: "musllinux_1_1_aarch64", CONTAINER: "cryptography-musllinux_1_1:aarch64", RUNNER: [self-hosted, Linux, ARM64]} + - PYTHON: { VERSION: "pp310-pypy310_pp73" } + MANYLINUX: { NAME: "musllinux_1_1_aarch64", CONTAINER: "cryptography-musllinux_1_1:aarch64", RUNNER: [self-hosted, Linux, ARM64]} + + - PYTHON: { VERSION: "pp39-pypy39_pp73" } + MANYLINUX: { NAME: "musllinux_1_2_x86_64", CONTAINER: "cryptography-musllinux_1_2:x86_64", RUNNER: "ubuntu-latest"} + - PYTHON: { VERSION: "pp310-pypy310_pp73" } + MANYLINUX: { NAME: "musllinux_1_2_x86_64", CONTAINER: "cryptography-musllinux_1_2:x86_64", RUNNER: "ubuntu-latest"} + - PYTHON: { VERSION: "pp39-pypy39_pp73" } + MANYLINUX: { NAME: "musllinux_1_2_aarch64", CONTAINER: "cryptography-musllinux_1_2:aarch64", RUNNER: [self-hosted, Linux, ARM64]} + - PYTHON: { VERSION: "pp310-pypy310_pp73" } + MANYLINUX: { NAME: "musllinux_1_2_aarch64", CONTAINER: "cryptography-musllinux_1_2:aarch64", RUNNER: [self-hosted, Linux, ARM64]} + + # We also don't build pypy wheels for anything except the latest manylinux + - PYTHON: { VERSION: "pp39-pypy39_pp73" } + MANYLINUX: { NAME: "manylinux2014_x86_64", CONTAINER: "cryptography-manylinux2014:x86_64", RUNNER: "ubuntu-latest"} + - PYTHON: { VERSION: "pp310-pypy310_pp73" } + MANYLINUX: { NAME: "manylinux2014_x86_64", CONTAINER: "cryptography-manylinux2014:x86_64", RUNNER: "ubuntu-latest"} + - PYTHON: { VERSION: "pp39-pypy39_pp73" } + MANYLINUX: { NAME: "manylinux2014_aarch64", CONTAINER: "cryptography-manylinux2014_aarch64", RUNNER: [self-hosted, Linux, ARM64]} + - PYTHON: { VERSION: "pp310-pypy310_pp73" } + MANYLINUX: { NAME: "manylinux2014_aarch64", CONTAINER: "cryptography-manylinux2014_aarch64", RUNNER: [self-hosted, Linux, ARM64]} name: "${{ matrix.PYTHON.VERSION }} for ${{ matrix.MANYLINUX.NAME }}" container: ghcr.io/pyca/${{ matrix.MANYLINUX.CONTAINER }} steps: + - name: Ridiculous alpine workaround for actions support on arm64 + run: | + # This modifies /etc/os-release so the JS actions + # from GH can't detect that it's on alpine:aarch64. It will + # then use a glibc nodejs, which works fine when gcompat + # is installed in the container (which it is) + sed -i "s:ID=alpine:ID=NotpineForGHA:" /etc/os-release + if: startsWith(matrix.MANYLINUX.NAME, 'musllinux') && endsWith(matrix.MANYLINUX.NAME, 'aarch64') - uses: actions/checkout@v3.6.0 with: # The tag to build or the tag received by the tag event @@ -44,15 +101,16 @@ jobs: - run: /opt/python/${{ matrix.PYTHON.VERSION }}/bin/python -m venv .venv - name: Install python dependencies run: .venv/bin/pip install -U pip wheel setuptools-rust - - name: Make sdist - run: .venv/bin/python setup.py sdist - - run: tar zxvf dist/bcrypt*.tar.gz && mkdir tmpwheelhouse + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: bcrypt-sdist + - run: mkdir tmpwheelhouse - name: Build the wheel run: | if [ -n "${{ matrix.PYTHON.ABI_VERSION }}" ]; then - PY_LIMITED_API="--py-limited-api=${{ matrix.PYTHON.ABI_VERSION }}" + PY_LIMITED_API="--config-settings=--build-option=--py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} --no-build-isolation" fi - cd bcrypt* && ../.venv/bin/python setup.py bdist_wheel $PY_LIMITED_API && mv dist/bcrypt*.whl ../tmpwheelhouse + .venv/bin/python -m pip wheel -v $PY_LIMITED_API bcrypt*.tar.gz -w dist/ && mv dist/bcrypt*.whl tmpwheelhouse env: RUSTUP_HOME: /root/.rustup - run: auditwheel repair tmpwheelhouse/bcrypt*.whl -w wheelhouse/ @@ -64,11 +122,12 @@ jobs: - run: mv wheelhouse/bcrypt*.whl bcrypt-wheelhouse/ - uses: actions/upload-artifact@v3.1.3 with: - name: "bcrypt-${{ github.event.inputs.version }}-${{ matrix.MANYLINUX.NAME }} -${{ matrix.PYTHON.ABI_VERSION }}" + name: "bcrypt-${{ github.event.inputs.version }}-${{ matrix.MANYLINUX.NAME }} -${{ matrix.PYTHON.VERSION }}" path: bcrypt-wheelhouse/ macos: - runs-on: macos-11 + needs: [sdist] + runs-on: macos-13 strategy: matrix: PYTHON: @@ -94,15 +153,19 @@ jobs: - run: ${{ matrix.PYTHON.BIN_PATH }} -m venv venv - run: venv/bin/pip install -U pip wheel setuptools-rust - - name: Make sdist - run: venv/bin/python setup.py sdist - - run: tar zxvf dist/bcrypt*.tar.gz && mkdir wheelhouse + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: bcrypt-sdist + - run: mkdir wheelhouse - name: Build the wheel - run: cd bcrypt* && ../venv/bin/python setup.py bdist_wheel --py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} && mv dist/bcrypt*.whl ../wheelhouse + run: | + if [ -n "${{ matrix.PYTHON.ABI_VERSION }}" ]; then + PY_LIMITED_API="--config-settings=--build-option=--py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} --no-build-isolation" + fi + venv/bin/python -m pip wheel -v $PY_LIMITED_API bcrypt*.tar.gz -w dist/ && mv dist/bcrypt*.whl wheelhouse env: PYTHON_VERSION: ${{ matrix.PYTHON.ABI_VERSION }} - MACOSX_DEPLOYMENT_TARGET: '10.10' - CFLAGS: '-arch arm64 -arch x86_64' + DEPLOYMENT_TARGET: '10.12' ARCHFLAGS: '-arch arm64 -arch x86_64' _PYTHON_HOST_PLATFORM: 'macosx-10.9-universal2' - run: venv/bin/pip install -f wheelhouse --no-index bcrypt @@ -117,6 +180,7 @@ jobs: path: bcrypt-wheelhouse/ windows: + needs: [sdist] runs-on: windows-latest strategy: matrix: @@ -132,6 +196,9 @@ jobs: # The tag to build or the tag received by the tag event ref: ${{ github.event.inputs.version || github.ref }} persist-credentials: false + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: bcrypt-sdist - name: Setup python uses: actions/setup-python@v4.7.1 with: @@ -143,11 +210,15 @@ jobs: target: ${{ matrix.WINDOWS.RUST_TRIPLE }} - run: python -m pip install -U pip wheel setuptools-rust - - name: Make sdist - run: python setup.py sdist - - run: tar zxvf dist/bcrypt*.tar.gz && mkdir wheelhouse + - run: mkdir wheelhouse + shell: bash + - name: Build the wheel + run: | + if [ -n "${{ matrix.PYTHON.ABI_VERSION }}" ]; then + PY_LIMITED_API="--config-settings=--build-option=--py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} --no-build-isolation" + fi + python -m pip wheel -v $PY_LIMITED_API bcrypt*.tar.gz -w dist/ && mv dist/bcrypt*.whl wheelhouse shell: bash - - run: cd bcrypt* && python setup.py bdist_wheel --py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} && mv dist/bcrypt*.whl ../wheelhouse - run: pip install -f wheelhouse --no-index bcrypt - run: | python -c "import bcrypt; password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)" diff --git a/release.py b/release.py index a6329bc..e104c65 100644 --- a/release.py +++ b/release.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function import getpass -import glob import io import json import os @@ -81,7 +80,7 @@ def download_artifacts_github_actions(session, token, run_url): return paths -def build_github_actions_wheels(token, version): +def build_github_actions_sdist_wheels(token, version): session = requests.Session() response = session.post( @@ -125,15 +124,11 @@ def release(version): run("git", "tag", "-s", version, "-m", "{0} release".format(version)) run("git", "push", "--tags") - run("python", "setup.py", "sdist") - - packages = glob.glob("dist/bcrypt-{0}*".format(version)) - github_actions_wheel_paths = build_github_actions_wheels( + github_actions_paths = build_github_actions_sdist_wheels( github_token, version ) - run("twine", "upload", *github_actions_wheel_paths) - run("twine", "upload", "-s", *packages) + run("twine", "upload", *github_actions_paths) if __name__ == "__main__":