Migrate wheel builders to GA (#198)

* Migrate Windows wheel builder to GA

* Migrate macOS wheel builder to GA

* Migrate manylinux1 wheel builder to GA

* Migrate release.py to use GA

.azure-pipelines/wheel-builder.yml

@@ -1,164 +0,0 @@
-trigger: none
-pr: none
-
-jobs:
-    - job: 'macOS'
-      pool:
-          vmImage: 'macOS-10.14'
-      strategy:
-          matrix:
-              Python27:
-                  python.version: '2.7'
-                  PYTHON_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.16/python-2.7.16-macosx10.6.pkg"
-                  PYTHON_BIN_PATH: /Library/Frameworks/Python.framework/Versions/2.7/bin/python
-              Python3:
-                  python.version: '3.5'
-                  PYTHON_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.7.3/python-3.7.3-macosx10.6.pkg"
-                  PYTHON_BIN_PATH: /Library/Frameworks/Python.framework/Versions/3.7/bin/python3
-      steps:
-          - script: |
-              set -e
-              set -x
-
-              curl "$PYTHON_DOWNLOAD_URL" -o python.pkg
-              sudo installer -pkg python.pkg -target /
-            displayName: Download and install Python
-
-          - script: $PYTHON_BIN_PATH -m pip install -U virtualenv
-            displayName: Install virtualenv
-          - script: $PYTHON_BIN_PATH -m virtualenv .venv
-            displayName: Create virtualenv
-          - script: .venv/bin/pip install -U pip==10.0.1
-            displayName: Downgrade pip lol
-          - script: .venv/bin/pip install -U wheel
-            displayName: Update wheel to the latest version
-          - script: .venv/bin/pip install cffi six
-            displayName: Install our Python dependencies
-
-          - script: |
-              set -e
-              set -x
-
-              REGEX="3\.([0-9])*"
-              if [[ "$PYTHON_VERSION" =~ $REGEX ]]; then
-                  PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}"
-              fi
-
-              .venv/bin/pip wheel bcrypt --wheel-dir=wheelhouse --no-binary bcrypt --no-deps $PY_LIMITED_API
-            displayName: Build the wheel
-          - script: .venv/bin/pip install --no-index -f wheelhouse bcrypt
-            displayName: Test installing the wheel
-          - script: |
-              .venv/bin/python -c "import bcrypt;password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)"
-            displayName: Test the installed wheel
-          - task: PublishBuildArtifacts@1
-            inputs:
-                pathToPublish: wheelhouse/
-                artifactName: bcrypt-macos-python$(python.version)
-
-    - job: 'manylinux1'
-      pool:
-          vmImage: 'ubuntu-16.04'
-      container: 'pyca/cryptography-manylinux1:x86_64'
-      strategy:
-          matrix:
-              Python27m:
-                  PYTHON_VERSION: 'cp27-cp27m'
-              Python27mu:
-                  PYTHON_VERSION: 'cp27-cp27mu'
-              Python3m:
-                  PYTHON_VERSION: 'cp35-cp35m'
-      steps:
-          - script: /opt/python/$PYTHON_VERSION/bin/python -m virtualenv .venv
-            displayName: Create virtualenv
-          - script: .venv/bin/pip install -U pip==10.0.1
-            displayName: Downgrade pip lol
-          - script: .venv/bin/pip install cffi six
-            displayName: Install our Python dependencies
-          - script: |
-              set -e
-              set -x
-
-              REGEX="cp3([0-9])*"
-              if [[ "$PYTHON_VERSION" =~ $REGEX ]]; then
-                  PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}"
-              fi
-              .venv/bin/pip wheel bcrypt --no-binary bcrypt --no-deps --wheel-dir=tmpwheelhouse $PY_LIMITED_API
-            displayName: Build the wheel
-          - script: auditwheel repair tmpwheelhouse/bcrypt*.whl -w wheelhouse/
-            displayName: Run auditwheel
-          - script: .venv/bin/pip install bcrypt --no-index -f wheelhouse/
-            displayName: Test installing the wheel
-          - script: |
-              .venv/bin/python -c "import bcrypt;password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)"
-            displayName: Test the installed wheel
-          - task: PublishBuildArtifacts@1
-            inputs:
-                pathToPublish: wheelhouse/
-                artifactName: bcrypt-manylinux1-$(PYTHON_VERSION)
-
-    - job: 'windows'
-      pool:
-          vmImage: 'windows-2019'
-      container: $[variables.containerImage]
-      strategy:
-          matrix:
-              Python27-x86:
-                  containerImage: 'pyca/cryptography-runner-windows:py27-x86'
-                  PYTHON_VERSION: '27'
-                  WINDOWS_ARCH: 'x86'
-              Python27-x86-64:
-                  containerImage: 'pyca/cryptography-runner-windows:py27-x86_64'
-                  PYTHON_VERSION: '27'
-                  WINDOWS_ARCH: 'x86_64'
-              Python35-x86:
-                  containerImage: 'pyca/cryptography-runner-windows:py35-x86'
-                  PYTHON_VERSION: '35'
-                  WINDOWS_ARCH: 'x86'
-              Python35-x86-64:
-                  containerImage: 'pyca/cryptography-runner-windows:py35-x86_64'
-                  PYTHON_VERSION: '35'
-                  WINDOWS_ARCH: 'x86_64'
-              Python36-x86:
-                  containerImage: 'pyca/cryptography-runner-windows:py3-x86'
-                  PYTHON_VERSION: '36'
-                  WINDOWS_ARCH: 'x86'
-              Python36-x86-64:
-                  containerImage: 'pyca/cryptography-runner-windows:py3-x86_64'
-                  PYTHON_VERSION: '36'
-                  WINDOWS_ARCH: 'x86_64'
-              Python37-x86:
-                  containerImage: 'pyca/cryptography-runner-windows:py3-x86'
-                  PYTHON_VERSION: '37'
-                  WINDOWS_ARCH: 'x86'
-              Python37-x86-64:
-                  containerImage: 'pyca/cryptography-runner-windows:py3-x86_64'
-                  PYTHON_VERSION: '37'
-                  WINDOWS_ARCH: 'x86_64'
-              Python38-x86:
-                  containerImage: 'pyca/cryptography-runner-windows:py3-x86'
-                  PYTHON_VERSION: '38'
-                  WINDOWS_ARCH: 'x86'
-              Python38-x86-64:
-                  containerImage: 'pyca/cryptography-runner-windows:py3-x86_64'
-                  PYTHON_VERSION: '38'
-                  WINDOWS_ARCH: 'x86_64'
-      steps:
-          - script: '"C:/Python%PYTHON_VERSION%/Scripts/pip" install wheel cffi six'
-            displayName: Install wheel and our Python dependencies
-          - script: |
-              C:/Python%PYTHON_VERSION%/Scripts/pip wheel bcrypt --wheel-dir=wheelhouse --no-binary bcrypt
-            displayName: Build the wheel
-          - script: '"C:/Python%PYTHON_VERSION%/Scripts/pip" install -f wheelhouse bcrypt --no-index'
-            displayName: Test installing the wheel
-          - script: |
-              "C:/Python%PYTHON_VERSION%/python" -c "import bcrypt;password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)"
-            displayName: Test the installed wheel
-          - script: mkdir bcrypt-wheelhouse
-            displayName: Create a directory for placing the final wheel in
-          - script: move wheelhouse\bcrypt*.whl bcrypt-wheelhouse\
-            displayName: Move the bcrypt wheel into the final wheel house
-          - task: PublishBuildArtifacts@1
-            inputs:
-                pathToPublish: bcrypt-wheelhouse/
-                artifactName: bcrypt-windows-$(WINDOWS_ARCH)-python$(PYTHON_VERSION)

.github/workflows/wheel-builder.yml

@@ -0,0 +1,114 @@
+on:
+  repository_dispatch:
+    types: [wheel-builder]
+
+jobs:
+  manylinux:
+    runs-on: ubuntu-latest
+    container: pyca/cryptography-manylinux1:x86_64
+    strategy:
+      matrix:
+        PYTHON: ["cp27-cp27m", "cp27-cp27mu", "cp35-cp35m"]
+    name: "Python ${{ matrix.PYTHON }} for manylinux1"
+    steps:
+      - run: /opt/python/${{ matrix.PYTHON }}/bin/python -m virtualenv .venv
+      # Downgrade pip, can't remember why
+      - run: .venv/bin/pip install -U pip==10.0.1
+      - run: |
+          REGEX="cp3([0-9])*"
+          if [[ "${{ matrix.PYTHON }}" =~ $REGEX ]]; then
+              PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}"
+          fi
+          .venv/bin/pip wheel bcrypt --no-binary bcrypt --no-deps --wheel-dir=tmpwheelhouse $PY_LIMITED_API
+      - run: auditwheel repair tmpwheelhouse/bcrypt*.whl -w wheelhouse/
+      - run: .venv/bin/pip install bcrypt --no-index -f wheelhouse/
+      - run: |
+          .venv/bin/python -c "import bcrypt; password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)"
+
+      - run: mkdir bcrypt-wheelhouse
+      - run: move wheelhouse/bcrypt*.whl bcrypt-wheelhouse/
+      - uses: actions/upload-artifact@v1
+        with:
+          name: "bcrypt-${{ github.event.client_payload.BUILD_VERSION }}-manylinux1-${{ matrix.PYTHON }}"
+          path: bcrypt-wheelhouse/
+
+  macos:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        PYTHON:
+          - {VERSION: '2.7', ABI_VERSION: '2.7'}
+          - {VERSION: '3.8', ABI_VERSION: '3.5'}
+    name: "Python ${{ matrix.PYTHON.VERSION }} on macOS"
+    steps:
+      - uses: actions/checkout@master
+      - name: Setup python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.PYTHON.VERSION }}
+
+      # Downgrade pip, I can't remember why
+      - run: pip install -U pip==10.0.1
+      - run: pip install -U wheel cffi six
+
+      - name: Build the wheel
+        run: |
+            REGEX="3\.([0-9])*"
+            if [[ "$PYTHON_VERSION" =~ $REGEX ]]; then
+                PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}"
+            fi
+
+            pip wheel bcrypt --wheel-dir=wheelhouse --no-binary bcrypt --no-deps $PY_LIMITED_API
+        env:
+          PYTHON_VERSION: ${{ matrix.PYTHON.ABI_VERSION }}
+      - run: pip install -f wheelhouse --no-index bcrypt
+      - run: |
+          python -c "import bcrypt;password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)"
+
+      - run: mkdir bcrypt-wheelhouse
+      - run: move wheelhouse/bcrypt*.whl bcrypt-wheelhouse/
+      - uses: actions/upload-artifact@v1
+        with:
+          name: "bcrypt-${{ github.event.client_payload.BUILD_VERSION }}-macOS-${{ matrix.PYTHON.VERSION }}"
+          path: bcrypt-wheelhouse/
+
+  windows:
+    runs-on: windows-latest
+    strategy:
+        WINDOWS:
+          - {ARCH: 'x86', WINDOWS: 'win32'}
+          - {ARCH: 'x64', WINDOWS: 'win64'}
+        PYTHON:
+          - {VERSION: "2.7", TOXENV: "py27"}
+          - {VERSION: "3.5", TOXENV: "py35"}
+          - {VERSION: "3.6", TOXENV: "py36"}
+          - {VERSION: "3.7", TOXENV: "py37"}
+          - {VERSION: "3.8", TOXENV: "py38"}
+    name: "Python ${{ matrix.PYTHON.VERSION }} on ${{ matrix.WINDOWS.WINDOWS }}"
+    steps:
+      - uses: actions/checkout@master
+      - name: Setup python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.PYTHON.VERSION }}
+          architecture: ${{ matrix.WINDOWS.ARCH }}
+      - name: Install MSVC for Python 2.7
+        run: |
+            Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi
+            Start-Process msiexec -Wait -ArgumentList @('/i', 'VCForPython27.msi', '/qn', 'ALLUSERS=1')
+            Remove-Item VCForPython27.msi -Force
+        shell: powershell
+        if: matrix.PYTHON.VERSION == '2.7'
+
+      - run: pip install wheel cffi six
+      - run: pip wheel bcrypt==${{ github.event.client_payload.BUILD_VERSION }} --wheel-dir=wheelhouse --no-binary bcrypt
+      - run: pip install -f wheelhouse --no-index bcrypt
+      - run: |
+          python -c "import bcrypt; password = b'super secret password';hashed = bcrypt.hashpw(password, bcrypt.gensalt());bcrypt.checkpw(password, hashed)"
+
+      - run: mkdir bcrypt-wheelhouse
+      - run: move wheelhouse\bcrypt*.whl bcrypt-wheelhouse\
+      - uses: actions/upload-artifact@v1
+        with:
+          name: "bcrypt-${{ github.event.client_payload.BUILD_VERSION }}-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.VERSION }}"
+          path: bcrypt-wheelhouse\

release.py

@@ -6,18 +6,16 @@ from __future__ import absolute_import, division, print_function
 
 import getpass
 import glob
+import io
+import json
 import os
 import subprocess
-import tempfile
 import time
 import zipfile
 
-from azure.devops.connection import Connection
-from azure.devops.v5_1.build.models import Build
-
 import click
 
-from msrest.authentication import BasicAuthentication
+import requests
 
 
 def run(*args, **kwargs):
@@ -25,59 +23,87 @@ def run(*args, **kwargs):
     subprocess.check_call(list(args), **kwargs)
 
 
-def wait_for_build_completed_azure(build_client, build_id):
+def wait_for_build_complete_github_actions(session, token, run_url):
     while True:
-        build = build_client.get_build("bcrypt", build_id)
-        if build.finish_time is not None:
+        response = session.get(run_url, headers={
+            "Content-Type": "application/json",
+            "Authorization": "token {}".format(token),
+        })
+        response.raise_for_status()
+        if response.json()["conclusion"] is not None:
             break
         time.sleep(3)
 
 
-def download_artifacts_azure(build_client, build_id):
-    artifacts = build_client.get_artifacts("bcrypt", build_id)
+def download_artifacts_github_actions(session, token, run_url):
+    response = session.get(run_url, headers={
+        "Content-Type": "application/json",
+        "Authorization": "token {}".format(token),
+    })
+    response.raise_for_status()
+
+    response = session.get(response.json()["artifacts_url"], headers={
+        "Content-Type": "application/json",
+        "Authorization": "token {}".format(token),
+    })
+    response.raise_for_status()
     paths = []
-    for artifact in artifacts:
-        contents = build_client.get_artifact_content_zip(
-            "bcrypt", build_id, artifact.name
-        )
-        with tempfile.NamedTemporaryFile() as f:
-            for chunk in contents:
-                f.write(chunk)
-            f.flush()
-            with zipfile.ZipFile(f.name) as z:
-                for name in z.namelist():
-                    if not name.endswith(".whl"):
-                        continue
-                    p = z.open(name)
-                    out_path = os.path.join(
-                        os.path.dirname(__file__),
-                        "dist",
-                        os.path.basename(name),
-                    )
-                    with open(out_path, "wb") as f:
-                        f.write(p.read())
-                    paths.append(out_path)
+    for artifact in response.json()["artifacts"]:
+        response = session.get(artifact["archive_download_url"], headers={
+            "Content-Type": "application/json",
+            "Authorization": "token {}".format(token),
+        })
+        with zipfile.ZipFile(io.BytesIO(response.content)) as z:
+            for name in z.namelist():
+                if not name.endswith(".whl"):
+                    continue
+                p = z.open(name)
+                out_path = os.path.join(
+                    os.path.dirname(__file__),
+                    "dist",
+                    os.path.basename(name),
+                )
+                with open(out_path, "wb") as f:
+                    f.write(p.read())
+                paths.append(out_path)
     return paths
 
 
-def build_wheels_azure(version):
-    token = getpass.getpass("Azure personal access token: ")
-    credentials = BasicAuthentication("", token)
-    connection = Connection(
-        base_url="https://dev.azure.com/pyca", creds=credentials
+def build_github_actions_wheels(token, version):
+    session = requests.Session()
+
+    response = session.post(
+        "https://api.github.com/repos/pyca/bcrypt/dispatches",
+        headers={
+            "Content-Type": "application/json",
+            "Accept": "application/vnd.github.everest-preview+json",
+            "Authorization": "token {}".format(token),
+        },
+        data=json.dumps({
+            "event_type": "wheel-builder",
+            "client_payload": {
+                "BUILD_VERSION": version,
+            },
+        }),
     )
-    build_client = connection.clients.get_build_client()
-    [definition] = build_client.get_definitions(
-        "bcrypt", "bcrypt-wheel-builder"
+    response.raise_for_status()
+
+    # Give it a few seconds for the run to kick off.
+    time.sleep(5)
+    response = session.get(
+        (
+            "https://api.github.com/repos/pyca/bcrypt/actions/workflows/"
+            "wheel-builder.yml/runs?event=repository_dispatch"
+        ),
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "token {}".format(token),
+        },
     )
-    build_description = Build(
-        definition=definition,
-    )
-    build = build_client.queue_build(
-        project="bcrypt", build=build_description
-    )
-    wait_for_build_completed_azure(build_client, build.id)
-    return download_artifacts_azure(build_client, build.id)
+    response.raise_for_status()
+    run_url = response.json()["workflow_runs"][0]["url"]
+    wait_for_build_complete_github_actions(session, token, run_url)
+    return download_artifacts_github_actions(session, token, run_url)
 
 
 @click.command()
@@ -86,6 +112,8 @@ def release(version):
     """
     ``version`` should be a string like '0.4' or '1.0'.
     """
+    github_token = getpass.getpass("Github person access token: ")
+
     run("git", "tag", "-s", version, "-m", "{0} release".format(version))
     run("git", "push", "--tags")
 
@@ -94,8 +122,10 @@ def release(version):
     packages = glob.glob("dist/bcrypt-{0}*".format(version))
     run("twine", "upload", "-s", *packages)
 
-    azure_wheel_paths = build_wheels_azure(version)
-    run("twine", "upload", *azure_wheel_paths)
+    github_actions_wheel_paths = build_github_actions_wheels(
+        github_token, version
+    )
+    run("twine", "upload", *github_actions_wheel_paths)
 
 
 if __name__ == "__main__":