From 6157ce5e302de0abb4db9d5aa1e858b203b9f7d5 Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Fri, 24 Jun 2022 17:44:22 +0200 Subject: [PATCH 1/9] Test cibuildwheel for macOS wheels Try out a basic cibuildwheel workflow just for macOS wheels for now. The main aim is to get arm64 wheels for Apple Silicon (M1) processors. On success this could be extended to manylinux wheels too... --- .github/workflows/wheels.yml | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..908081c --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,48 @@ +name: Build wheels + +on: + # Only a manual trigger for now + workflow_dispatch: + +env: + CIBW_BEFORE_ALL_MACOS: > + brew tap casacore/tap && + brew install casacore --with-python + # Support native arm64 and universal2 = Intel + ARM for Apple Silicon + CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" + # XXX (but don't test it yet - we first need some M1 runners) + CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64" + +jobs: + make_sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist/*.tar.gz + + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest] + steps: + - uses: actions/checkout@v3 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.6.0 + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse/*.whl From 723fc4932f9ee301449d1ad6a222b7a454cfed6c Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Fri, 24 Jun 2022 20:54:19 +0200 Subject: [PATCH 2/9] Use HEAD version of casacore-data Compiling the python package does not depend on the exact version of the measures data. The latest is fine. This avoids crashing the build if casacore-data is out of date, which happens from time to time. Since ftp.astron.nl only stores the latest 3 months or so, this requires casacore-data to be updated every 3 months in sync with it, and it doesn't always happen. --- .github/workflows/wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 908081c..c1f06ea 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -5,8 +5,11 @@ on: workflow_dispatch: env: + # Install HEAD version of casacore-data because we don't need exact version + # when compiling python-casacore. This avoids failure if it's not up to date. CIBW_BEFORE_ALL_MACOS: > brew tap casacore/tap && + brew install casacore-data --HEAD && brew install casacore --with-python # Support native arm64 and universal2 = Intel + ARM for Apple Silicon CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" From 06c3181d3cf36d11c678737ee31e72d6a3335006 Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Mon, 21 Aug 2023 13:18:07 +0200 Subject: [PATCH 3/9] Enable M1 tests and upgrade cibuildwheel This is based on the latest github-minimal.yml example in cibuildwheel. Enable tests on arm64 again, since there should be M1 runners now. Drop the universal build to save time and space, since Homebrew doesn't dig it either. --- .github/workflows/wheels.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c1f06ea..9c44ee2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -12,9 +12,7 @@ env: brew install casacore-data --HEAD && brew install casacore --with-python # Support native arm64 and universal2 = Intel + ARM for Apple Silicon - CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" - # XXX (but don't test it yet - we first need some M1 runners) - CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64" + CIBW_ARCHS_MACOS: "x86_64 arm64" jobs: make_sdist: @@ -42,10 +40,10 @@ jobs: - uses: actions/checkout@v3 - name: Build wheels - uses: pypa/cibuildwheel@v2.6.0 + uses: pypa/cibuildwheel@v2.15.0 - name: Upload wheels uses: actions/upload-artifact@v3 with: name: wheels - path: wheelhouse/*.whl + path: ./wheelhouse/*.whl From e250145783dce2d41d2b8ef860f9b667e0686d97 Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Mon, 21 Aug 2023 13:32:23 +0200 Subject: [PATCH 4/9] Upgrade to checkout and upload-artifact v3 The Actions tab is complaining about v2 that uses the deprecated Node12 and this causes the tests to fail. See the following post for more details: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/ --- .github/workflows/linux.yml | 4 ++-- .github/workflows/osx.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4cec58e..24fc751 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -23,8 +23,8 @@ jobs: - py3_casacore_v3.4 steps: - name: Checkout - uses: actions/checkout@v2 - + uses: actions/checkout@v3 + - name: Build container run: docker build . -t ${{ matrix.dist }} -f .github/workflows/${{ matrix.dist }}.docker diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 21b3783..2fc47ed 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -14,11 +14,11 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: install homebrew packages run: | - brew tap casacore/tap + brew tap casacore/tap brew install casacore --with-python - name: make virtualenv @@ -40,6 +40,6 @@ jobs: run: venv/bin/delocate-wheel -v dist/*.whl - name: Publish OS X binary wheels - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: path: dist/*.whl From 44c703cb758f675ff47a434056ebb9781845091a Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Mon, 21 Aug 2023 13:34:47 +0200 Subject: [PATCH 5/9] Run wheels on pushing to test branch Unfortunately the manual dispatch button only works on the default (master) branch, so we need to trigger via pushing. --- .github/workflows/wheels.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9c44ee2..d2a6f73 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,7 +1,9 @@ name: Build wheels on: - # Only a manual trigger for now + # Just testing for now, and the dispatch on works on master branch... + push: + branches: [ cibuildwheel ] workflow_dispatch: env: From e3d6a84107d0bfa2db0b983d6eb84edea2b98c5c Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Mon, 21 Aug 2023 16:15:29 +0200 Subject: [PATCH 6/9] Fix comment --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d2a6f73..2e13c06 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -13,7 +13,7 @@ env: brew tap casacore/tap && brew install casacore-data --HEAD && brew install casacore --with-python - # Support native arm64 and universal2 = Intel + ARM for Apple Silicon + # Support both Apple architectures natively CIBW_ARCHS_MACOS: "x86_64 arm64" jobs: From a44d7afc4d512c665d6a975595704afdf90baa1d Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Mon, 21 Aug 2023 16:15:56 +0200 Subject: [PATCH 7/9] Try workaround for CMake error Python-casacore CMake fails with: Could NOT find Python3 (missing: Python3_NumPy_INCLUDE_DIRS NumPy) This is similar to https://issues.apache.org/jira/browse/ARROW-11151, where a workaround is to install numpy first (even though Homebrew also installs it). --- .github/workflows/wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2e13c06..b175661 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -9,9 +9,11 @@ on: env: # Install HEAD version of casacore-data because we don't need exact version # when compiling python-casacore. This avoids failure if it's not up to date. + # The pip install is a workaround (see https://issues.apache.org/jira/browse/ARROW-11151). CIBW_BEFORE_ALL_MACOS: > brew tap casacore/tap && brew install casacore-data --HEAD && + pip3 install numpy && brew install casacore --with-python # Support both Apple architectures natively CIBW_ARCHS_MACOS: "x86_64 arm64" From f4729956ff57f0096143ab0a44f225cfc1e3e343 Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Tue, 22 Aug 2023 00:20:43 +0200 Subject: [PATCH 8/9] Only build wheels for latest Homebrew Python This is necessary since we otherwise have to install multiple boost-python3's, one per minor Python version in cibuildwheel. It is sort-of possible (see https://github.com/MarkusJx/install-boost) but with some restrictions (no Apple Silicon?) Also restore unit test restrictions, since GitHub still does not have M1 runner support (see https://github.com/github/roadmap/issues/528). --- .github/workflows/wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b175661..1e207f7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -7,6 +7,8 @@ on: workflow_dispatch: env: + # Just build for the latest Homebrew Python on macOS + CIBW_BUID: cp311-macosx* # Install HEAD version of casacore-data because we don't need exact version # when compiling python-casacore. This avoids failure if it's not up to date. # The pip install is a workaround (see https://issues.apache.org/jira/browse/ARROW-11151). @@ -17,6 +19,8 @@ env: brew install casacore --with-python # Support both Apple architectures natively CIBW_ARCHS_MACOS: "x86_64 arm64" + # XXX (but don't test Apple Silicon yet - we first need some M1 runners - Q4 2023?) + CIBW_TEST_SKIP: "*-macosx_arm64" jobs: make_sdist: From 81a4039e33a47b2d6caf52d08a908413ff867f74 Mon Sep 17 00:00:00 2001 From: Ludwig Schwardt Date: Tue, 22 Aug 2023 01:10:17 +0200 Subject: [PATCH 9/9] Fix typo --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 1e207f7..1ace0da 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -8,7 +8,7 @@ on: env: # Just build for the latest Homebrew Python on macOS - CIBW_BUID: cp311-macosx* + CIBW_BUILD: cp311-macosx* # Install HEAD version of casacore-data because we don't need exact version # when compiling python-casacore. This avoids failure if it's not up to date. # The pip install is a workaround (see https://issues.apache.org/jira/browse/ARROW-11151).