Skip to content

Commit 76a0869

Browse files
author
Release Manager
committed
gh-38272: `dist.yml`: Build musllinux wheels, build linux aarch64 wheels via QEMU <!-- ^ Please provide a concise and informative title. --> <!-- ^ Don't put issue numbers in the title, do this in the PR description below. --> <!-- ^ For example, instead of "Fixes #12345" use "Introduce new method to calculate 1 + 2". --> <!-- v Describe your changes below in detail. --> <!-- v Why is this change required? What problem does it solve? --> <!-- v If this PR resolves an open issue, please link to it here. For example, "Fixes #12345". --> Other changes: - Reusing the built wheels for build dependencies. This speeds up the build when no compatible binary wheel is on PyPI yet. - Separate job steps for the various distributions, to improve readability of the output. Test run: https://github.com/mkoeppe/sage/actions/runs/9666490294 The aarch64 build takes very long (e.g. 20 minutes for each wheel of **sagemath-objects**, total 2.5 hours for sagemath-objects, total 5 hours for all manylinux platforms) because of the CPU emulation. That's normal. ### 📝 Checklist <!-- Put an `x` in all the boxes that apply. --> - [x] The title is concise and informative. - [x] The description explains in detail what this PR is about. - [ ] I have linked a relevant issue or discussion. - [ ] I have created tests covering the changes. - [ ] I have updated the documentation and checked the documentation preview. ### ⌛ Dependencies <!-- List all open PRs that this PR logically depends on. For example, --> <!-- - #12345: short description why this is a dependency --> <!-- - #34567: ... --> URL: #38272 Reported by: Matthias Köppe Reviewer(s): Kwankyu Lee
2 parents 03bc844 + 4e53812 commit 76a0869

File tree

1 file changed

+42
-19
lines changed

1 file changed

+42
-19
lines changed

.github/workflows/dist.yml

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -155,41 +155,48 @@ jobs:
155155
if: env.CAN_DEPLOY == 'true'
156156

157157
build_wheels:
158-
name: Build wheels on ${{ matrix.os }}, arch ${{ matrix.arch }}
158+
name: wheels ${{ matrix.build }}*_${{ matrix.arch }}
159159
runs-on: ${{ matrix.os }}
160160
needs: sdists_for_pypi
161161
strategy:
162162
fail-fast: false
163163
matrix:
164+
os: [ubuntu-latest]
165+
arch: [x86_64, i686, aarch64]
166+
build: [manylinux, musllinux]
164167
include:
165-
- os: ubuntu-latest
166-
arch: x86_64
167-
- os: ubuntu-latest
168-
arch: i686
169168
- os: macos-13
170169
arch: x86_64
170+
build: macosx
171171
- os: macos-14
172172
arch: arm64
173+
build: macosx
173174
env:
174175
CAN_DEPLOY: ${{ secrets.SAGEMATH_PYPI_API_TOKEN != '' }}
175176
# SPKGs to install as system packages
176177
SPKGS: _bootstrap _prereq
177178
# Non-Python packages to install as spkgs
178179
TARGETS_PRE: gmp mpfr mpc bliss coxeter3 mcqd meataxe sirocco boost_cropped tdlib
180+
CIBW_BUILD: "*${{ matrix.build }}*"
179181
# Disable building PyPy wheels on all platforms
180-
# Disable musllinux until #33083 provides alpine package information
181-
CIBW_SKIP: "pp* *-musllinux*"
182+
CIBW_SKIP: "pp*"
182183
#
183184
CIBW_ARCHS: ${{ matrix.arch }}
184185
# https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python
185186
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9, <3.13"
186187
# Environment during wheel build
187-
CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt"
188+
CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt PIP_FIND_LINKS=file://$(pwd)/wheelhouse SAGE_NUM_THREADS=6"
188189
# Use 'build', not 'pip wheel'
189190
CIBW_BUILD_FRONTEND: build
190191
steps:
191192
- uses: actions/checkout@v4
192193

194+
- name: Set up QEMU
195+
if: runner.os == 'Linux' && matrix.arch != 'x86_64' && matrix.arch != 'i686'
196+
uses: docker/setup-qemu-action@v3
197+
with:
198+
platforms: all
199+
193200
- uses: actions/download-artifact@v4
194201
with:
195202
name: dist
@@ -209,7 +216,7 @@ jobs:
209216
eval $(sage-print-system-package-command auto --sudo --yes --no-install-recommends --spkg install _bootstrap)
210217
./bootstrap
211218
212-
- name: Build platform wheels
219+
- name: Unpack and prepare
213220
# We build the wheels from the sdists so that MANIFEST filtering becomes effective.
214221
# But we must run cibuildwheel with the unpacked source directory, not a tarball,
215222
# so that SAGE_ROOT is copied into the build containers.
@@ -223,21 +230,37 @@ jobs:
223230
#
224231
# omit sagemath-{meataxe,sirocco} for now -- needs sagemath-modules
225232
run: |
226-
"${{ steps.python.outputs.python-path }}" -m pip install pipx
233+
"${{ steps.python.outputs.python-path }}" -m pip install cibuildwheel==2.18.0
227234
export PATH=build/bin:$PATH
228-
export CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt"
235+
echo CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || $(sage-print-system-package-command alpine --yes --no-install-recommends install $(sage-get-system-packages alpine $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt" >> "$GITHUB_ENV"
229236
mkdir -p unpacked
230-
for pkg in sagemath*objects sagemath*categories sagemath*bliss sagemath*coxeter3 sagemath*mcqd sagemath*tdlib; do
231-
case "$pkg:${{ matrix.arch }}" in
232-
sagemath*tdlib:i686) continue;; # broken - boost-related
233-
esac
234-
(cd unpacked && tar xfz - ) < dist/$pkg*.tar.gz
235-
"${{ steps.python.outputs.python-path }}" -m pipx run cibuildwheel==2.18.0 unpacked/$pkg*
237+
for sdist in dist/$pkg*.tar.gz; do
238+
(cd unpacked && tar xfz - ) < $sdist
236239
done
237240
241+
- name: sagemath-objects
242+
run: |
243+
"${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*objects*
244+
245+
- name: sagemath-categories
246+
run: |
247+
"${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*categories*
248+
249+
- name: sagemath-bliss
250+
run: |
251+
"${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*bliss*
252+
253+
- name: sagemath-coxeter3
254+
run: |
255+
"${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*coxeter3*
256+
257+
- name: sagemath-mcqd
258+
run: |
259+
"${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*mcqd*
260+
238261
- uses: actions/upload-artifact@v4
239262
with:
240-
name: ${{ matrix.os }}-${{ matrix.arch }}-wheels
263+
name: ${{ matrix.os }}-${{ matrix.build }}-${{ matrix.arch }}-wheels
241264
path: ./wheelhouse/*.whl
242265

243266
upload_wheels:
@@ -250,7 +273,7 @@ jobs:
250273

251274
- uses: actions/download-artifact@v4
252275
with:
253-
pattern: "*-*-wheels"
276+
pattern: "*-*-*-wheels"
254277
path: wheelhouse
255278
merge-multiple: true
256279

0 commit comments

Comments
 (0)