Skip to content

Commit 21dd8ed

Browse files
mattipmattip
authored
Merge pull request #193 from matthew-brett/refactor-woa-build
Refactor Windows on ARM build script
2 parents 29ffffb + 472f034 commit 21dd8ed

File tree

6 files changed

+217
-53
lines changed

6 files changed

+217
-53
lines changed

‎.github/workflows/posix.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ on:
1111

1212
env:
1313
REPO_DIR: OpenBLAS
14-
OPENBLAS_COMMIT: "v0.3.29"
14+
OPENBLAS_COMMIT: "v0.3.29-265-gb5456c1b"
1515
MACOSX_DEPLOYMENT_TARGET: 10.9
1616

1717
jobs:

‎.github/workflows/windows-arm.yml

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: Windows-on-ARM
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
env:
10+
OPENBLAS_COMMIT: "v0.3.29-265-gb5456c1b"
11+
OPENBLAS_ROOT: "c:\\opt"
12+
# Preserve working directory for calls into bash
13+
# Without this, invoking bash will cd to the home directory
14+
CHERE_INVOKING: "yes"
15+
BASH_PATH: "C:\\Program Files\\Git\\bin\\bash.exe"
16+
PLAT: arm64
17+
INTERFACE64: 0
18+
BUILD_BITS: 32
19+
20+
jobs:
21+
build:
22+
runs-on: windows-11-arm
23+
timeout-minutes: 90
24+
25+
steps:
26+
27+
- uses: actions/[email protected]
28+
29+
- name: Set up Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: 3.12
33+
architecture: arm64
34+
35+
- name: Setup visual studio
36+
uses: microsoft/setup-msbuild@v2
37+
38+
- name: Download, install 7zip.
39+
run: |
40+
Invoke-WebRequest https://www.7-zip.org/a/7z2409-arm64.exe -UseBasicParsing -OutFile 7z_arm.exe
41+
Start-Process -FilePath ".\7z_arm.exe" -ArgumentList "/S" -Wait
42+
echo "C:\Program Files\7-Zip" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
43+
44+
- name: Download and install LLVM installer
45+
run: |
46+
Invoke-WebRequest https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/LLVM-19.1.5-woa64.exe -UseBasicParsing -OutFile LLVM-woa64.exe
47+
Start-Process -FilePath ".\LLVM-woa64.exe" -ArgumentList "/S" -Wait
48+
echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
49+
50+
- name: Update CMake for WoA
51+
run: |
52+
pip install cmake
53+
get-command cmake
54+
55+
- name: Build
56+
run: |
57+
git submodule update --init --recursive
58+
.\tools\build_steps_win_arm64.bat
59+
60+
- name: Pack
61+
run: |
62+
cd local
63+
cp -r "scipy_openblas${env:BUILD_BITS}" $env:BUILD_BITS
64+
7z a ../builds/openblas-${env:PLAT}-${env:INTERFACE64}.zip -tzip $env:BUILD_BITS
65+
66+
- name: Test 32-bit interface wheel
67+
run: |
68+
python -m pip install --no-index --find-links dist scipy_openblas32
69+
python -m scipy_openblas32
70+
python -c "import scipy_openblas32; print(scipy_openblas32.get_pkg_config())"
71+
72+
- uses: actions/[email protected]
73+
with:
74+
name: wheels-${{ env.PLAT }}-${{ env.INTERFACE64 }}
75+
path: dist/scipy_openblas*.whl
76+
77+
- uses: actions/[email protected]
78+
with:
79+
name: openblas-${{ env.PLAT }}-${{ env.INTERFACE64 }}
80+
path: builds/openblas*.zip
81+
82+
- name: Install Anaconda client
83+
run: |
84+
# Rust installation needed for rpds-py.
85+
Invoke-WebRequest https://static.rust-lang.org/rustup/dist/aarch64-pc-windows-msvc/rustup-init.exe -UseBasicParsing -Outfile rustup-init.exe
86+
.\rustup-init.exe -y
87+
$env:PATH="$env:PATH;$env:USERPROFILE\.cargo\bin"
88+
pip install anaconda-client
89+
90+
- name: Upload
91+
# see https://github.com/marketplace/actions/setup-miniconda for why
92+
# `-el {0}` is required.
93+
shell: bash -el {0}
94+
env:
95+
ANACONDA_SCIENTIFIC_PYTHON_UPLOAD: ${{ secrets.ANACONDA_SCIENTIFIC_PYTHON_UPLOAD }}
96+
run: |
97+
source tools/upload_to_anaconda_staging.sh
98+
upload_wheels

‎.github/workflows/windows.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88
workflow_dispatch: null
99

1010
env:
11-
OPENBLAS_COMMIT: "v0.3.29"
11+
OPENBLAS_COMMIT: "v0.3.29-265-gb5456c1b"
1212
OPENBLAS_ROOT: "c:\\opt"
1313
# Preserve working directory for calls into bash
1414
# Without this, invoking bash will cd to the home directory

‎OpenBLAS

‎pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
99
[project]
1010
name = "scipy-openblas64"
1111
# v0.3.29
12-
version = "0.3.29.0.0"
12+
version = "0.3.29.265.0"
1313
requires-python = ">=3.7"
1414
description = "Provides OpenBLAS for python packaging"
1515
readme = "README.md"

‎tools/build_steps_win_arm64.bat

100644100755
Lines changed: 115 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,92 @@
11
:: Build script for scipy_openblas wheel on Windows on ARM64
22

3-
:: Usage: build_steps_win_arm64.bat [build_bits]
4-
:: e.g build_steps_win_arm64.bat 64
3+
:: Usage: build_steps_win_arm64.bat [build_bits] [if_bits]
4+
:: e.g build_steps_win_arm64.bat 64 64
55

6-
:: BUILD_BITS (default binary architecture, 32 or 64, unspec -> 64).
6+
:: build_bits (default binary architecture, 32 or 64, unspec -> 64).
7+
:: if_bits (default interface size, 32 or 64, unspec -> 32)
8+
:: If INTERFACE64 environment variable is 1, then if_bits defaults to 64
79
:: Expects these binaries on the PATH:
8-
:: clang-cl, flang-new, cmake, perl
10+
:: clang-cl, flang-new, cmake, perl
11+
:: Uses environment variable:
12+
:: OPENBLAS_COMMIT (unspec -> current submodule commit, if contains
13+
:: Windows on ARM build fixes (see below), otherwise earliest commit
14+
:: with those fixes).
15+
16+
:: First commit containing WoA build fixes.
17+
:: Minimum OpenBLAS commit to build; we'll update to this if commit not
18+
:: present.
19+
set first_woa_buildable_commit="de2380e5a6149706a633322a16a0f66faa5591fc"
920

1021
@echo off
1122
setlocal enabledelayedexpansion
1223

1324
if "%1"=="" (
14-
set BUILD_BIT=64
25+
set build_bits=64
26+
) else (
27+
set build_bits=%1
28+
)
29+
if "%INTERFACE64%"=="1" (
30+
set "if_default=64"
31+
) else (
32+
set "if_default=32"
33+
)
34+
if "%2"=="" (
35+
set "if_bits=%if_default%"
1536
) else (
16-
set BUILD_BIT=%1
37+
set "if_bits=%2"
1738
)
18-
echo Building for %BUILD_BIT%-bit configuration...
39+
echo Building for %build_bits%-bit binary, %if_bits%-bit interface...
1940

2041
:: Define destination directory
21-
move "..\local\scipy_openblas64" "..\local\scipy_openblas32"
22-
set "DEST_DIR=%CD%\..\local\scipy_openblas32"
23-
cd ..
24-
25-
:: Check if 'openblas' folder exists and is empty
26-
if exist "openblas" (
27-
dir /b "openblas" | findstr . >nul
28-
if errorlevel 1 (
29-
echo OpenBLAS folder exists but is empty. Deleting and recloning...
30-
rmdir /s /q "openblas"
42+
pushd "%~dp0\.."
43+
set "ob_out_root=%CD%\local\scipy_openblas"
44+
set "ob_64=%ob_out_root%64"
45+
set "ob_32=%ob_out_root%32"
46+
set "local_dir=%CD%\local"
47+
for /d %%D in ("%local_dir%\*") do (
48+
if /I not "%%~nxD"=="scipy_openblas64" (
49+
rmdir /S /Q "%%D"
3150
)
3251
)
33-
34-
:: Clone OpenBLAS if not present
35-
if not exist "openblas" (
36-
echo Cloning OpenBLAS repository with submodules...
37-
git clone --recursive https://github.com/OpenMathLib/OpenBLAS.git OpenBLAS
38-
if errorlevel 1 exit /b 1
52+
if "%if_bits%"=="64" (
53+
set "DEST_DIR=%ob_64%"
54+
) else (
55+
if exist "%ob_64%" (
56+
xcopy /Y /H "%ob_64%\*.py" "%CD%\ob64_backup\"
57+
move "%ob_64%" "%ob_32%"
58+
set "DEST_DIR=%ob_32%"
59+
)
3960
)
61+
62+
:: Clone OpenBLAS
63+
echo Cloning OpenBLAS repository with submodules...
64+
git submodule update --init --recursive OpenBLAS
65+
if errorlevel 1 exit /b 1
4066

41-
:: Enter OpenBLAS directory and checkout develop branch
42-
cd openblas
43-
git checkout develop
44-
45-
echo Checked out to the latest branch of OpenBLAS.
67+
:: Enter OpenBLAS directory and checkout buildable commit
68+
cd OpenBLAS
69+
if defined OPENBLAS_COMMIT (
70+
echo Checking out OpenBLAS commit %OPENBLAS_COMMIT%
71+
git checkout %OPENBLAS_COMMIT%
72+
)
73+
git merge-base --is-ancestor %first_woa_buildable_commit% HEAD 2>NUL
74+
if errorlevel 1 (
75+
echo OpenBLAS commit does not contain WoA build fixes.
76+
echo Commit needs to contain %first_woa_buildable_commit%.
77+
exit /b 2
78+
)
4679

80+
:: Set suffixed-ILP64 flags
81+
if "%if_bits%"=="64" (
82+
set "interface_flags=-DINTERFACE64=1 -DSYMBOLSUFFIX=64_"
83+
) else (
84+
set "interface_flags="
85+
)
86+
4787
:: Create build directory and navigate to it
48-
if not exist build mkdir build
49-
cd build
88+
if exist build (rmdir /S /Q build || exit /b 1)
89+
mkdir build || exit /b 1 & cd build || exit /b 1
5090

5191
echo Setting up ARM64 Developer Command Prompt and running CMake...
5292

@@ -55,36 +95,42 @@ for /f "usebackq tokens=*" %%i in (`"C:\Program Files (x86)\Microsoft Visual Stu
5595

5696
:: Run CMake and Ninja build
5797
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DTARGET=ARMV8 -DBUILD_SHARED_LIBS=ON -DARCH=arm64 ^
58-
-DBINARY=%BUILD_BIT% -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_C_COMPILER=clang-cl ^
59-
-DCMAKE_Fortran_COMPILER=flang-new -DSYMBOLPREFIX="scipy_" -DLIBNAMEPREFIX="scipy_"
98+
-DBINARY=%build_bits% -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_C_COMPILER=clang-cl ^
99+
-DCMAKE_Fortran_COMPILER=flang-new -DSYMBOLPREFIX="scipy_" -DLIBNAMEPREFIX="scipy_" %interface_flags%
60100
if errorlevel 1 exit /b 1
61-
101+
62102
ninja
63103
if errorlevel 1 exit /b 1
64104

65105
echo Build complete. Returning to Batch.
66106

67-
:: Rewrite the name of the project to scipy-openblas32
68-
echo Rewrite to scipy_openblas32
69-
cd ../..
70-
powershell -Command "(Get-Content 'pyproject.toml') -replace 'openblas64', 'openblas32' | Set-Content 'pyproject.toml'"
71-
powershell -Command "(Get-Content 'local\scipy_openblas32\__main__.py') -replace 'openblas64', 'openblas32' | Out-File 'local\scipy_openblas32\__main__.py' -Encoding utf8"
72-
powershell -Command "(Get-Content 'local\scipy_openblas32\__init__.py') -replace 'openblas64', 'openblas32' | Out-File 'local\scipy_openblas32\__init__.py' -Encoding utf8"
73-
powershell -Command "(Get-Content 'local\scipy_openblas32\__init__.py') -replace 'openblas_get_config64_', 'openblas_get_config' | Out-File 'local\scipy_openblas32\__init__.py' -Encoding utf8"
74-
powershell -Command "(Get-Content 'local\scipy_openblas32\__init__.py') -replace 'cflags =.*', 'cflags = \"-DBLAS_SYMBOL_PREFIX=scipy_\"' | Out-File 'local\scipy_openblas32\__init__.py' -Encoding utf8"
107+
if "%if_bits%"=="32" (
108+
echo Rewrite to scipy_openblas32
109+
cd ../..
110+
set out_pyproject=pyproject_64_32.toml
111+
powershell -Command "(Get-Content 'pyproject.toml') -replace 'openblas64', 'openblas32' | Set-Content !out_pyproject!"
112+
powershell -Command "(Get-Content 'local\scipy_openblas32\__main__.py') -replace 'openblas64', 'openblas32' | Out-File 'local\scipy_openblas32\__main__.py' -Encoding utf8"
113+
powershell -Command "(Get-Content 'local\scipy_openblas32\__init__.py') -replace 'openblas64', 'openblas32' | Out-File 'local\scipy_openblas32\__init__.py' -Encoding utf8"
114+
powershell -Command "(Get-Content 'local\scipy_openblas32\__init__.py') -replace 'openblas_get_config64_', 'openblas_get_config' | Out-File 'local\scipy_openblas32\__init__.py' -Encoding utf8"
115+
powershell -Command "(Get-Content 'local\scipy_openblas32\__init__.py') -replace 'cflags =.*', 'cflags = \"-DBLAS_SYMBOL_PREFIX=scipy_\"' | Out-File 'local\scipy_openblas32\__init__.py' -Encoding utf8"
116+
)
75117

76118
:: Prepare destination directory
77119
cd OpenBLAS/build
78120
echo Preparing destination directory at %DEST_DIR%...
79-
if not exist "%DEST_DIR%\lib\cmake\openblas" mkdir "%DEST_DIR%\lib\cmake\openblas"
121+
if not exist "%DEST_DIR%\lib\cmake\OpenBLAS" mkdir "%DEST_DIR%\lib\cmake\OpenBLAS"
80122
if not exist "%DEST_DIR%\include" mkdir "%DEST_DIR%\include"
81123

82124
:: Move library files
83125
echo Moving library files...
84126
if exist lib\release (
85127
move /Y lib\release\*.dll "%DEST_DIR%\lib\"
86128
if errorlevel 1 exit /b 1
87-
move /Y lib\release\*.dll.a "%DEST_DIR%\lib\scipy_openblas.lib"
129+
for %%f in (lib\release\*.dll.a) do (
130+
set "orig_name=%%~nxf"
131+
call set "base_name=%%orig_name:.dll.a=%%"
132+
move /Y "%%f" "%DEST_DIR%\lib\!base_name!.lib"
133+
)
88134
if errorlevel 1 exit /b 1
89135
) else (
90136
echo Error: lib/release directory not found!
@@ -99,7 +145,7 @@ if exist openblasconfigversion.cmake copy /Y openblasconfigversion.cmake "%DEST_
99145
:: Copy header files
100146
echo Copying generated header files...
101147
if exist generated xcopy /E /Y generated "%DEST_DIR%\include\"
102-
if exist lapacke_mangling copy /Y lapacke_mangling "%DEST_DIR%\include\"
148+
if exist lapacke_mangling.h copy /Y lapacke_mangling.h "%DEST_DIR%\include\"
103149
if exist openblas_config.h copy /Y openblas_config.h "%DEST_DIR%\include\"
104150

105151

@@ -113,12 +159,32 @@ cd ../..
113159

114160
:: Build the Wheel & Install It
115161
echo Running 'python -m build' to build the wheel...
116-
python -m build
117-
if errorlevel 1 exit /b 1
118-
162+
python -c "import build" 2>NUL || pip install build
163+
if "%if_bits%"=="64" (
164+
python -m build
165+
if errorlevel 1 exit /b 1
166+
) else (
167+
move /Y pyproject.toml pyproject.toml.bak
168+
move /Y %out_pyproject% pyproject.toml
169+
python -m build
170+
if errorlevel 1 exit /b 1
171+
move /Y pyproject.toml.bak pyproject.toml
172+
)
173+
if "%if_bits%"=="32" (
174+
move /Y "%CD%\ob64_backup" "%ob_64%"
175+
)
176+
177+
:: Rename the wheel
178+
for %%f in (dist\*any.whl) do (
179+
set WHEEL_FILE=dist\%%f
180+
set "filename=%%~nxf"
181+
set "newname=!filename:any.whl=win_arm64.whl!"
182+
ren "dist\!filename!" "!newname!"
183+
)
184+
119185
:: Locate the built wheel
120186
for /f %%f in ('dir /b dist\scipy_openblas*.whl 2^>nul') do set WHEEL_FILE=dist\%%f
121-
187+
122188
if not defined WHEEL_FILE (
123189
echo Error: No wheel file found in dist folder.
124190
exit /b 1
@@ -129,4 +195,4 @@ pip install "%WHEEL_FILE%"
129195
if errorlevel 1 exit /b 1
130196

131197
echo Done.
132-
exit /b 0
198+
exit /b 0

0 commit comments

Comments
 (0)