diff --git a/.github/workflows/build-python-packages.yml b/.github/workflows/build-python-packages.yml index 3f24c128..802038c1 100644 --- a/.github/workflows/build-python-packages.yml +++ b/.github/workflows/build-python-packages.yml @@ -5,17 +5,22 @@ on: inputs: VERSION: description: 'Python version to build and upload' - default: '3.12.3' + default: '3.13.0' required: true PUBLISH_RELEASES: description: 'Whether to publish releases' required: true type: boolean default: false + THREADING_BUILD_MODES: + description: 'CPython threading build modes' + required: true + type: str + default: 'freethreaded' PLATFORMS: description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)' required: true - default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' + default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,macos-13_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86' pull_request: paths-ignore: - 'versions-manifest.json' @@ -40,32 +45,42 @@ jobs: id: generate-matrix run: | [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim() + [String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim() $matrix = @() foreach ($configuration in $configurations) { - $parts = $configuration.Split("_") - $os = $parts[0] - $arch = if ($parts[1]) {$parts[1]} else {"x64"} - switch -wildcard ($os) { - "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")} - "*macos*" { $platform = 'darwin' } - "*windows*" { $platform = 'win32' } - } - - if ($configuration -eq "ubuntu-22.04_arm64") { - $os = "setup-actions-ubuntu-arm64-2-core" - } - elseif ($configuration -eq "ubuntu-24.04_arm64") { - $os = "setup-actions-ubuntu24-arm64-2-core" - } - elseif ($configuration -eq "windows-2019_arm64") { - $os = "setup-actions-windows-arm64-4-core" - } - - $matrix += @{ - 'platform' = $platform - 'os' = $os - 'arch' = $arch + foreach ($buildMode in $buildModes) { + $parts = $configuration.Split("_") + $os = $parts[0] + $arch = if ($parts[1]) {$parts[1]} else {"x64"} + switch -wildcard ($os) { + "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")} + "*macos*" { $platform = 'darwin' } + "*windows*" { $platform = 'win32' } + } + + if ($configuration -eq "ubuntu-22.04_arm64") { + $os = "setup-actions-ubuntu-arm64-2-core" + } + elseif ($configuration -eq "ubuntu-24.04_arm64") { + $os = "setup-actions-ubuntu24-arm64-2-core" + } + elseif ($configuration -eq "windows-2019_arm64") { + $os = "setup-actions-windows-arm64-4-core" + } + + if ($buildMode -eq "freethreaded") { + if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") { + continue; + } + $arch += "-freethreaded" + } + + $matrix += @{ + 'platform' = $platform + 'os' = $os + 'arch' = $arch + } } } echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT @@ -201,6 +216,9 @@ jobs: python-version: ${{ env.VERSION }} architecture: ${{ matrix.arch }} + - name: Python version + run: python -VVV + - name: Verbose sysconfig dump if: runner.os == 'Linux' || runner.os == 'macOS' run: python ./sources/python-config-output.py diff --git a/.github/workflows/create-pr-to-update-manifest.yml b/.github/workflows/create-pr-to-update-manifest.yml new file mode 100644 index 00000000..b4795e92 --- /dev/null +++ b/.github/workflows/create-pr-to-update-manifest.yml @@ -0,0 +1,43 @@ +# This reusable workflow is used by actions/*-versions repositories +# It is designed to create a PR with update of versions-manifest.json when a new release is published +# The GITHUB_TOKEN secret is used to create versions-manifest.json and publish related PR + +name: Create Pull Request (called indirectly) +on: + workflow_call: + inputs: + tool-name: + description: 'Name of the tool for which PR is created' + required: true + type: string + +defaults: + run: + shell: pwsh + +jobs: + create_pr: + name: Create Pull Request + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Create versions-manifest.json + run: | + ./nogil-helpers/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" ` + -GitHubAccessToken "${{ secrets.GITHUB_TOKEN }}" ` + -OutputFile "./versions-manifest.json" ` + -ConfigurationFile "./config/${{ inputs.tool-name }}-manifest-config.json" + + - name: Create GitHub PR + run: | + $formattedDate = Get-Date -Format "MM/dd/yyyy" + ./helpers/github/create-pull-request.ps1 ` + -RepositoryFullName "$env:GITHUB_REPOSITORY" ` + -AccessToken "${{ secrets.GITHUB_TOKEN }}" ` + -BranchName "update-versions-manifest-file" ` + -CommitMessage "Update versions-manifest" ` + -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" ` + -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}" \ No newline at end of file diff --git a/.github/workflows/create-pr.yml b/.github/workflows/create-pr.yml index 3b6abfd3..d08c6d52 100644 --- a/.github/workflows/create-pr.yml +++ b/.github/workflows/create-pr.yml @@ -4,7 +4,7 @@ on: jobs: create-pr: - uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main + uses: ./.github/workflows/create-pr-to-update-manifest.yml with: tool-name: "python" secrets: inherit diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 6b36fddc..fe442c48 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -151,6 +151,37 @@ class macOSPythonBuilder : NixPythonBuilder { return $pkgLocation } + [string] GetFrameworkName() { + <# + .SYNOPSIS + Get the Python installation Package name. + #> + + if ($this.IsFreeThreaded()) { + return "PythonT.framework" + } else { + return "Python.framework" + } + } + + [string] GetPkgChoices() { + <# + .SYNOPSIS + Reads the configuration XML file for the Python installer + #> + + $config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" } + $choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml" + $choicesTemplate = Get-Content -Path $choicesFile -Raw + + $variablesToReplace = @{ + "{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)"; + } + + $variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) } + return $choicesTemplate + } + [void] CreateInstallationScriptPkg() { <# .SYNOPSIS @@ -165,6 +196,8 @@ class macOSPythonBuilder : NixPythonBuilder { "{{__VERSION_FULL__}}" = $this.Version; "{{__PKG_NAME__}}" = $this.GetPkgName(); "{{__ARCH__}}" = $this.Architecture; + "{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName(); + "{{__PKG_CHOICES__}}" = $this.GetPkgChoices(); } $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } diff --git a/builders/nix-python-builder.psm1 b/builders/nix-python-builder.psm1 index b15878e0..ea73539d 100644 --- a/builders/nix-python-builder.psm1 +++ b/builders/nix-python-builder.psm1 @@ -115,7 +115,7 @@ class NixPythonBuilder : PythonBuilder { Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)" $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File - Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue + Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue Execute-Command -Command "make install" -ErrorAction Continue Write-Debug "Done; Make log location: $buildOutputLocation" diff --git a/builders/python-builder.psm1 b/builders/python-builder.psm1 index c2541d37..fb8fe7b4 100644 --- a/builders/python-builder.psm1 +++ b/builders/python-builder.psm1 @@ -94,6 +94,24 @@ class PythonBuilder { return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)" } + [string] GetHardwareArchitecture() { + <# + .SYNOPSIS + The hardware architecture (x64, arm64) without any Python free threading suffix. + #> + + return $this.Architecture.Replace("-freethreaded", "") + } + + [bool] IsFreeThreaded() { + <# + .SYNOPSIS + Check if Python version is free threaded. + #> + + return $this.Architecture.EndsWith("-freethreaded") + } + [void] PreparePythonToolcacheLocation() { <# .SYNOPSIS diff --git a/builders/ubuntu-python-builder.psm1 b/builders/ubuntu-python-builder.psm1 index 35b159c5..d7b13eb0 100644 --- a/builders/ubuntu-python-builder.psm1 +++ b/builders/ubuntu-python-builder.psm1 @@ -37,6 +37,14 @@ class UbuntuPythonBuilder : NixPythonBuilder { $configureString += " --enable-shared" $configureString += " --enable-optimizations" + if ($this.IsFreeThreaded()) { + if ($this.Version -lt "3.13.0") { + Write-Host "Python versions lower than 3.13.0 do not support free threading" + exit 1 + } + $configureString += " --disable-gil" + } + ### Compile with support of loadable sqlite extensions. ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension) $configureString += " --enable-loadable-sqlite-extensions" diff --git a/builders/win-python-builder.psm1 b/builders/win-python-builder.psm1 index 95d11bc3..9614764a 100644 --- a/builders/win-python-builder.psm1 +++ b/builders/win-python-builder.psm1 @@ -54,13 +54,13 @@ class WinPythonBuilder : PythonBuilder { #> $ArchitectureExtension = "" - if ($this.Architecture -eq "x64") { + if ($this.GetHardwareArchitecture() -eq "x64") { if ($this.Version -ge "3.5") { $ArchitectureExtension = "-amd64" } else { $ArchitectureExtension = ".amd64" } - }elseif ($this.Architecture -eq "arm64") { + } elseif ($this.GetHardwareArchitecture() -eq "arm64") { $ArchitectureExtension = "-arm64" } @@ -113,6 +113,7 @@ class WinPythonBuilder : PythonBuilder { $variablesToReplace = @{ "{{__ARCHITECTURE__}}" = $this.Architecture; + "{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture(); "{{__VERSION__}}" = $this.Version; "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName } diff --git a/config/macos-pkg-choices-default.xml b/config/macos-pkg-choices-default.xml new file mode 100644 index 00000000..4cfb0166 --- /dev/null +++ b/config/macos-pkg-choices-default.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/config/macos-pkg-choices-freethreaded.xml b/config/macos-pkg-choices-freethreaded.xml new file mode 100644 index 00000000..1a6f403c --- /dev/null +++ b/config/macos-pkg-choices-freethreaded.xml @@ -0,0 +1,14 @@ + + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}} + + + diff --git a/config/python-manifest-config.json b/config/python-manifest-config.json index d2529b00..7390b920 100644 --- a/config/python-manifest-config.json +++ b/config/python-manifest-config.json @@ -1,5 +1,5 @@ { - "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)", + "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)", "groups": { "arch": 4, "platform": 2, diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh index 5e1aa6ef..a6c909ee 100644 --- a/installers/macos-pkg-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -2,6 +2,11 @@ set -e PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" PYTHON_PKG_NAME="{{__PKG_NAME__}}" +PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}" +PYTHON_PKG_CHOICES=$(cat << 'EOF' +{{__PKG_CHOICES__}} +EOF +) ARCH="{{__ARCH__}}" MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) @@ -20,7 +25,7 @@ fi PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH -PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}" +PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}" PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}" echo "Check if Python hostedtoolcache folder exist..." @@ -38,8 +43,11 @@ else done fi +PYTHON_PKG_CHOICES_FILES=$(mktemp) +echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES + echo "Install Python binaries from prebuilt package" -sudo installer -pkg $PYTHON_PKG_NAME -target / +sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target / echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" echo "Create Python $PYTHON_FULL_VERSION folder" @@ -53,7 +61,9 @@ ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)" ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python +chmod +x python +# Note that bin is a symlink so referencing .. from bin will not work as expected cd bin/ # This symlink already exists if Python version with the same major.minor version is installed, @@ -62,11 +72,15 @@ if [ ! -f $PYTHON_MAJOR_MINOR ]; then ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR fi +if [ ! -f $PYTHON_MAJOR ]; then + ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR +fi + if [ ! -f python ]; then ln -s $PYTHON_MAJOR_DOT_MINOR python fi -chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python +chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python echo "Upgrading pip..." export PIP_ROOT_USER_ACTION=ignore diff --git a/installers/win-setup-template.ps1 b/installers/win-setup-template.ps1 index e2a33b8b..5aaad80e 100644 --- a/installers/win-setup-template.ps1 +++ b/installers/win-setup-template.ps1 @@ -1,4 +1,5 @@ [String] $Architecture = "{{__ARCHITECTURE__}}" +[String] $HardwareArchitecture = "{{__HARDWARE_ARCHITECTURE__}}" [String] $Version = "{{__VERSION__}}" [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}" @@ -25,7 +26,7 @@ function Remove-RegistryEntries { [Parameter(Mandatory)][Int32] $MinorVersion ) - $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion + $versionFilter = Get-RegistryVersionFilter -Architecture $HardwareArchitecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products" if (Test-Path -Path Registry::$regPath) { @@ -61,13 +62,15 @@ function Remove-RegistryEntries { function Get-ExecParams { param( [Parameter(Mandatory)][Boolean] $IsMSI, + [Parameter(Mandatory)][Boolean] $IsFreeThreaded, [Parameter(Mandatory)][String] $PythonArchPath ) if ($IsMSI) { "TARGETDIR=$PythonArchPath ALLUSERS=1" } else { - "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1" + $Include_freethreaded = if ($IsFreeThreaded) { "Include_freethreaded=1" } else { "" } + "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1 $Include_freethreaded" } } @@ -81,6 +84,7 @@ $PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture $IsMSI = $PythonExecName -match "msi" +$IsFreeThreaded = $Architecture -match "-freethreaded" $MajorVersion = $Version.Split('.')[0] $MinorVersion = $Version.Split('.')[1] @@ -120,13 +124,24 @@ Write-Host "Copy Python binaries to $PythonArchPath" Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null Write-Host "Install Python $Version in $PythonToolcachePath..." -$ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath +$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet" if ($LASTEXITCODE -ne 0) { Throw "Error happened during Python installation" } +# print out all files in $PythonArchPath +Write-Host "Files in $PythonArchPath" +$files = Get-ChildItem -Path $PythonArchPath -File -Recurse +Write-Output $files + +if ($IsFreeThreaded) { + # Delete python.exe and create a symlink to free-threaded exe + Remove-Item -Path "$PythonArchPath\python.exe" -Force + New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe" +} + Write-Host "Create `python3` symlink" if ($MajorVersion -ne "2") { New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe" diff --git a/nogil-helpers/manifest-generator.ps1 b/nogil-helpers/manifest-generator.ps1 new file mode 100644 index 00000000..a725caa9 --- /dev/null +++ b/nogil-helpers/manifest-generator.ps1 @@ -0,0 +1,35 @@ +<# +.SYNOPSIS +Generate versions manifest based on repository releases +.DESCRIPTION +Versions manifest is needed to find the latest assets for particular version of tool +.PARAMETER RepositoryFullName +Required parameter. The owner and repository name. For example, 'actions/versions-package-tools' +.PARAMETER GitHubAccessToken +Required parameter. PAT Token to overcome GitHub API Rate limit +.PARAMETER OutputFile +Required parameter. File "*.json" where generated results will be saved +.PARAMETER ConfigurationFile +Path to the json file with parsing configuration +#> + +param ( + [Parameter(Mandatory)] [string] $RepositoryFullName, + [Parameter(Mandatory)] [string] $GitHubAccessToken, + [Parameter(Mandatory)] [string] $OutputFile, + [Parameter(Mandatory)] [string] $ConfigurationFile +) + +Import-Module (Join-Path $PSScriptRoot "../helpers/github/github-api.psm1") +Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force + +$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile + +$gitHubApi = Get-GitHubApi -RepositoryFullName $RepositoryFullName -AccessToken $GitHubAccessToken +$noGILreleases = $gitHubApi.GetReleases() + +$upstreamApi = Get-GitHubApi -RepositoryFullName "actions/python-versions" -AccessToken $GitHubAccessToken +$releases = $upstreamApi.GetReleases() + +$versionIndex = Build-VersionsManifest -Releases $releases -NoGILReleases $noGILreleases -Configuration $configuration +$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/nogil-helpers/manifest-utils.psm1 b/nogil-helpers/manifest-utils.psm1 new file mode 100644 index 00000000..5ec5d96c --- /dev/null +++ b/nogil-helpers/manifest-utils.psm1 @@ -0,0 +1,151 @@ +function Read-ConfigurationFile { + param ([Parameter(Mandatory)][string]$Filepath) + return Get-Content $Filepath -Raw | ConvertFrom-Json +} + +function New-AssetItem { + param ( + [Parameter(Mandatory)][object]$ReleaseAsset, + [Parameter(Mandatory)][object]$Configuration + ) + $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex) + if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" } + + $result = New-Object PSObject + $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty + $Configuration.groups.PSObject.Properties | ForEach-Object { + if (($_.Value).GetType().Name.StartsWith("Int")) { + $value = $regexResult.Groups[$_.Value].Value + } else { + $value = $_.Value + } + + if (-not ([string]::IsNullOrEmpty($value))) { + $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty + } + } + + $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty + return $result +} + +function Get-VersionFromRelease { + param ( + [Parameter(Mandatory)][object]$Release + ) + # Release name can contain additional information after ':' so filter it + [string]$releaseName = $Release.name.Split(':')[0] + [Semver]$version = $null + if (![Semver]::TryParse($releaseName, [ref]$version)) { + throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" + } + + return $version +} + +function Build-VersionsManifest { + param ( + [Parameter(Mandatory)][array]$Releases, + [Parameter(Mandatory)][array]$NoGILReleases, + [Parameter(Mandatory)][object]$Configuration + ) + + $Releases = $Releases | Sort-Object -Property "published_at" -Descending + $NoGILReleases = $NoGILReleases | Sort-Object -Property "published_at" -Descending + $ltsRules = Get-LtsRules -Configuration $Configuration + + $versionsHash = @{} + foreach ($release in $Releases) { + if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { + continue + } + + [Semver]$version = Get-VersionFromRelease $release + $versionKey = $version.ToString() + + if ($versionsHash.ContainsKey($versionKey)) { + continue + } + + $ltsStatus = Get-VersionLtsStatus -Version $versionKey -LtsRules $ltsRules + $stable = $version.PreReleaseLabel ? $false : $true + [array]$releaseAssets = $release.assets | Where { $_.Name -ne "hashes.sha256" } | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration } + + $versionHash = [PSCustomObject]@{} + $versionHash | Add-Member -Name "version" -Value $versionKey -MemberType NoteProperty + $versionHash | Add-Member -Name "stable" -Value $stable -MemberType NoteProperty + if ($ltsStatus) { + $versionHash | Add-Member -Name "lts" -Value $ltsStatus -MemberType NoteProperty + } + $versionHash | Add-Member -Name "release_url" -Value $release.html_url -MemberType NoteProperty + $versionHash | Add-Member -Name "files" -Value $releaseAssets -MemberType NoteProperty + $versionsHash.Add($versionKey, $versionHash) + } + + $noGILversionsHash = @{} + foreach ($release in $NoGILReleases) { + if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { + continue + } + + [Semver]$version = Get-VersionFromRelease $release + $versionKey = $version.ToString() + + if ($noGILversionsHash.ContainsKey($versionKey)) { + continue + } + + $ltsStatus = Get-VersionLtsStatus -Version $versionKey -LtsRules $ltsRules + $stable = $version.PreReleaseLabel ? $false : $true + [array]$releaseAssets = $release.assets | Where { $_.Name -ne "hashes.sha256" } | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration } + + $versionHash = [PSCustomObject]@{} + $versionHash | Add-Member -Name "version" -Value $versionKey -MemberType NoteProperty + $versionHash | Add-Member -Name "stable" -Value $stable -MemberType NoteProperty + if ($ltsStatus) { + $versionHash | Add-Member -Name "lts" -Value $ltsStatus -MemberType NoteProperty + } + $versionHash | Add-Member -Name "release_url" -Value $release.html_url -MemberType NoteProperty + $versionHash | Add-Member -Name "files" -Value $releaseAssets -MemberType NoteProperty + + if ($versionsHash.ContainsKey($versionKey)) { + # filter out filenames already in the versionsHash + $releaseAssets = $releaseAssets | Where-Object { $versionsHash[$versionKey].files.filename -notcontains $_.filename } + $versionsHash[$versionKey].files += $releaseAssets + } else { + $versionsHash.Add($versionKey, $versionHash) + } + $noGILversionsHash.Add($versionKey, $versionHash) + } + + # Sort versions by descending + return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Semver]$_.version }; Descending = $true } +} + +function Get-LtsRules { + param ( + [Parameter(Mandatory)][object]$Configuration + ) + + $ruleExpression = $Configuration."lts_rule_expression" + if ($ruleExpression) { + Invoke-Expression $ruleExpression + } else { + @() + } +} + +function Get-VersionLtsStatus { + param ( + [Parameter(Mandatory)][string]$Version, + [array]$LtsRules + ) + + foreach ($ltsRule in $LtsRules) { + if (($Version -eq $ltsRule.Name) -or ($Version.StartsWith("$($ltsRule.Name)."))) { + return $ltsRule.Value + } + } + + return $null +} diff --git a/tests/python-tests.ps1 b/tests/python-tests.ps1 index 706a3b42..0f9d2abe 100644 --- a/tests/python-tests.ps1 +++ b/tests/python-tests.ps1 @@ -7,6 +7,8 @@ param ( $Architecture ) +$HardwareArchitecture = $Architecture -replace "-freethreaded", "" + Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1") Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1") @@ -58,7 +60,7 @@ Describe "Tests" { # } # } - if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) { + if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($HardwareArchitecture -ne "arm64"))) { It "Check if sqlite3 module is installed" { "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode } diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index ea40a8ae..c1030098 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -8,25 +8,31 @@ version = sys.argv[1] nativeVersion = sys.argv[2] architecture = sys.argv[3] +hw_architecture = architecture.replace('-freethreaded', '') versions=version.split(".") version_major=int(versions[0]) version_minor=int(versions[1]) -pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64")) +pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (hw_architecture == "arm64")) lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY') is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED') have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE") +is_free_threaded = sysconfig.get_config_var('Py_GIL_DISABLED') ### Define expected variables if os_type == 'Linux': expected_ld_library_extension = 'so' if os_type == 'Darwin': expected_ld_library_extension = 'dylib' +if is_free_threaded: + framework_name = 'PythonT.framework' +else: + framework_name = 'Python.framework' if pkg_installer: - expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib' + expected_lib_dir_path = f'/Library/Frameworks/{framework_name}/Versions/{version_major}.{version_minor}/lib' else: expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'