diff --git a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec index de2689704..361995fb4 100644 --- a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec +++ b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec @@ -48,6 +48,11 @@ + + + + + diff --git a/eng/pack/scripts/rc_mac_arm64_deps.sh b/eng/pack/scripts/rc_mac_arm64_deps.sh new file mode 100644 index 000000000..d26561d03 --- /dev/null +++ b/eng/pack/scripts/rc_mac_arm64_deps.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +echo "=== Upgrading pip and installing build dependencies ===" +python -m pip install --upgrade pip setuptools wheel cython + +echo "=== Cloning gRPC repo ===" +rm -rf grpc +git clone --recursive https://github.com/grpc/grpc + +echo "=== Building grpcio wheel from source ===" +cd grpc +git submodule update --init --recursive +export GRPC_PYTHON_BUILD_WITH_CYTHON=1 + +# Build the wheel into dist/ +python -m pip wheel . -w dist + +# Log contents of dist +echo "=== Checking dist directory ===" +if [ -d dist ]; then + ls -lh dist +else + echo "dist/ directory not found!" +fi + +# Log and install grpcio +GRPC_WHEEL=$(ls dist/grpcio-*.whl | head -n 1) +echo "Built grpcio wheel: $(basename "$GRPC_WHEEL")" +echo "=== Install grpcio wheel $(basename "$GRPC_WHEEL") into root ===" +python -m pip install "$GRPC_WHEEL" + +cd .. + +# Change back to project root +cd workers + +echo "=== Install other deps into root ===" +python -m pip install . +python -m pip install grpcio-tools==1.70.0 + +echo "=== Installing grpcio into deps/ ===" +python -m pip install "$GRPC_WHEEL" --target "$BUILD_SOURCESDIRECTORY/deps" + +echo "=== Installing other deps into deps/ ===" +python -m pip install --upgrade pip setuptools wheel cython --target "$BUILD_SOURCESDIRECTORY/deps" +python -m pip install . azure-functions --no-compile --target "$BUILD_SOURCESDIRECTORY/deps" --find-links ../grpc/dist +python -m pip install grpcio-tools==1.70.0 --no-compile --target "$BUILD_SOURCESDIRECTORY/deps" --find-links ../grpc/dist + +echo "=== Install invoke and build protos ===" +python -m pip install invoke +cd tests +python -m invoke -c test_setup build-protos + +echo "=== Copying .artifactignore ===" +cd .. +cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps" + +echo "=== Copying protos ===" +version_minor=$(echo $1 | cut -d '.' -f 2) +if [[ $version_minor -lt 13 ]]; then + cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker" +else + cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker" +fi \ No newline at end of file diff --git a/eng/pack/scripts/rc_nix_deps.sh b/eng/pack/scripts/rc_nix_deps.sh new file mode 100644 index 000000000..6a89e85e9 --- /dev/null +++ b/eng/pack/scripts/rc_nix_deps.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +python -m venv .env +source .env/bin/activate +python -m pip install --upgrade pip + +cd workers +python -m pip install . +python -m pip install grpcio~=1.70.0 +python -m pip install grpcio-tools~=1.70.0 + +python -m pip install . --no-compile --target "$BUILD_SOURCESDIRECTORY/deps" +python -m pip install grpcio~=1.70.0 --no-compile --target "$BUILD_SOURCESDIRECTORY/deps" +python -m pip install grpcio-tools~=1.70.0 --no-compile --target "$BUILD_SOURCESDIRECTORY/deps" + +python -m pip install invoke +cd tests +python -m invoke -c test_setup build-protos + +cd .. +cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps" + +version_minor=$(echo $1 | cut -d '.' -f 2) +if [[ $version_minor -lt 13 ]]; then + cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker" +else + cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker" +fi \ No newline at end of file diff --git a/eng/pack/scripts/rc_win_deps.ps1 b/eng/pack/scripts/rc_win_deps.ps1 new file mode 100644 index 000000000..5a3b0130c --- /dev/null +++ b/eng/pack/scripts/rc_win_deps.ps1 @@ -0,0 +1,74 @@ +param ( + [string]$pythonVersion +) +$versionParts = $pythonVersion -split '\.' # Splitting by dot +$versionMinor = [int]$versionParts[1] + +Write-Host "=== Upgrading pip and installing build dependencies ===" +python -m pip install --upgrade pip setuptools wheel cython + +Write-Host "=== Cloning gRPC repo ===" +if (Test-Path grpc) { + Remove-Item -Recurse -Force grpc +} +git clone --recursive https://github.com/grpc/grpc + +Write-Host "=== Building grpcio from source ===" +Set-Location grpc +$env:GRPC_PYTHON_BUILD_WITH_CYTHON = "1" + +# Build the wheel into dist/ +python -m pip wheel . -w dist + +# Log contents of dist +Write-Host "=== Checking dist directory ===" +if (Test-Path dist) { + Get-ChildItem dist +} else { + Write-Host "dist/ directory not found!" +} + +# Log and install grpc +$grpcWheel = Get-ChildItem dist\grpcio-*.whl | Select-Object -First 1 +Write-Host "Built grpcio wheel: $($grpcWheel.Name)" +Write-Host "=== Install grpcio wheel $($grpcWheel.Name) into root ===" +$grpcWheel = Get-ChildItem dist\grpcio-*.whl | Select-Object -First 1 +python -m pip install $grpcWheel.FullName + +cd .. + +# Change back to project root +Set-Location workers + +Write-Host "=== Install other deps into root ===" +python -m pip install . +python -m pip install grpcio-tools==1.70.0 + +$depsPath = Join-Path -Path $env:BUILD_SOURCESDIRECTORY -ChildPath "deps" + +# Install both grpc wheels into deps +Write-Host "=== Installing grpcio into deps/ ===" +python -m pip install $grpcWheel.FullName --target $depsPath + +Write-Host "=== Installing other deps into deps/ ===" +python -m pip install --upgrade pip setuptools wheel cython --target $depsPath +python -m pip install . azure-functions --no-compile --target $depsPath --find-links ..\grpc\dist +python -m pip install grpcio-tools==1.70.0 --no-compile --target $depsPath --find-links ..\grpc\dist + +Write-Host "=== Install invoke and build protos ===" +python -m pip install invoke +cd tests +python -m invoke -c test_setup build-protos + +Write-Host "=== Copying .artifactignore ===" +cd .. +Copy-Item -Path ".artifactignore" -Destination $depsPath.ToString() + +Write-Host "=== Copying protos ===" +if ($versionMinor -lt 13) { + $protosPath = Join-Path -Path $depsPath -ChildPath "azure_functions_worker/protos" + Copy-Item -Path "azure_functions_worker/protos/*" -Destination $protosPath.ToString() -Recurse -Force +} else { + $protosPath = Join-Path -Path $depsPath -ChildPath "proxy_worker/protos" + Copy-Item -Path "proxy_worker/protos/*" -Destination $protosPath.ToString() -Recurse -Force +} \ No newline at end of file diff --git a/eng/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml index 2ea91858f..71b4c097f 100644 --- a/eng/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -7,6 +7,7 @@ steps: - task: UsePythonVersion@0 inputs: versionSpec: ${{ parameters.pythonVersion }} + allowUnstable: true addToPath: true - powershell: | # Parse the Python minor version @@ -22,6 +23,16 @@ steps: Write-Host "Minor version: $PY_MINOR" Write-Host "##vso[task.setvariable variable=minorVersion;]$PY_MINOR" + # Detect if this is an RC build + if ($PY_VER -match "rc") { + Write-Host "RC version detected" + Write-Host "##vso[task.setvariable variable=isRC;]true" + } + else { + Write-Host "Stable version detected" + Write-Host "##vso[task.setvariable variable=isRC;]false" + } + # Set build-related variables based on Python minor version if( $PY_MINOR -ge 13 ) { @@ -45,7 +56,15 @@ steps: disableAutoCwd: true scriptPath: 'eng/pack/scripts/mac_arm64_deps.sh' args: '${{ parameters.pythonVersion }}' - displayName: 'Install Dependencies' + displayName: 'Install dependencies' + condition: eq(variables['isRC'], 'false') +- task: ShellScript@2 + inputs: + disableAutoCwd: true + scriptPath: 'eng/pack/scripts/rc_mac_arm64_deps.sh' + args: '${{ parameters.pythonVersion }}' + displayName: 'Build dependencies from source' + condition: eq(variables['isRC'], 'true') - bash: | pip install pip-audit cd workers @@ -161,6 +180,14 @@ steps: !werkzeug/debug/shared/debugger.js !azure_functions_worker/** !dateutil/** + !Cython/** + !__pycache__/** + !bin/** + !cython.py + !pip/** + !pyximport/** + !typing_extensions.py + !wheel/** targetFolder: '$(Build.ArtifactStagingDirectory)' condition: eq(variables['proxyWorker'], true) displayName: 'Copy proxy_worker files' diff --git a/eng/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml index c29188c56..3d2cd8a0b 100644 --- a/eng/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -7,6 +7,7 @@ steps: - task: UsePythonVersion@0 inputs: versionSpec: ${{ parameters.pythonVersion }} + allowUnstable: true addToPath: true - powershell: | # Parse the Python minor version @@ -22,6 +23,16 @@ steps: Write-Host "Minor version: $PY_MINOR" Write-Host "##vso[task.setvariable variable=minorVersion;]$PY_MINOR" + # Detect if this is an RC build + if ($PY_VER -match "rc") { + Write-Host "RC version detected" + Write-Host "##vso[task.setvariable variable=isRC;]true" + } + else { + Write-Host "Stable version detected" + Write-Host "##vso[task.setvariable variable=isRC;]false" + } + # Set build-related variables based on Python minor version if( $PY_MINOR -ge 13 ) { @@ -46,6 +57,14 @@ steps: scriptPath: 'eng/pack/scripts/nix_deps.sh' args: '${{ parameters.pythonVersion }}' displayName: 'Install Dependencies' + condition: eq(variables['isRC'], 'false') +- task: ShellScript@2 + inputs: + disableAutoCwd: true + scriptPath: 'eng/pack/scripts/rc_nix_deps.sh' + args: '${{ parameters.pythonVersion }}' + displayName: 'Build dependencies from source' + condition: eq(variables['isRC'], 'true') - bash: | pip install pip-audit cd workers diff --git a/eng/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml index 833f3e9ef..af90096cc 100644 --- a/eng/pack/templates/win_env_gen.yml +++ b/eng/pack/templates/win_env_gen.yml @@ -7,6 +7,7 @@ steps: - task: UsePythonVersion@0 inputs: versionSpec: ${{ parameters.pythonVersion }} + allowUnstable: true architecture: ${{ parameters.architecture }} addToPath: true - powershell: | @@ -23,6 +24,16 @@ steps: Write-Host "Minor version: $PY_MINOR" Write-Host "##vso[task.setvariable variable=minorVersion;]$PY_MINOR" + # Detect if this is an RC build + if ($PY_VER -match "rc") { + Write-Host "RC version detected" + Write-Host "##vso[task.setvariable variable=isRC;]true" + } + else { + Write-Host "Stable version detected" + Write-Host "##vso[task.setvariable variable=isRC;]false" + } + # Set build-related variables based on Python minor version if( $PY_MINOR -ge 13 ) { @@ -45,6 +56,14 @@ steps: inputs: filePath: 'eng\pack\scripts\win_deps.ps1' arguments: '${{ parameters.pythonVersion }}' + displayName: 'Install dependencies' + condition: eq(variables['isRC'], 'false') +- task: PowerShell@2 + inputs: + filePath: 'eng\pack\scripts\rc_win_deps.ps1' + arguments: '${{ parameters.pythonVersion }}' + displayName: 'Build dependencies from source' + condition: eq(variables['isRC'], 'true') - bash: | pip install pip-audit cd workers @@ -160,6 +179,14 @@ steps: !werkzeug\debug\shared\debugger.js !dateutil\** !azure_functions_worker\** + !Cython\** + !__pycache__\** + !bin\** + !cython.py + !pip\** + !pyximport\** + !typing_extensions.py + !wheel\** targetFolder: '$(Build.ArtifactStagingDirectory)' condition: eq(variables['proxyWorker'], true) displayName: 'Copy proxy_worker files' diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index 91f2b94df..5b0b1581f 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -1,5 +1,6 @@ jobs: - job: Build_WINDOWS_X64 + timeoutInMinutes: 180 pool: name: 1es-pool-azfunc image: 1es-windows-2022 @@ -8,32 +9,43 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' + normalizedPythonVersion: '3.7' Python38V4: pythonVersion: '3.8' + normalizedPythonVersion: '3.8' Python39V4: pythonVersion: '3.9' + normalizedPythonVersion: '3.9' Python310V4: pythonVersion: '3.10' + normalizedPythonVersion: '3.10' Python311V4: pythonVersion: '3.11' + normalizedPythonVersion: '3.11' Python312V4: pythonVersion: '3.12' + normalizedPythonVersion: '3.12' Python313V4: pythonVersion: '3.13' + normalizedPythonVersion: '3.13' + Python314V4: + pythonVersion: '3.14.0-rc.2' + normalizedPythonVersion: '3.14' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_WINDOWS_X64" + artifactName: "$(normalizedPythonVersion)_WINDOWS_X64" steps: - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' architecture: 'x64' - artifactName: '$(pythonVersion)_WINDOWS_X64' + artifactName: '$(normalizedPythonVersion)_WINDOWS_X64' grpcBuild: 'win_amd64.pyd' - job: Build_WINDOWS_X86 + timeoutInMinutes: 180 pool: name: 1es-pool-azfunc image: 1es-windows-2022 @@ -42,32 +54,43 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' + normalizedPythonVersion: '3.7' Python38V4: pythonVersion: '3.8' + normalizedPythonVersion: '3.8' Python39V4: pythonVersion: '3.9' + normalizedPythonVersion: '3.9' Python310V4: pythonVersion: '3.10' + normalizedPythonVersion: '3.10' Python311V4: pythonVersion: '3.11' + normalizedPythonVersion: '3.11' Python312V4: pythonVersion: '3.12' + normalizedPythonVersion: '3.12' Python313V4: pythonVersion: '3.13' + normalizedPythonVersion: '3.13' + Python314V4: + pythonVersion: '3.14.0-rc.2' + normalizedPythonVersion: '3.14' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_WINDOWS_X86" + artifactName: "$(normalizedPythonVersion)_WINDOWS_X86" steps: - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' architecture: 'x86' - artifactName: '$(pythonVersion)_WINDOWS_x86' + artifactName: '$(normalizedPythonVersion)_WINDOWS_x86' grpcBuild: 'win32.pyd' - job: Build_LINUX_X64 + timeoutInMinutes: 180 pool: name: 1es-pool-azfunc image: 1es-ubuntu-22.04 @@ -76,31 +99,42 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' + normalizedPythonVersion: '3.7' Python38V4: pythonVersion: '3.8' + normalizedPythonVersion: '3.8' Python39V4: pythonVersion: '3.9' + normalizedPythonVersion: '3.9' Python310V4: pythonVersion: '3.10' + normalizedPythonVersion: '3.10' Python311V4: pythonVersion: '3.11' + normalizedPythonVersion: '3.11' Python312V4: pythonVersion: '3.12' + normalizedPythonVersion: '3.12' Python313V4: pythonVersion: '3.13' + normalizedPythonVersion: '3.13' + Python314V4: + pythonVersion: '3.14.0-rc.2' + normalizedPythonVersion: '3.14' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_LINUX_X64" + artifactName: "$(normalizedPythonVersion)_LINUX_X64" steps: - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - artifactName: '$(pythonVersion)_LINUX_X64' + artifactName: '$(normalizedPythonVersion)_LINUX_X64' grpcBuild: 'x86_64-linux-gnu.so' - job: Build_OSX_X64 + timeoutInMinutes: 180 pool: name: Azure Pipelines image: macOS-latest @@ -109,31 +143,42 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' + normalizedPythonVersion: '3.7' Python38V4: pythonVersion: '3.8' + normalizedPythonVersion: '3.8' Python39V4: pythonVersion: '3.9' + normalizedPythonVersion: '3.9' Python310V4: pythonVersion: '3.10' + normalizedPythonVersion: '3.10' Python311V4: pythonVersion: '3.11' + normalizedPythonVersion: '3.11' Python312V4: pythonVersion: '3.12' + normalizedPythonVersion: '3.12' Python313V4: pythonVersion: '3.13' + normalizedPythonVersion: '3.13' + Python314V4: + pythonVersion: '3.14.0-rc.2' + normalizedPythonVersion: '3.14' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_OSX_X64" + artifactName: "$(normalizedPythonVersion)_OSX_X64" steps: - - template: ../../../pack/templates/nix_env_gen.yml + - template: ../../../pack/templates/macos_64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - artifactName: '$(pythonVersion)_OSX_X64' + artifactName: '$(normalizedPythonVersion)_OSX_X64' grpcBuild: 'darwin.so' - job: Build_OSX_ARM64 + timeoutInMinutes: 180 pool: name: Azure Pipelines image: macOS-latest @@ -142,27 +187,36 @@ jobs: matrix: Python39V4: pythonVersion: '3.9' + normalizedPythonVersion: '3.9' Python310V4: pythonVersion: '3.10' + normalizedPythonVersion: '3.10' Python311V4: pythonVersion: '3.11' + normalizedPythonVersion: '3.11' Python312V4: pythonVersion: '3.12' + normalizedPythonVersion: '3.12' Python313V4: pythonVersion: '3.13' + normalizedPythonVersion: '3.13' + Python314V4: + pythonVersion: '3.14.0-rc.2' + normalizedPythonVersion: '3.14' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_OSX_ARM64" + artifactName: "$(normalizedPythonVersion)_OSX_ARM64" steps: - template: ../../../pack/templates/macos_64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - artifactName: '$(pythonVersion)_OSX_ARM64' + artifactName: '$(normalizedPythonVersion)_OSX_ARM64' grpcBuild: 'darwin.so' - job: Build_LINUX_ARM64 + timeoutInMinutes: 240 pool: name: 1es-pool-azfunc image: 1es-ubuntu-22.04 @@ -171,25 +225,30 @@ jobs: matrix: Python39V4: pythonVersion: '3.9' + normalizedPythonVersion: '3.9' Python310V4: pythonVersion: '3.10' + normalizedPythonVersion: '3.10' Python311V4: pythonVersion: '3.11' + normalizedPythonVersion: '3.11' Python312V4: pythonVersion: '3.12' + normalizedPythonVersion: '3.12' Python313V4: pythonVersion: '3.13' + normalizedPythonVersion: '3.13' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_LINUX_ARM64" + artifactName: "$(normalizedPythonVersion)_LINUX_ARM64" steps: - template: ../../../pack/templates/nix_arm64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - artifactName: '$(pythonVersion)_LINUX_ARM64' + artifactName: '$(normalizedPythonVersion)_LINUX_ARM64' grpcBuild: 'aarch64-linux-gnu.so' - job: PackageWorkers diff --git a/workers/pyproject.toml b/workers/pyproject.toml index 69497c60a..e561a0b77 100644 --- a/workers/pyproject.toml +++ b/workers/pyproject.toml @@ -35,10 +35,10 @@ dependencies = [ "protobuf~=5.29.0; python_version >= '3.13'", "grpcio-tools~=1.43.0; python_version == '3.7'", "grpcio-tools~=1.59.0; python_version >= '3.8' and python_version < '3.13'", - "grpcio-tools~=1.70.0; python_version >= '3.13'", + "grpcio-tools~=1.70.0; python_version == '3.13'", "grpcio~=1.43.0; python_version == '3.7'", "grpcio ~=1.59.0; python_version >= '3.8' and python_version < '3.13'", - "grpcio~=1.70.0; python_version >= '3.13'", + "grpcio~=1.70.0; python_version == '3.13'", "uvloop~=0.21.0; python_version >= '3.13' and sys_platform != 'win32'", "azurefunctions-extensions-base==1.1.0; python_version >= '3.8'", "azure-functions-runtime==1.0.0; python_version >= '3.13'", diff --git a/workers/python/prodV4/worker.config.json b/workers/python/prodV4/worker.config.json index eaa3c50cb..d638b9980 100644 --- a/workers/python/prodV4/worker.config.json +++ b/workers/python/prodV4/worker.config.json @@ -3,7 +3,7 @@ "language":"python", "defaultRuntimeVersion":"3.12", "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], - "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"], "supportedArchitectures":["X64", "X86", "Arm64"], "extensions":[".py"], "defaultExecutablePath":"python",