Merge branch 'master' into int-implicit-lossless

Author: serhiy-storchaka

.azure-pipelines/ci.yml

@@ -35,14 +35,16 @@ jobs:
 - job: macOS_CI_Tests
   displayName: macOS CI Tests
   dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  # bpo-39837: macOS tests on Azure Pipelines are disabled
+  condition: false
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-macos'
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -59,7 +61,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +118,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -131,7 +133,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
@@ -145,7 +147,7 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(Build.SourceBranchName)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml

.azure-pipelines/docs-steps.yml

@@ -12,7 +12,7 @@ steps:
   inputs:
     versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==2.0.1 blurb python-docs-theme
+- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme
   displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:
@@ -21,7 +21,7 @@ steps:
     displayName: 'Build documentation'
 
 - ${{ if eq(parameters.latex, 'true') }}:
-  - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full 
+  - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
     displayName: 'Install LaTeX'
 
   - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'

.azure-pipelines/macos-steps.yml

@@ -6,14 +6,16 @@ steps:
 - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
   displayName: 'Configure CPython (debug)'
 
-- script: make -s -j4
+- script: make -j4
   displayName: 'Build CPython'
 
 - script: make pythoninfo
   displayName: 'Display build info'
 
 - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
+  continueOnError: true
+  timeoutInMinutes: 30
 
 - task: PublishTestResults@2
   displayName: 'Publish Test Results'

.azure-pipelines/posix-steps.yml

@@ -20,7 +20,7 @@ steps:
 - script: ./configure --with-pydebug
   displayName: 'Configure CPython (debug)'
 
-- script: make -s -j4
+- script: make -j4
   displayName: 'Build CPython'
 
 - ${{ if eq(parameters.coverage, 'true') }}:
@@ -49,7 +49,7 @@ steps:
   - script: ./venv/bin/python -m coverage xml
     displayName: 'Generate coverage.xml'
 
-  - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+  - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml
     displayName: 'Publish code coverage results'
 
 

.azure-pipelines/pr.yml

@@ -33,14 +33,16 @@ jobs:
 - job: macOS_PR_Tests
   displayName: macOS PR Tests
   dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  # bpo-39837: macOS tests on Azure Pipelines are disabled
+  condition: false
 
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-macos'
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -59,7 +61,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +118,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -131,7 +133,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
@@ -145,7 +147,10 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+      winarm64:
+        arch: arm64
+        buildOpt: '-p arm64'
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml

.azure-pipelines/windows-release/build-steps.yml

@@ -43,7 +43,7 @@ steps:
 
 - powershell: |
     $env:SigningCertificate = $null
-    .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default
+    $(_HostPython) PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch)
     makecat "${env:CAT}.cdf"
     del "${env:CAT}.cdf"
     if (-not (Test-Path "${env:CAT}.cat")) {
@@ -52,6 +52,7 @@ steps:
   displayName: 'Generate catalog'
   env:
     CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+    PYTHON_HEXVERSION: $(VersionHex)
 
 - task: PublishPipelineArtifact@0
   displayName: 'Publish binaries'

.azure-pipelines/windows-release/layout-command.yml

@@ -1,12 +1,20 @@
 steps:
+- task: DownloadPipelineArtifact@1
+  displayName: 'Download artifact: bin_$(HostArch)'
+  condition: and(succeeded(), variables['HostArch'])
+  inputs:
+    artifactName: bin_$(HostArch)
+    targetPath: $(Build.BinariesDirectory)\bin_$(HostArch)
+
 - powershell: >
     Write-Host (
     '##vso[task.setvariable variable=LayoutCmd]&
-    "{0}\bin\python.exe"
+    "$(Python)"
     "{1}\PC\layout"
     -vv
     --source "{1}"
     --build "{0}\bin"
+    --arch "$(Name)"
     --temp "{0}\layout-temp"
     --include-cat "{0}\bin\python.cat"
     --doc-build "{0}\doc"'

.azure-pipelines/windows-release/msi-steps.yml

@@ -54,6 +54,7 @@ steps:
   - powershell: |
       copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - script: |
       call Tools\msi\get_externals.bat

.azure-pipelines/windows-release/stage-build.yml

@@ -3,7 +3,7 @@ jobs:
   displayName: Docs build
   pool:
     name: 'Windows Release'
-    #vmName: win2016-vs2017
+    #vmImage: windows-2019
 
   workspace:
     clean: all
@@ -16,14 +16,16 @@ jobs:
     env:
       BUILDDIR: $(Build.BinariesDirectory)\Doc
 
-  #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm"
-  #  displayName: 'Cheat at building CHM docs'
-
   - script: Doc\make.bat htmlhelp
     displayName: 'Build CHM docs'
     env:
       BUILDDIR: $(Build.BinariesDirectory)\Doc
 
+  #- powershell: |
+  #    mkdir -Force "$(Build.BinariesDirectory)\Doc\htmlhelp"
+  #    iwr "https://www.python.org/ftp/python/3.8.0/python380.chm" -OutFile "$(Build.BinariesDirectory)\Doc\htmlhelp\python390a0.chm"
+  #  displayName: 'Cheat at building CHM docs'
+
   - task: CopyFiles@2
     displayName: 'Assemble artifact: Doc'
     inputs:
@@ -43,7 +45,7 @@ jobs:
   displayName: Python build
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all
@@ -55,16 +57,31 @@ jobs:
         Arch: win32
         Platform: x86
         Configuration: Release
+        _HostPython: .\python
       win32_d:
         Name: win32_d
         Arch: win32
         Platform: x86
         Configuration: Debug
+        _HostPython: .\python
       amd64_d:
         Name: amd64_d
         Arch: amd64
         Platform: x64
         Configuration: Debug
+        _HostPython: .\python
+      arm64:
+        Name: arm64
+        Arch: arm64
+        Platform: ARM64
+        Configuration: Release
+        _HostPython: python
+      arm64_d:
+        Name: arm64_d
+        Arch: arm64
+        Platform: ARM64
+        Configuration: Debug
+        _HostPython: python
 
   steps:
     - template: ./build-steps.yml
@@ -74,7 +91,7 @@ jobs:
   condition: and(succeeded(), ne(variables['DoPGO'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all
@@ -86,6 +103,7 @@ jobs:
         Arch: amd64
         Platform: x64
         Configuration: Release
+        _HostPython: .\python
 
   steps:
     - template: ./build-steps.yml
@@ -111,6 +129,7 @@ jobs:
         Arch: amd64
         Platform: x64
         Configuration: Release
+        _HostPython: .\python
 
   steps:
     - template: ./build-steps.yml
@@ -122,7 +141,7 @@ jobs:
   displayName: Publish Tcl/Tk Library
 
   pool:
-    vmName: windows-latest
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-layout-embed.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoEmbed'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all
@@ -19,6 +19,11 @@ jobs:
         Name: amd64
         Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
   - template: ./checkout.yml

.azure-pipelines/windows-release/stage-layout-full.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoLayout'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all
@@ -13,11 +13,18 @@ jobs:
     matrix:
       win32:
         Name: win32
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -43,21 +50,23 @@ jobs:
 
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: tcltk_lib_$(Name)'
+    condition: and(succeeded(), variables['TclLibrary'])
     inputs:
       artifactName: tcltk_lib_$(Name)
       targetPath: $(Build.BinariesDirectory)\tcltk_lib
 
   - powershell: |
-      copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
+      copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
   - powershell: |
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default
     displayName: 'Generate full layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_full_$(Name)'