Merge remote-tracking branch 'upstream/main' into nested-async-comp

Author: pablogsal

.azure-pipelines/ci.yml

@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+trigger: ['main', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./docs-steps.yml
@@ -56,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_CI_Tests
-  displayName: ManyLinux1 CI Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(build.sourceBranchName)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_CI_Tests
   displayName: Ubuntu CI Tests (coverage)
   dependsOn: Prebuild
@@ -113,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
@@ -139,7 +104,7 @@ jobs:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(Build.SourceBranchName)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/docs-steps.yml

@@ -12,11 +12,12 @@ steps:
   inputs:
     versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme
+- script: python -m pip install -r requirements.txt
+  workingDirectory: '$(build.sourcesDirectory)/Doc'
   displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:
-  - script: make check suspicious html PYTHON=python
+  - script: make check html PYTHON=python
     workingDirectory: '$(build.sourcesDirectory)/Doc'
     displayName: 'Build documentation'
 
@@ -31,7 +32,7 @@ steps:
 - ${{ if eq(parameters.upload, 'true') }}:
   - task: PublishBuildArtifacts@1
     displayName: 'Publish docs'
-  
+
     inputs:
       PathToPublish: '$(build.sourcesDirectory)/Doc/build'
       ArtifactName: docs

.azure-pipelines/pr.yml

@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+pr: ['main', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./docs-steps.yml
@@ -56,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_PR_Tests
-  displayName: ManyLinux1 PR Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_PR_Tests
   displayName: Ubuntu PR Tests (coverage)
   dependsOn: Prebuild
@@ -113,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
@@ -139,7 +104,7 @@ jobs:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/windows-release/msi-steps.yml

@@ -1,6 +1,12 @@
 steps:
   - template: ./checkout.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: doc'
     inputs:

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -105,19 +105,26 @@ jobs:
     clean: all
 
   steps:
-  - checkout: none
+  - template: ./checkout.yml
   - template: ./find-sdk.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - task: DownloadBuildArtifacts@0
     displayName: 'Download Artifact: unsigned_msix'
     inputs:
       artifactName: unsigned_msix
       downloadPath: $(Build.BinariesDirectory)
 
+  # MSIX must be signed and timestamped simultaneously
   - powershell: |
       $failed = $true
       foreach ($retry in 1..3) {
-          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix)
+          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix)
           if ($?) {
               $failed = $false
               break

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmImage: windows-2019
+    name: 'Windows Release'
 
   workspace:
     clean: all
@@ -36,6 +36,14 @@ jobs:
       nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive
     displayName: 'Create nuget package'
 
+  - powershell: |
+      gci *.nupkg | %{
+        nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite
+      }
+    displayName: 'Sign nuget package'
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), variables['SigningCertificate'])
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: nuget'
     inputs:

.azure-pipelines/windows-release/stage-sign.yml

@@ -26,6 +26,12 @@ jobs:
   - template: ./checkout.yml
   - template: ./find-sdk.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - powershell: |
       Write-Host "##vso[build.addbuildtag]signed"
     displayName: 'Add build tags'
@@ -51,7 +57,7 @@ jobs:
       $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }})
       $failed = $true
       foreach ($retry in 1..10) {
-          signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files
+          signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files
           if ($?) {
               $failed = $false
               break

.gitattributes

@@ -47,7 +47,7 @@ PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
 Python/importlib.h          linguist-generated=true
 Python/importlib_external.h linguist-generated=true
-Include/Python-ast.h        linguist-generated=true
+Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true
 Python/opcode_targets.h     linguist-generated=true
@@ -57,3 +57,14 @@ Doc/library/token-list.inc  linguist-generated=true
 Include/token.h             linguist-generated=true
 Lib/token.py                linguist-generated=true
 Parser/token.c              linguist-generated=true
+Programs/test_frozenmain.h  linguist-generated=true
+
+# Language aware diff headers
+# https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more
+# https://gist.github.com/tekin/12500956bd56784728e490d8cef9cb81
+*.c     diff=cpp
+*.h     diff=cpp
+*.css   diff=css
+*.html  diff=html
+*.py    diff=python
+*.md    diff=markdown