Merge branch 'main' into davidfowl/remove-logging-scopes

Author: davidfowl

.azure/pipelines/azure-pipelines-mirror-within-azdo.yml

@@ -13,7 +13,8 @@ parameters:
   type: boolean
 
 variables:
-  - group: Mirror-Credentials
+- group: Mirror-Credentials
+- template: /eng/common/templates/variables/pool-providers.yml
 
 # Merges code from one AzDO branch into another
 jobs:
@@ -26,7 +27,7 @@ jobs:
       - job: Merge_Azure_DevOps_Branches
         enableSBOM: false
         pool:
-          name: NetCore1ESPool-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals 1es-windows-2019
         variables:
         - name: WorkingDirectoryName

.azure/pipelines/benchmarks.yml

@@ -6,6 +6,9 @@ trigger: none
 pr:
 - '*'
 
+variables:
+- template: /eng/common/templates/variables/pool-providers.yml
+
 jobs:
 - template: jobs/default-build.yml
   parameters:

.azure/pipelines/blazor-daily-tests.yml

@@ -1,5 +1,5 @@
 # Uses Scheduled Triggers, which aren't supported in YAML yet.
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=vsts&tabs=yaml#scheduled
+# https://learn.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=vsts&tabs=yaml#scheduled
 
 # Daily Tests for Blazor
 # These use Sauce Labs resources, hence they run daily rather than per-commit.
@@ -17,6 +17,7 @@ variables:
     value: 'blazor-e2e-sc-proxy-tunnel'
   - name: E2ETESTS_Sauce__HostName
     value: 'sauce.local'
+  - template: /eng/common/templates/variables/pool-providers.yml
 
 jobs:
 - template: jobs/default-build.yml

.azure/pipelines/ci.yml

@@ -1,5 +1,5 @@
 #
-# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details on this file.
+# See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for details on this file.
 #
 
 # Configure which branches trigger builds
@@ -12,6 +12,9 @@ trigger:
     - internal/release/*
 
 # Run PR validation on all branches
+# This doesn't have any path exclusions, even for things like docs, because
+# we have it configured in GitHub as a required check, and for it to pass
+# it must actually run, even if it's not relevant to a particular change.
 pr:
   autoCancel: true
   branches:
@@ -49,12 +52,6 @@ parameters:
   default: false
   displayName: Produce binlogs?
   type: boolean
-# Choose whether to test source indexing.
-# Will cause inaccessible links on https://source.dot.net/ unless commits are also available in GitHub.
-- name: testSourceIndexing
-  default: false
-  displayName: Test source indexing? !Danger! see comments in YAML.
-  type: boolean
 
 variables:
 - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
@@ -71,6 +68,9 @@ variables:
     value: true
 - name: _UseHelixOpenQueues
   value: ${{ ne(variables['System.TeamProject'], 'internal') }}
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}:
+  - name: enableSourceIndex
+    value: true
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - name: _BuildArgs
     value: /p:TeamName=$(_TeamName)
@@ -85,12 +85,6 @@ variables:
     value: /p:Publish=true
            /p:GenerateChecksums=true
            /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
-  # Variables for source indexing afterBuild step and job.
-  - name: sourceIndexPackageVersion
-    value: 1.0.1-20210614.1
-  - name: sourceIndexPackageSource
-    value: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
-  - group: source-dot-net stage1 variables
   - ${{ if ne(parameters.produceBinlogs, 'true') }}:
     # Do not log most Windows steps in official builds; this is the slowest job. Site extensions step always logs.
     - name: WindowsArm64LogArgs
@@ -152,6 +146,7 @@ variables:
       value: test
 - name: runCodeQL3000
   value: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }}
+- template: /eng/common/templates/variables/pool-providers.yml
 
 stages:
 - stage: build
@@ -352,59 +347,6 @@ stages:
         - name: Windows_ANCMIISExpress_Msi
           path: artifacts/bin/AncmIISExpressV2
 
-    # Build Windows ARM
-    - template: jobs/default-build.yml
-      parameters:
-        codeSign: true
-        jobName: Windows_arm_build
-        jobDisplayName: "Build: Windows ARM"
-        agentOs: Windows
-        buildArgs:
-          -arch arm
-          -sign
-          -pack
-          -noBuildNodeJS
-          -noBuildJava
-          -binaryLog
-          /p:DotNetSignType=$(_SignType)
-          /p:OnlyPackPlatformSpecificPackages=true
-          /p:AssetManifestFileName=aspnetcore-win-arm.xml
-          $(_BuildArgs)
-          $(_PublishArgs)
-          $(_InternalRuntimeDownloadArgs)
-        ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
-          afterBuild:
-          - task: UseDotNet@2
-            displayName: Use .NET Core sdk 3.1
-            inputs:
-              packageType: sdk
-              version: 3.1.x
-              installationPath: $(Build.SourcesDirectory)/.dotnet
-              workingDirectory: $(Build.SourcesDirectory)
-          - powershell: . $(Build.SourcesDirectory)/activate.ps1;
-              dotnet tool install BinLogToSln
-              --version $(SourceIndexPackageVersion)
-              --add-source $(SourceIndexPackageSource)
-              --tool-path $(Build.SourcesDirectory)/.tools;
-              $(Build.SourcesDirectory)/.tools/BinLogToSln
-              -i $(Build.SourcesDirectory)/artifacts/log/Release/Build.binlog
-              -r $(Build.SourcesDirectory)
-              -n $(Build.Repository.Name)
-              -o $(Build.ArtifactStagingDirectory)/sourceIndex/
-            displayName: Process binary log into indexable sln
-        installNodeJs: false
-        installJdk: false
-        artifacts:
-        - name: Windows_arm_Logs
-          path: artifacts/log/
-          publishOnError: true
-          includeForks: true
-        - name: Windows_arm_Packages
-          path: artifacts/packages/
-        - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
-            - name: Windows_arm_SourceIndex
-              path: $(Build.ArtifactStagingDirectory)/sourceIndex/
-
     # Build MacOS arm64
     - template: jobs/default-build.yml
       parameters:
@@ -486,13 +428,13 @@ stages:
               $(_InternalRuntimeDownloadArgs)
           displayName: Run build.sh
         - script: git clean -xfd src/**/obj/;
-            ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs --init-nuget
+            ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs
             -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb
             $(_BuildArgs)
             $(_InternalRuntimeDownloadArgs)
           displayName: Build Debian installers
         - script: git clean -xfd src/**/obj/;
-            ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs --init-nuget
+            ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs
             -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm
             -p:AssetManifestFileName=aspnetcore-Linux_x64.xml
             $(_BuildArgs)
@@ -563,7 +505,7 @@ stages:
               $(_InternalRuntimeDownloadArgs)
           displayName: Run build.sh
         - script: git clean -xfd src/**/obj/;
-            ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs  --init-nuget
+            ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs
             -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm
             -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml
             $(_BuildArgs)
@@ -697,7 +639,7 @@ stages:
           # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job.
           cancelTimeoutInMinutes: 30
           buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true
-                     /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false
+                     /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true
                      $(_InternalRuntimeDownloadArgs)
           beforeBuild:
           - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
@@ -797,13 +739,26 @@ stages:
               # Log environment variables in binary logs to ease debugging
               MSBUILDLOGALLENVIRONMENTVARIABLES: true
 
+    - ${{ if eq(variables.enableSourceIndex, 'true') }}:
+      - template: /eng/common/templates/job/source-index-stage1.yml
+        parameters:
+          sourceIndexBuildCommand: ./eng/build.cmd -Configuration Release -ci -noBuildJava -binaryLog /p:OnlyPackPlatformSpecificPackages=true
+          binlogPath: artifacts/log/Release/Build.binlog
+          presteps:
+          - task: NodeTool@0
+            displayName: Install Node 18.x
+            inputs:
+              versionSpec: 18.x
+          pool:
+            name: $(DncEngInternalBuildPool)
+            demands: ImageOverride -equals 1es-windows-2022
+
     # Publish to the BAR and perform source indexing. Wait until everything else is done.
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - template: /eng/common/templates/job/publish-build-assets.yml
         parameters:
           dependsOn:
             - Windows_build
-            - Windows_arm_build
             - ${{ if ne(variables.PostBuildSign, 'true') }}:
               - CodeSign_Xplat_MacOS_arm64
               - CodeSign_Xplat_MacOS_x64
@@ -830,74 +785,15 @@ stages:
                 - MacOS_Test
                 - Linux_Test
                 - Helix_x64
+            - ${{ if eq(variables.enableSourceIndex, 'true') }}:
+              - SourceIndexStage1
             - Source_Build_Managed
           pool:
-            name: NetCore1ESPool-Internal
+            name: $(DncEngInternalBuildPool)
             demands: ImageOverride -equals 1es-windows-2019
           publishUsingPipelines: ${{ variables._PublishUsingPipelines }}
           enablePublishBuildArtifacts: true # publish artifacts/log files
           publishAssetsImmediately: true # Don't use a separate stage for darc publishing.
-      - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
-        - job: SourceIndexUpload
-          displayName: Upload indexable solution
-          dependsOn:
-            - Windows_arm_build
-            # In addition to the dependency above that provides assets, ensure the build was successful overall.
-            - Windows_build
-            - ${{ if ne(variables.PostBuildSign, 'true') }}:
-              - CodeSign_Xplat_MacOS_arm64
-              - CodeSign_Xplat_MacOS_x64
-              - CodeSign_Xplat_Linux_x64
-              - CodeSign_Xplat_Linux_arm
-              - CodeSign_Xplat_Linux_arm64
-              - CodeSign_Xplat_Linux_musl_x64
-              - CodeSign_Xplat_Linux_musl_arm
-              - CodeSign_Xplat_Linux_musl_arm64
-            - ${{ if eq(variables.PostBuildSign, 'true') }}:
-              - MacOs_arm64_build
-              - MacOs_x64_build
-              - Linux_x64_build
-              - Linux_arm_build
-              - Linux_arm64_build
-              - Linux_musl_x64_build
-              - Linux_musl_arm_build
-              - Linux_musl_arm64_build
-            - ${{ if in(variables['Build.Reason'], 'Manual') }}:
-              - Code_check
-              - ${{ if ne(parameters.skipTests, 'true') }}:
-                - Windows_Test
-                - MacOS_Test
-                - Linux_Test
-                - Helix_x64
-            - Source_Build_Managed
-          pool:
-            name: NetCore1ESPool-Internal
-            # Visual Studio Enterprise - no BuildTools agents exist internally and job must run on Windows
-            demands: ImageOverride -equals 1es-windows-2019
-          steps:
-            - task: DownloadBuildArtifacts@0
-              inputs:
-                artifactName: Windows_arm_SourceIndex
-                cleanDestinationFolder: true
-            # Ignore repository's global.json and any existing .NET SDK.
-            - task: UseDotNet@2
-              displayName: Use .NET Core sdk 3.1
-              inputs:
-                packageType: sdk
-                version: 3.1.x
-                installationPath: $(Agent.TempDirectory)/.dotnet
-                workingDirectory: $(Agent.TempDirectory)
-            - script: $(Agent.TempDirectory)/.dotnet/dotnet tool install UploadIndexStage1
-                --version $(SourceIndexPackageVersion)
-                --add-source $(SourceIndexPackageSource)
-                --tool-path $(Agent.TempDirectory)/.tools &&
-                $(Agent.TempDirectory)/.tools/UploadIndexStage1
-                -i $(System.ArtifactsDirectory)/Windows_arm_SourceIndex/
-                -n $(Build.Repository.Name)
-              displayName: Upload indexable solution
-              workingDirectory: $(Agent.TempDirectory)
-              env:
-                BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}:
   - template: /eng/common/templates/post-build/post-build.yml

.azure/pipelines/components-e2e-tests.yml

@@ -1,5 +1,5 @@
 #
-# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details on this file.
+# See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for details on this file.
 #
 
 # Configure which branches trigger builds
@@ -25,6 +25,7 @@ variables:
   value: true
 - name: _TeamName
   value:  AspNetCore
+- template: /eng/common/templates/variables/pool-providers.yml
 
 jobs:
 - template: jobs/default-build.yml
@@ -34,13 +35,13 @@ jobs:
     agentOs: Linux
     isAzDOTestingJob: true
     enablePublishTestResults: false
-    timeoutInMinutes: 75
+    timeoutInMinutes: 120
     steps:
     - script: git submodule update --init
       displayName: Update submodules
     - script: ./restore.sh
       displayName: Run restore.sh
-    - script: yarn install --frozen-lockfile --cwd ./src/Components/test/E2ETest
+    - script: yarn install --frozen-lockfile --cwd ./src/Components/test/E2ETest || yarn install --frozen-lockfile --cwd ./src/Components/test/E2ETest
       displayName: NPM install
     - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore
       displayName: Build
@@ -74,6 +75,10 @@ jobs:
       condition: always()
 
     artifacts:
+    - name: Components_E2E_Logs
+      path: artifacts/log/
+      publishOnError: true
+      includeForks: true
     - name: Components_E2E_Test_Logs
       path: '$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)'
       includeForks: true

.azure/pipelines/devBuilds.yml

@@ -1,5 +1,5 @@
 #
-# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details on this file.
+# See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for details on this file.
 #
 
 # Configure which branches trigger builds
@@ -8,6 +8,9 @@ trigger: none
 # no PR builds
 pr: none
 
+variables:
+- template: /eng/common/templates/variables/pool-providers.yml
+
 stages:
 - stage: build_components
   displayName: Build Components

.azure/pipelines/helix-matrix.yml

@@ -20,6 +20,7 @@ variables:
   value: ${{ ne(variables['System.TeamProject'], 'internal') }}
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNet-HelixApi-Access
+- template: /eng/common/templates/variables/pool-providers.yml
 
 jobs:
 - template: jobs/default-build.yml