Merge pull request #25280 from dotnet/wtgodbe/MergePreview8

Merge preview8 branch to public

Author: dougbu

.azure/pipelines/ci.yml

@@ -622,17 +622,18 @@ stages:
         timeoutInMinutes: 240
         steps:
         # Build the shared framework
-        - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+        - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
           displayName: Build shared fx
-        - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -noBuildNative -projects src/Grpc/**/*.csproj
+        - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -noBuildNative -projects src/Grpc/**/*.csproj $(_InternalRuntimeDownloadArgs)
           displayName: Restore interop projects
         - script: ./build.cmd -ci -nobl -noBuildRepoTasks -noRestore -test -all -noBuildNative -projects eng\helix\helix.proj
-                  /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true
+                  /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true $(_InternalRuntimeDownloadArgs)
                   /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
           displayName: Run build.cmd helix target
           env:
             HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
             SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
+
         artifacts:
         - name: Helix_logs
           path: artifacts/log/
@@ -666,7 +667,7 @@ stages:
             arguments: $(Build.SourcesDirectory)/NuGet.config $Token
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
-    - script: ./eng/scripts/ci-source-build.sh --ci --nobl --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false
+    - script: ./eng/scripts/ci-source-build.sh --ci --nobl --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false $(_InternalRuntimeDownloadArgs)
       displayName: Run ci-source-build.sh
     - task: PublishBuildArtifacts@1
       displayName: Upload logs

eng/common/tools.ps1

@@ -240,8 +240,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
   return $installScript
 }
 
-function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') {
-  InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
+  InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
 }
 
 function InstallDotNet([string] $dotnetRoot,
@@ -250,7 +250,8 @@ function InstallDotNet([string] $dotnetRoot,
   [string] $runtime = '',
   [bool] $skipNonVersionedFiles = $false,
   [string] $runtimeSourceFeed = '',
-  [string] $runtimeSourceFeedKey = '') {
+  [string] $runtimeSourceFeedKey = '',
+  [switch] $noPath) {
 
   $installScript = GetDotNetInstallScript $dotnetRoot
   $installParameters = @{
@@ -261,6 +262,7 @@ function InstallDotNet([string] $dotnetRoot,
   if ($architecture) { $installParameters.Architecture = $architecture }
   if ($runtime) { $installParameters.Runtime = $runtime }
   if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+  if ($noPath) { $installParameters.NoPath = $True }
 
   try {
     & $installScript @installParameters

eng/helix/content/InstallDotNet.ps1

@@ -17,6 +17,8 @@ shift
 set $aspnetref=%9
 shift
 set $helixTimeout=%9
+shift
+set $feedCred=%9
 
 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
 set DOTNET_ROOT=%DOTNET_HOME%\%$arch%
@@ -26,10 +28,16 @@ set DOTNET_CLI_HOME=%HELIX_CORRELATION_PAYLOAD%\home
 
 set PATH=%DOTNET_ROOT%;!PATH!;%HELIX_CORRELATION_PAYLOAD%\node\bin
 echo Set path to: %PATH%
-echo "Invoking InstallDotNet.ps1 %$arch% %$sdkVersion% %$runtimeVersion% %DOTNET_ROOT%"
-powershell.exe -NoProfile -ExecutionPolicy unrestricted -file InstallDotNet.ps1 %$arch% %$sdkVersion% %$runtimeVersion% %DOTNET_ROOT%
+
+powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true '' '' $true"
+IF [%$feedCred%] == [] (
+    powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true '' '' $true"
+) else (
+    powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true https://dotnetclimsrc.blob.core.windows.net/dotnet %$feedCred% $true"
+)
 
 set exit_code=0
+
 echo "Restore: dotnet restore RunTests\RunTests.csproj --source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json --source https://api.nuget.org/v3/index.json --ignore-failed-sources..."
 dotnet restore RunTests\RunTests.csproj --source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json --source https://api.nuget.org/v3/index.json --ignore-failed-sources
 

eng/helix/content/runtests.cmd

@@ -30,49 +30,24 @@ RED="\033[0;31m"
 YELLOW="\033[0;33m"
 MAGENTA="\033[0;95m"
 
-curl -o dotnet-install.sh -sSL https://dot.net/v1/dotnet-install.sh
-if [ $? -ne 0 ]; then
-    download_retries=3
-    while [ $download_retries -gt 0 ]; do
-        curl -o dotnet-install.sh -sSL https://dot.net/v1/dotnet-install.sh
-        if [ $? -ne 0 ]; then
-            let download_retries=download_retries-1
-            echo -e "${YELLOW}Failed to download dotnet-install.sh. Retries left: $download_retries.${RESET}"
-        else
-            download_retries=0
-        fi
-    done
-fi
-
-# Call "sync" between "chmod" and execution to prevent "text file busy" error in Docker (aufs)
-chmod +x "dotnet-install.sh"; sync
-
-./dotnet-install.sh --version $dotnet_sdk_version --install-dir "$DOTNET_ROOT"
-if [ $? -ne 0 ]; then
-    sdk_retries=3
-    while [ $sdk_retries -gt 0 ]; do
-        ./dotnet-install.sh --version $dotnet_sdk_version --install-dir "$DOTNET_ROOT"
-        if [ $? -ne 0 ]; then
-            let sdk_retries=sdk_retries-1
-            echo -e "${YELLOW}Failed to install .NET Core SDK $version. Retries left: $sdk_retries.${RESET}"
-        else
-            sdk_retries=0
-        fi
-    done
-fi
-
-./dotnet-install.sh --runtime dotnet --version $dotnet_runtime_version --install-dir "$DOTNET_ROOT"
-if [ $? -ne 0 ]; then
-    runtime_retries=3
-    while [ $runtime_retries -gt 0 ]; do
-        ./dotnet-install.sh --runtime dotnet --version $dotnet_runtime_version --install-dir "$DOTNET_ROOT"
-        if [ $? -ne 0 ]; then
-            let runtime_retries=runtime_retries-1
-            echo -e "${YELLOW}Failed to install .NET Core runtime $version. Retries left: $runtime_retries.${RESET}"
-        else
-            runtime_retries=0
-        fi
-    done
+. eng/common/tools.sh
+InstallDotNet $DOTNET_ROOT $dotnet_sdk_version "" "" true || {
+  exit_code=$?
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+  ExitWithExitCode $exit_code
+}
+if [[ -z "${11:-}" ]]; then
+    InstallDotNet $DOTNET_ROOT $dotnet_runtime_version "" dotnet true || {
+      exit_code=$?
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+      ExitWithExitCode $exit_code
+    }
+else
+    InstallDotNet $DOTNET_ROOT $dotnet_runtime_version "" dotnet true https://dotnetclimsrc.blob.core.windows.net/dotnet ${11} || {
+      exit_code=$?
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+      ExitWithExitCode $exit_code
+    }
 fi
 
 if [ -e /proc/self/coredump_filter ]; then
@@ -82,7 +57,7 @@ if [ -e /proc/self/coredump_filter ]; then
   echo -n 0x3F > /proc/self/coredump_filter
 fi
 
-# dontet-install.sh seems to affect the Linux filesystem and causes test flakiness unless we sync the filesystem before running tests
+# dotnet-install.sh seems to affect the Linux filesystem and causes test flakiness unless we sync the filesystem before running tests
 sync
 
 exit_code=0

eng/helix/content/runtests.sh

@@ -33,10 +33,76 @@ reporoot="$(dirname "$(dirname "$scriptroot")")"
 #    mv "$reporoot/global.bak.json" "$reporoot/global.json"
 #}" EXIT
 
+dotnet_runtime_source_feed=''
+dotnet_runtime_source_feed_key=''
+other_args=()
+
+#
+# Functions
+#
+__usage() {
+    echo "Usage: $(basename "${BASH_SOURCE[0]}") [options] [[--] <Arguments>...]
+
+Arguments:
+    <Arguments>...                    Arguments passed to the command. Variable number of arguments allowed.
+
+    --dotnet-runtime-source-feed      Additional feed that can be used when downloading .NET runtimes
+    --dotnet-runtime-source-feed-key  Key for feed that can be used when downloading .NET runtimes
+
+Description:
+   This script is meant for testing source build by imitating some of the input parameters and conditions.
+"
+
+    if [[ "${1:-}" != '--no-exit' ]]; then
+        exit 2
+    fi
+}
+
+__error() {
+    echo -e "${RED}error: $*${RESET}" 1>&2
+}
+
+#
+# main
+#
+
+while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    case "$opt" in
+        -\?|-h|-help)
+            __usage --no-exit
+            exit 0
+            ;;
+        -dotnet-runtime-source-feed|-dotnetruntimesourcefeed)
+            shift
+            [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed" && __usage
+            dotnet_runtime_source_feed="${1:-}"
+            ;;
+        -dotnet-runtime-source-feed-key|-dotnetruntimesourcefeedkey)
+            shift
+            [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed-key" && __usage
+            dotnet_runtime_source_feed_key="${1:-}"
+            ;;
+        *)
+            other_args[${#other_args[*]}]="$1"
+            ;;
+    esac
+    shift
+done
+
+# Set up additional runtime args
+runtime_feed_args=()
+if [ ! -z "$dotnet_runtime_source_feed$dotnet_runtime_source_feed_key" ]; then
+    runtimeFeedArg="/p:DotNetRuntimeSourceFeed=$dotnet_runtime_source_feed"
+    runtimeFeedKeyArg="/p:DotNetRuntimeSourceFeedKey=$dotnet_runtime_source_feed_key"
+    runtime_feed_args[${#runtime_feed_args[*]}]=$runtimeFeedArg
+    runtime_feed_args[${#runtime_feed_args[*]}]=$runtimeFeedKeyArg
+fi
+
 # Build repo tasks
-"$reporoot/eng/common/build.sh" --restore --build --ci --configuration Release /p:ProjectToBuild=$reporoot/eng/tools/RepoTasks/RepoTasks.csproj
+"$reporoot/eng/common/build.sh" --restore --build --ci --configuration Release /p:ProjectToBuild=$reporoot/eng/tools/RepoTasks/RepoTasks.csproj ${runtime_feed_args[@]+"${runtime_feed_args[@]}"}
 
 export DotNetBuildFromSource='true'
 
 # Build projects
-"$reporoot/eng/common/build.sh" --restore --build --pack "$@"
+"$reporoot/eng/common/build.sh" --restore --build --pack ${other_args[@]+"${other_args[@]}"} ${runtime_feed_args[@]+"${runtime_feed_args[@]}"}

eng/scripts/ci-source-build.sh

@@ -24,6 +24,20 @@
   <ItemGroup Condition="$(BuildHelixPayload)">
     <Content Include="@(HelixContent)" />
     <Content Include="$(RepoRoot)eng\scripts\Download.ps1" />
+    <Content Include="$(RepoRoot)NuGet.config" />
+    <Content Include="$(RepoRoot)global.json" />
+    <ContentWithTargetPath Include="
+        $(RepoRoot)eng\common\pipeline-logging-functions.*;
+        $(RepoRoot)eng\common\tools.*">
+      <TargetPath>eng\common\%(Filename)%(Extension)</TargetPath>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
+    </ContentWithTargetPath>
+    <ContentWithTargetPath Include="$(RepoRoot)eng\common\dotnet-install-scripts\*">
+      <TargetPath>eng\common\dotnet-install-scripts\%(Filename)%(Extension)</TargetPath>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
+    </ContentWithTargetPath>
   </ItemGroup>
 
 <!--