Update helix installation retry logic (#30022)

* Update helix installation retry logic

Restart SDK and runtime installation upon retry

Author: John Luo

.azure/pipelines/helix-matrix.yml

@@ -1,5 +1,9 @@
 # We only want to run full helix matrix on main
-pr: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - '*'
 trigger: none
 schedules:
 # Cron timezone is UTC.

eng/helix/content/runtests.ps1

@@ -11,53 +11,72 @@ param(
     [string]$FeedCred
 )
 
-$currentDirectory = Get-Location
-$random = Get-Random -Maximum 1024
-
-$env:DOTNET_HOME = Join-Path $currentDirectory "sdk$random"
-$env:DOTNET_ROOT = Join-Path $env:DOTNET_HOME $Arch
 $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1
 $env:DOTNET_MULTILEVEL_LOOKUP = 0
-$env:DOTNET_CLI_HOME = Join-Path $currentDirectory "home$random"
-$env:PATH = "$env:DOTNET_ROOT;$env:PATH;$env:HELIX_CORRELATION_PAYLOAD\node\bin"
 
-Write-Host "Set PATH to: $env:PATH"
+$currentDirectory = Get-Location
+$envPath = "$env:PATH;$env:HELIX_CORRELATION_PAYLOAD\node\bin"
 
 function InvokeInstallDotnet([string]$command) {
+    Write-Host "InstallDotNet $command"
+
+    $timeoutSeconds = 180
+    $proc = Start-Process -filePath powershell.exe -ArgumentList "-noLogo -NoProfile -ExecutionPolicy unrestricted -command `"$command`"" -NoNewWindow -PassThru
+
+    $proc | Wait-Process -Timeout $timeoutSeconds
+    $exitCode = $proc.ExitCode
+
+    if ($exitCode -eq 0) {
+        Write-Host "InstallDotNet $command completed successfully"
+        return $true
+    }
+    elseif ([string]::IsNullOrWhiteSpace($exitCode)) {
+        Write-Warning "InstallDotNet $command timed out after $timeoutSeconds seconds"
+    }
+    else {
+        Write-Warning "InstallDotNet $command failed with exit code $exitCode"
+    }
+
+    $proc | Stop-Process -Force
+
+    return $false
+}
+
+function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCredParam) {
     foreach ($i in 1..5) {
-        Write-Host "InstallDotNet $command"
+        $random = Get-Random -Maximum 1024
+        $env:DOTNET_HOME = Join-Path $currentDirectory "sdk$random"
+        $env:DOTNET_ROOT = Join-Path $env:DOTNET_HOME $Arch
+        $env:DOTNET_CLI_HOME = Join-Path $currentDirectory "home$random"
+        $env:PATH = "$env:DOTNET_ROOT;$envPath"
 
-        $timeoutSeconds = 120
-        $proc = Start-Process -filePath powershell.exe -ArgumentList "-noLogo -NoProfile -ExecutionPolicy unrestricted -command `"$command`"" -NoNewWindow -PassThru
+        Write-Host "Set PATH to: $env:PATH"
 
-        $proc | Wait-Process -Timeout $timeoutSeconds
-        $exitCode = $proc.ExitCode
+        $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true `'$Feed`' `'$FeedCredParam`' `$true"
 
-        if ($exitCode -eq 0) {
-            Write-Host "InstallDotNet $command completed successfully"
-            return
+        if (!$success) {
+            Write-Host "Retrying..."
+            continue
         }
 
-        if ([string]::IsNullOrWhiteSpace($exitCode)) {
-            Write-Warning "InstallDotNet $command timed out after $timeoutSeconds seconds retrying..."
-        }
-        else {
-            Write-Warning "InstallDotNet $command failed with exit code $exitCode retrying..."
+        $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $RuntimeVersion $Arch dotnet `$true `'$Feed`' `'$FeedCredParam`' `$true"
+
+        if (!$success) {
+            Write-Host "Retrying..."
+            continue
         }
 
-        $proc | Stop-Process -Force
+        return
     }
 
-    Write-Warning "InstallDotNet $command exceeded retry limit"
+    Write-Error "InstallDotNet $command exceeded retry limit"
     exit 1
 }
 
-if ($FeedCred -eq $null) {
-    InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true `'`' `'`' `$true")
-    InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $RuntimeVersion $Arch dotnet `$true `'`' `'`' `$true")
+if ([string]::IsNullOrEmpty($FeedCred)) {
+    InstallDotnetSDKAndRuntime
 } else {
-    InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true https://dotnetclimsrc.blob.core.windows.net/dotnet $FeedCred `$true")
-    InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $RuntimeVersion $Arch dotnet `$true https://dotnetclimsrc.blob.core.windows.net/dotnet $FeedCred `$true")
+    InstallDotnetSDKAndRuntime "https://dotnetclimsrc.blob.core.windows.net/dotnet" $FeedCred
 }
 
 Write-Host "Restore: dotnet restore RunTests\RunTests.csproj --ignore-failed-sources"