From 655a1110d93829f54b1115cda69bfaacae737b31 Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 8 Feb 2021 18:49:08 -0800 Subject: [PATCH 1/3] Update helix installation retry logic Restart SDK and runtime installation upon retry --- .azure/pipelines/helix-matrix.yml | 6 ++- eng/helix/content/runtests.ps1 | 73 +++++++++++++++++++------------ 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/.azure/pipelines/helix-matrix.yml b/.azure/pipelines/helix-matrix.yml index 558afc7bbaa2..520fb47cdf77 100644 --- a/.azure/pipelines/helix-matrix.yml +++ b/.azure/pipelines/helix-matrix.yml @@ -1,5 +1,9 @@ # We only want to run full helix matrix on main -pr: none +pr: + autoCancel: true + branches: + include: + - '*' trigger: none schedules: # Cron timezone is UTC. diff --git a/eng/helix/content/runtests.ps1 b/eng/helix/content/runtests.ps1 index d0a9c898a34f..fb255bbd8a3a 100644 --- a/eng/helix/content/runtests.ps1 +++ b/eng/helix/content/runtests.ps1 @@ -11,53 +11,72 @@ param( [string]$FeedCred ) -$currentDirectory = Get-Location -$random = Get-Random -Maximum 1024 - -$env:DOTNET_HOME = Join-Path $currentDirectory "sdk$random" -$env:DOTNET_ROOT = Join-Path $env:DOTNET_HOME $Arch $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 $env:DOTNET_MULTILEVEL_LOOKUP = 0 -$env:DOTNET_CLI_HOME = Join-Path $currentDirectory "home$random" -$env:PATH = "$env:DOTNET_ROOT;$env:PATH;$env:HELIX_CORRELATION_PAYLOAD\node\bin" -Write-Host "Set PATH to: $env:PATH" +$currentDirectory = Get-Location +$envPath = "$env:PATH;$env:HELIX_CORRELATION_PAYLOAD\node\bin" function InvokeInstallDotnet([string]$command) { + Write-Host "InstallDotNet $command" + + $timeoutSeconds = 180 + $proc = Start-Process -filePath powershell.exe -ArgumentList "-noLogo -NoProfile -ExecutionPolicy unrestricted -command `"$command`"" -NoNewWindow -PassThru + + $proc | Wait-Process -Timeout $timeoutSeconds + $exitCode = $proc.ExitCode + + if ($exitCode -eq 0) { + Write-Host "InstallDotNet $command completed successfully" + return $true + } + elseif ([string]::IsNullOrWhiteSpace($exitCode)) { + Write-Warning "InstallDotNet $command timed out after $timeoutSeconds seconds" + } + else { + Write-Warning "InstallDotNet $command failed with exit code $exitCode" + } + + $proc | Stop-Process -Force + + return $false +} + +function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCred) { foreach ($i in 1..5) { - Write-Host "InstallDotNet $command" + $random = Get-Random -Maximum 1024 + $env:DOTNET_HOME = Join-Path $currentDirectory "sdk$random" + $env:DOTNET_ROOT = Join-Path $env:DOTNET_HOME $Arch + $env:DOTNET_CLI_HOME = Join-Path $currentDirectory "home$random" + $env:PATH = "$env:DOTNET_ROOT;$envPath" - $timeoutSeconds = 120 - $proc = Start-Process -filePath powershell.exe -ArgumentList "-noLogo -NoProfile -ExecutionPolicy unrestricted -command `"$command`"" -NoNewWindow -PassThru + Write-Host "Set PATH to: $env:PATH" - $proc | Wait-Process -Timeout $timeoutSeconds - $exitCode = $proc.ExitCode + $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true `'$Feed`' `'$FeedCred`' `$true" - if ($exitCode -eq 0) { - Write-Host "InstallDotNet $command completed successfully" - return + if (!$success) { + Write-Host "Retrying..." + continue } - if ([string]::IsNullOrWhiteSpace($exitCode)) { - Write-Warning "InstallDotNet $command timed out after $timeoutSeconds seconds retrying..." - } - else { - Write-Warning "InstallDotNet $command failed with exit code $exitCode retrying..." + $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch dotnet `$true `'$Feed`' `'$FeedCred`' `$true" + + if (!$success) { + Write-Host "Retrying..." + continue } - $proc | Stop-Process -Force + return } - Write-Warning "InstallDotNet $command exceeded retry limit" + Write-Error "InstallDotNet $command exceeded retry limit" exit 1 } if ($FeedCred -eq $null) { - InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true `'`' `'`' `$true") - InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $RuntimeVersion $Arch dotnet `$true `'`' `'`' `$true") + InstallDotnetSDKAndRuntime } else { - InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true https://dotnetclimsrc.blob.core.windows.net/dotnet $FeedCred `$true") - InvokeInstallDotnet(". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $RuntimeVersion $Arch dotnet `$true https://dotnetclimsrc.blob.core.windows.net/dotnet $FeedCred `$true") + InstallDotnetSDKAndRuntime "https://dotnetclimsrc.blob.core.windows.net/dotnet" $FeedCred } Write-Host "Restore: dotnet restore RunTests\RunTests.csproj --ignore-failed-sources" From 18f3e8752863212d1ce5717ad5ebdd63ccd1854a Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 8 Feb 2021 20:27:12 -0800 Subject: [PATCH 2/3] Debug? --- eng/helix/content/runtests.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/helix/content/runtests.ps1 b/eng/helix/content/runtests.ps1 index fb255bbd8a3a..089d1328ff32 100644 --- a/eng/helix/content/runtests.ps1 +++ b/eng/helix/content/runtests.ps1 @@ -42,7 +42,7 @@ function InvokeInstallDotnet([string]$command) { return $false } -function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCred) { +function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCredParam) { foreach ($i in 1..5) { $random = Get-Random -Maximum 1024 $env:DOTNET_HOME = Join-Path $currentDirectory "sdk$random" @@ -52,14 +52,14 @@ function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCred) { Write-Host "Set PATH to: $env:PATH" - $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true `'$Feed`' `'$FeedCred`' `$true" + $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch `'`' `$true `'$Feed`' `'$FeedCredParam`' `$true" if (!$success) { Write-Host "Retrying..." continue } - $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch dotnet `$true `'$Feed`' `'$FeedCred`' `$true" + $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch dotnet `$true `'$Feed`' `'$FeedCredParam`' `$true" if (!$success) { Write-Host "Retrying..." @@ -73,7 +73,7 @@ function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCred) { exit 1 } -if ($FeedCred -eq $null) { +if ([string]::IsNullOrEmpty($FeedCred)) { InstallDotnetSDKAndRuntime } else { InstallDotnetSDKAndRuntime "https://dotnetclimsrc.blob.core.windows.net/dotnet" $FeedCred From bcdad3ef0b027ae314d7b5871c5d72c716c10563 Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 8 Feb 2021 21:20:43 -0800 Subject: [PATCH 3/3] Update runtests.ps1 --- eng/helix/content/runtests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix/content/runtests.ps1 b/eng/helix/content/runtests.ps1 index 089d1328ff32..a6a7da67eede 100644 --- a/eng/helix/content/runtests.ps1 +++ b/eng/helix/content/runtests.ps1 @@ -59,7 +59,7 @@ function InstallDotnetSDKAndRuntime([string]$Feed, [string]$FeedCredParam) { continue } - $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $SdkVersion $Arch dotnet `$true `'$Feed`' `'$FeedCredParam`' `$true" + $success = InvokeInstallDotnet ". eng\common\tools.ps1; InstallDotNet $env:DOTNET_ROOT $RuntimeVersion $Arch dotnet `$true `'$Feed`' `'$FeedCredParam`' `$true" if (!$success) { Write-Host "Retrying..."