From 10443751b511f6a47bc01bf08467beda1a9cc880 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 10 Nov 2021 11:58:00 +0100 Subject: [PATCH 01/28] Inline regex string --- src/dotnet-install.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 8f81eddba2..f98c29084e 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -129,8 +129,6 @@ if ($SharedRuntime -and (-not $Runtime)) { $Runtime = "dotnet" } -# example path with regex: shared/1.0.0-beta-12345/somepath -$VersionRegEx="/\d+\.\d+[^/]+/" $OverrideNonVersionedFiles = !$SkipNonVersionedFiles function Say($str) { @@ -722,7 +720,8 @@ function Get-Absolute-Path([string]$RelativeOrAbsolutePath) { } function Get-Path-Prefix-With-Version($path) { - $match = [regex]::match($path, $VersionRegEx) + # example path with regex: shared/1.0.0-beta-12345/somepath + $match = [regex]::match($path, "/\d+\.\d+[^/]+/") if ($match.Success) { return $entry.FullName.Substring(0, $match.Index + $match.Length) } From a5b14ab18bcb6e76993db88375fdb6830ceb3c9c Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 10 Nov 2021 12:01:47 +0100 Subject: [PATCH 02/28] Remove unused binFolderRelativePath --- src/dotnet-install.ps1 | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index f98c29084e..7d970033a4 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -123,8 +123,6 @@ if ($NoCdn) { $AzureFeed = $UncachedFeed } -$BinFolderRelativePath="" - if ($SharedRuntime -and (-not $Runtime)) { $Runtime = "dotnet" } @@ -840,8 +838,8 @@ function SafeRemoveFile($Path) { } } -function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) { - $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath) +function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot) { + $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath "") if (-Not $NoPath) { $SuffixedBinPath = "$BinPath;" if (-Not $env:path.Contains($SuffixedBinPath)) { @@ -1071,7 +1069,7 @@ if ($SpecificVersion -ne $EffectiveVersion) $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion if ($isAssetInstalled) { Say "$assetName version $SpecificVersion is already installed." - Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot return } @@ -1201,7 +1199,7 @@ if (!$isAssetInstalled) { SafeRemoveFile -Path $ZipPath -Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath +Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot Say "Note that the script does not resolve dependencies during installation." Say "To check the list of dependencies, go to https://docs.microsoft.com/dotnet/core/install/windows#dependencies" From aafd1a13b5907125bb1973e43a581e3679e725ce Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 10 Nov 2021 13:27:54 +0100 Subject: [PATCH 03/28] Add feedcredential validation function --- src/dotnet-install.ps1 | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 7d970033a4..877b9382d2 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -225,7 +225,23 @@ function Get-CLIArchitecture-From-Architecture([string]$Architecture) { } } - +function ValidateFeedCredential() +{ + if ($Internal -and [string]::IsNullOrWhitespace($FeedCredential)) { + $message = "Provide credentials via -FeedCredential parameter." + if ($DryRun) { + Say-Warning "$message" + } else { + throw "$message" + } + } + + #FeedCredential should start with "?", for it to be added to the end of the link. + #adding "?" at the beginning of the FeedCredential if needed. + if ((![string]::IsNullOrWhitespace($FeedCredential)) -and ($FeedCredential[0] -ne '?')) { + $FeedCredential = "?" + $FeedCredential + } +} function Get-NormalizedQuality([string]$Quality) { Say-Invocation $MyInvocation @@ -932,21 +948,6 @@ Say "- The SDK needs to be installed without user interaction and without admin Say "- The SDK installation doesn't need to persist across multiple CI runs." Say "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.`r`n" -if ($Internal -and [string]::IsNullOrWhitespace($FeedCredential)) { - $message = "Provide credentials via -FeedCredential parameter." - if ($DryRun) { - Say-Warning "$message" - } else { - throw "$message" - } -} - -#FeedCredential should start with "?", for it to be added to the end of the link. -#adding "?" at the beginning of the FeedCredential if needed. -if ((![string]::IsNullOrWhitespace($FeedCredential)) -and ($FeedCredential[0] -ne '?')) { - $FeedCredential = "?" + $FeedCredential -} - $CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture $NormalizedQuality = Get-NormalizedQuality $Quality Say-Verbose "Normalized quality: '$NormalizedQuality'" @@ -954,6 +955,7 @@ $NormalizedChannel = Get-NormalizedChannel $Channel Say-Verbose "Normalized channel: '$NormalizedChannel'" $NormalizedProduct = Get-NormalizedProduct $Runtime Say-Verbose "Normalized product: '$NormalizedProduct'" +ValidateFeedCredential $DownloadLink = $null #try to get download location from aka.ms link From 080f160570e095013450fc619855ca715124d22e Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 10 Nov 2021 13:30:38 +0100 Subject: [PATCH 04/28] Move initializations below functions --- src/dotnet-install.ps1 | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 877b9382d2..65843a0839 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -119,16 +119,6 @@ Set-StrictMode -Version Latest $ErrorActionPreference="Stop" $ProgressPreference="SilentlyContinue" -if ($NoCdn) { - $AzureFeed = $UncachedFeed -} - -if ($SharedRuntime -and (-not $Runtime)) { - $Runtime = "dotnet" -} - -$OverrideNonVersionedFiles = !$SkipNonVersionedFiles - function Say($str) { try { Write-Host "dotnet-install: $str" @@ -948,6 +938,16 @@ Say "- The SDK needs to be installed without user interaction and without admin Say "- The SDK installation doesn't need to persist across multiple CI runs." Say "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.`r`n" +if ($NoCdn) { + $AzureFeed = $UncachedFeed +} + +if ($SharedRuntime -and (-not $Runtime)) { + $Runtime = "dotnet" +} + +$OverrideNonVersionedFiles = !$SkipNonVersionedFiles + $CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture $NormalizedQuality = Get-NormalizedQuality $Quality Say-Verbose "Normalized quality: '$NormalizedQuality'" From 4a70f1de2419b500182f9d279d80d9da391d3f24 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 10 Nov 2021 14:09:38 +0100 Subject: [PATCH 05/28] Get architecture cleanup --- src/dotnet-install.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 65843a0839..177be3930f 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -193,9 +193,7 @@ function Get-Machine-Architecture() { # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. # Possible values: amd64, x64, x86, arm64, arm - - if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) - { + if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) { return $ENV:PROCESSOR_ARCHITEW6432 } @@ -205,8 +203,11 @@ function Get-Machine-Architecture() { function Get-CLIArchitecture-From-Architecture([string]$Architecture) { Say-Invocation $MyInvocation + if ($Architecture -eq "") { + $Architecture = Get-Machine-Architecture + } + switch ($Architecture.ToLowerInvariant()) { - { $_ -eq "" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) } { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" } { $_ -eq "x86" } { return "x86" } { $_ -eq "arm" } { return "arm" } From f0f41cd362ca9e27af6f0dde6d7efcb0a2b1830c Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 10 Nov 2021 15:38:16 +0100 Subject: [PATCH 06/28] Rename latest version file parse method --- src/dotnet-install.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 177be3930f..c21c24a357 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -285,7 +285,7 @@ function Get-NormalizedProduct([string]$Runtime) { # Line 2: # 4-part version # For the aspnetcore runtime (1 line): # Line 1: # 4-part version -function Get-Version-Info-From-Version-Text([string]$VersionText) { +function Get-Version-From-LatestVersion-File-Content([string]$VersionText) { Say-Invocation $MyInvocation $Data = -split $VersionText @@ -465,7 +465,7 @@ function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel) { default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." } } - $VersionInfo = Get-Version-Info-From-Version-Text $VersionText + $VersionInfo = Get-Version-From-LatestVersion-File-Content $VersionText return $VersionInfo } From edfb13d38077f6afc4f1a99b3ce027b3321dab3a Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Sun, 14 Nov 2021 16:24:40 +0100 Subject: [PATCH 07/28] Move dryrun output into separate function --- src/dotnet-install.ps1 | 72 ++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index c21c24a357..69cff2418d 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -427,7 +427,7 @@ function GetHTTPResponse([Uri] $Uri, [bool]$HeaderOnly, [bool]$DisableRedirect, } } -function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel) { +function Get-Version-From-LatestVersion-File([string]$AzureFeed, [string]$Channel) { Say-Invocation $MyInvocation $VersionFileUrl = $null @@ -512,7 +512,7 @@ function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, if (-not $JSonFile) { if ($Version.ToLowerInvariant() -eq "latest") { - $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel + $LatestVersionInfo = Get-Version-From-LatestVersion-File -AzureFeed $AzureFeed -Channel $Channel return $LatestVersionInfo.Version } else { @@ -861,6 +861,41 @@ function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot) { } } +function PrintDryRunOutput() +{ + Say "Payload URLs:" + Say "Primary named payload URL: ${DownloadLink}" + if ($LegacyDownloadLink) { + Say "Legacy named payload URL: ${LegacyDownloadLink}" + } + $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`"" + if ($Runtime -eq "dotnet") { + $RepeatableCommand+=" -Runtime `"dotnet`"" + } + elseif ($Runtime -eq "aspnetcore") { + $RepeatableCommand+=" -Runtime `"aspnetcore`"" + } + + if (-not [string]::IsNullOrEmpty($NormalizedQuality)) + { + $RepeatableCommand+=" -Quality `"$NormalizedQuality`"" + } + + foreach ($key in $MyInvocation.BoundParameters.Keys) { + if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version","Quality","FeedCredential") -contains $key)) { + $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`"" + } + } + if ($MyInvocation.BoundParameters.Keys -contains "FeedCredential") { + $RepeatableCommand+=" -FeedCredential `"`"" + } + Say "Repeatable invocation: $RepeatableCommand" + if ($SpecificVersion -ne $EffectiveVersion) + { + Say "NOTE: Due to finding a version manifest with this runtime, it would actually install with version '$EffectiveVersion'" + } +} + function Get-AkaMSDownloadLink([string]$Channel, [string]$Quality, [bool]$Internal, [string]$Product, [string]$Architecture) { Say-Invocation $MyInvocation @@ -1007,38 +1042,7 @@ Say-Verbose "InstallRoot: $InstallRoot" $ScriptName = $MyInvocation.MyCommand.Name if ($DryRun) { - Say "Payload URLs:" - Say "Primary named payload URL: ${DownloadLink}" - if ($LegacyDownloadLink) { - Say "Legacy named payload URL: ${LegacyDownloadLink}" - } - $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`"" - if ($Runtime -eq "dotnet") { - $RepeatableCommand+=" -Runtime `"dotnet`"" - } - elseif ($Runtime -eq "aspnetcore") { - $RepeatableCommand+=" -Runtime `"aspnetcore`"" - } - - if (-not [string]::IsNullOrEmpty($NormalizedQuality)) - { - $RepeatableCommand+=" -Quality `"$NormalizedQuality`"" - } - - foreach ($key in $MyInvocation.BoundParameters.Keys) { - if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version","Quality","FeedCredential") -contains $key)) { - $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`"" - } - } - if ($MyInvocation.BoundParameters.Keys -contains "FeedCredential") { - $RepeatableCommand+=" -FeedCredential `"`"" - } - Say "Repeatable invocation: $RepeatableCommand" - if ($SpecificVersion -ne $EffectiveVersion) - { - Say "NOTE: Due to finding a version manifest with this runtime, it would actually install with version '$EffectiveVersion'" - } - + PrintDryRunOutput return } From 0b1c283b30322207dfa53c3f484e400c41263b2a Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Sun, 14 Nov 2021 18:35:44 +0100 Subject: [PATCH 08/28] Split aka.ms failure case --- src/dotnet-install.ps1 | 52 +++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 69cff2418d..3acbf80fd1 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -1111,6 +1111,8 @@ try { DownloadFile -Source $DownloadLink -OutPath $ZipPath } catch { + $DownloadFailed = $true + if ($PSItem.Exception.Data.Contains("StatusCode")) { $PrimaryDownloadStatusCode = $PSItem.Exception.Data["StatusCode"] } @@ -1128,37 +1130,35 @@ catch { } SafeRemoveFile -Path $ZipPath +} - if ($LegacyDownloadLink) { - $DownloadLink = $LegacyDownloadLink - $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) - Say-Verbose "Legacy zip path: $ZipPath" - Say "Downloading legacy link $DownloadLink" - try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath +# Downloading primary link has failed. But we still have a legacy link to try. +if ($DownloadFailed -and $LegacyDownloadLink) { + $DownloadLink = $LegacyDownloadLink + $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) + Say-Verbose "Legacy zip path: $ZipPath" + Say "Downloading legacy link $DownloadLink" + try { + DownloadFile -Source $DownloadLink -OutPath $ZipPath + } + catch { + if ($PSItem.Exception.Data.Contains("StatusCode")) { + $LegacyDownloadStatusCode = $PSItem.Exception.Data["StatusCode"] } - catch { - if ($PSItem.Exception.Data.Contains("StatusCode")) { - $LegacyDownloadStatusCode = $PSItem.Exception.Data["StatusCode"] - } - if ($PSItem.Exception.Data.Contains("ErrorMessage")) { - $LegacyDownloadFailedMsg = $PSItem.Exception.Data["ErrorMessage"] - } else { - $LegacyDownloadFailedMsg = $PSItem.Exception.Message - } - - if ($LegacyDownloadStatusCode -eq 404) { - Say "The resource at $DownloadLink is not available." - } else { - Say $PSItem.Exception.Message - } + if ($PSItem.Exception.Data.Contains("ErrorMessage")) { + $LegacyDownloadFailedMsg = $PSItem.Exception.Data["ErrorMessage"] + } else { + $LegacyDownloadFailedMsg = $PSItem.Exception.Message + } - SafeRemoveFile -Path $ZipPath - $DownloadFailed = $true + if ($LegacyDownloadStatusCode -eq 404) { + Say "The resource at $DownloadLink is not available." + } else { + Say $PSItem.Exception.Message } - } - else { + + SafeRemoveFile -Path $ZipPath $DownloadFailed = $true } } From 4bc5edabf385e6ae8b407e4c10ce60aabecd6e1d Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Sun, 14 Nov 2021 18:45:59 +0100 Subject: [PATCH 09/28] Small cleanup --- src/dotnet-install.ps1 | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 3acbf80fd1..6fbf8e959e 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -740,7 +740,7 @@ function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([Sys $ret = @() foreach ($entry in $Zip.Entries) { $dir = Get-Path-Prefix-With-Version $entry.FullName - if ($dir -ne $null) { + if ($null -ne $dir) { $path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir) if (-Not (Test-Path $path -PathType Container)) { $ret += $dir @@ -781,7 +781,7 @@ function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) { foreach ($entry in $Zip.Entries) { $PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName - if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) { + if (($null -eq $PathWithVersion) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) { $DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName) $DestinationDir = Split-Path -Parent $DestinationPath $OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath)) @@ -793,7 +793,7 @@ function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) { } } finally { - if ($Zip -ne $null) { + if ($null -ne $Zip) { $Zip.Dispose() } } @@ -822,7 +822,7 @@ function DownloadFile($Source, [string]$OutPath) { $File.Close() } finally { - if ($Stream -ne $null) { + if ($null -ne $Stream) { $Stream.Dispose() } } @@ -1098,7 +1098,7 @@ if ( ($diskInfo -ne $null) -and ($diskInfo.Free / 1MB -le 100)) { $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) Say-Verbose "Zip path: $ZipPath" -$DownloadFailed = $false +$DownloadFailed = $true $PrimaryDownloadStatusCode = 0 $LegacyDownloadStatusCode = 0 @@ -1109,10 +1109,9 @@ $LegacyDownloadFailedMsg = "" Say "Downloading primary link $DownloadLink" try { DownloadFile -Source $DownloadLink -OutPath $ZipPath + $DownloadFailed = $false } catch { - $DownloadFailed = $true - if ($PSItem.Exception.Data.Contains("StatusCode")) { $PrimaryDownloadStatusCode = $PSItem.Exception.Data["StatusCode"] } @@ -1140,6 +1139,7 @@ if ($DownloadFailed -and $LegacyDownloadLink) { Say "Downloading legacy link $DownloadLink" try { DownloadFile -Source $DownloadLink -OutPath $ZipPath + $DownloadFailed = $false } catch { if ($PSItem.Exception.Data.Contains("StatusCode")) { @@ -1159,7 +1159,6 @@ if ($DownloadFailed -and $LegacyDownloadLink) { } SafeRemoveFile -Path $ZipPath - $DownloadFailed = $true } } From 63f3bcb1556fd796f2ad1c09ae2d0d0289f8c6ce Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Sun, 14 Nov 2021 19:05:36 +0100 Subject: [PATCH 10/28] Convert AzureFeed to a list --- src/dotnet-install.ps1 | 45 +++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 6fbf8e959e..66337c6f18 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -66,11 +66,13 @@ Displays diagnostics information. .PARAMETER AzureFeed Default: https://dotnetcli.azureedge.net/dotnet - This parameter typically is not changed by the user. - It allows changing the URL for the Azure feed used by this installer. + For internal use only. + Allows using a different storage to download SDK archives from. + This parameter is only used if $NoCdn is false. .PARAMETER UncachedFeed - This parameter typically is not changed by the user. - It allows changing the URL for the Uncached feed used by this installer. + For internal use only. + Allows using a different storage to download SDK archives from. + This parameter is only used if $NoCdn is true. .PARAMETER ProxyAddress If set, the installer will use the proxy when making web requests .PARAMETER ProxyUseDefaultCredentials @@ -101,11 +103,11 @@ param( [string]$Architecture="", [string]$Runtime, [Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")] - [switch]$SharedRuntime, + [Parameter(DontShow=$true)][switch]$SharedRuntime, [switch]$DryRun, [switch]$NoPath, - [string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet", - [string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet", + [Parameter(DontShow=$true)][string]$AzureFeed, + [Parameter(DontShow=$true)][string]$UncachedFeed, [string]$FeedCredential, [string]$ProxyAddress, [switch]$ProxyUseDefaultCredentials, @@ -969,15 +971,36 @@ function Get-AkaMSDownloadLink([string]$Channel, [string]$Quality, [bool]$Intern } +function Get-Feeds-To-Use() +{ + $feeds = @( + "https://dotnetcli.azureedge.net/dotnet", + "https://dotnetbuilds.azureedge.net/public" + ) + + if (-not [string]::IsNullOrEmpty($AzureFeed)) { + $feeds = @($AddSource) + } + + if ($NoCdn) { + $feeds = @( + "https://dotnetcli.blob.core.windows.net/dotnet", + "https://dotnetbuilds.blob.core.windows.net/public" + ) + + if (-not [string]::IsNullOrEmpty($UncachedFeed)) { + $feeds = @($UncachedFeed) + } + } + + return $feeds +} + Say "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" Say "- The SDK needs to be installed without user interaction and without admin rights." Say "- The SDK installation doesn't need to persist across multiple CI runs." Say "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.`r`n" -if ($NoCdn) { - $AzureFeed = $UncachedFeed -} - if ($SharedRuntime -and (-not $Runtime)) { $Runtime = "dotnet" } From 4f4fb28211ddfa527cfa0da974bf87af9a4ec666 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 22 Nov 2021 14:55:23 +0100 Subject: [PATCH 11/28] Generate all links before downloading --- src/dotnet-install.ps1 | 144 +++++++++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 57 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 66337c6f18..7b8eee1b4e 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -434,16 +434,16 @@ function Get-Version-From-LatestVersion-File([string]$AzureFeed, [string]$Channe $VersionFileUrl = $null if ($Runtime -eq "dotnet") { - $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" + $VersionFileUrl = "$AzureFeed/Runtime/$Channel/latest.version" } elseif ($Runtime -eq "aspnetcore") { - $VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version" + $VersionFileUrl = "$AzureFeed/aspnetcore/Runtime/$Channel/latest.version" } elseif ($Runtime -eq "windowsdesktop") { - $VersionFileUrl = "$UncachedFeed/WindowsDesktop/$Channel/latest.version" + $VersionFileUrl = "$AzureFeed/WindowsDesktop/$Channel/latest.version" } elseif (-not $Runtime) { - $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version" + $VersionFileUrl = "$AzureFeed/Sdk/$Channel/latest.version" } else { throw "Invalid value for `$Runtime" @@ -996,6 +996,48 @@ function Get-Feeds-To-Use() return $feeds } +function Resolve-AssetName-And-RelativePath([string] $Runtime) { + + if ($Runtime -eq "dotnet") { + $assetName = ".NET Core Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App" + } + elseif ($Runtime -eq "aspnetcore") { + $assetName = "ASP.NET Core Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App" + } + elseif ($Runtime -eq "windowsdesktop") { + $assetName = ".NET Core Windows Desktop Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App" + } + elseif (-not $Runtime) { + $assetName = ".NET Core SDK" + $dotnetPackageRelativePath = "sdk" + } + else { + throw "Invalid value for `$Runtime" + } + + return ($assetName, $dotnetPackageRelativePath) +} + +function Prepare-Install-Directory { + New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null + + $installDrive = $((Get-Item $InstallRoot -Force).PSDrive.Name); + $diskInfo = $null + try{ + $diskInfo = Get-PSDrive -Name $installDrive + } + catch{ + Say-Warning "Failed to check the disk space. Installation will continue, but it may fail if you do not have enough disk space." + } + + if ( ($null -ne $diskInfo) -and ($diskInfo.Free / 1MB -le 100)) { + throw "There is not enough disk space on drive ${installDrive}:" + } +} + Say "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" Say "- The SDK needs to be installed without user interaction and without admin rights." Say "- The SDK installation doesn't need to persist across multiple CI runs." @@ -1015,7 +1057,14 @@ Say-Verbose "Normalized channel: '$NormalizedChannel'" $NormalizedProduct = Get-NormalizedProduct $Runtime Say-Verbose "Normalized product: '$NormalizedProduct'" ValidateFeedCredential -$DownloadLink = $null + +$InstallRoot = Resolve-Installation-Path $InstallDir +Say-Verbose "InstallRoot: $InstallRoot" +$ScriptName = $MyInvocation.MyCommand.Name +($assetName, $dotnetPackageRelativePath) = Resolve-AssetName-And-RelativePath -Runtime $Runtime + +$feeds = Get-Feeds-To-Use +$DownloadLinks = @() #try to get download location from aka.ms link #not applicable when exact version is specified via command or json file @@ -1033,60 +1082,54 @@ if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { } else { Say-Verbose "Retrieved primary named payload URL from aka.ms link: '$AkaMsDownloadLink'." - $DownloadLink = $AkaMsDownloadLink Say-Verbose "Downloading using legacy url will not be attempted." - $LegacyDownloadLink = $null #get version from the path - $pathParts = $DownloadLink.Split('/') + $pathParts = $AkaMsDownloadLink.Split('/') if ($pathParts.Length -ge 2) { $SpecificVersion = $pathParts[$pathParts.Length - 2] Say-Verbose "Version: '$SpecificVersion'." } else { - Say-Error "Failed to extract the version from download link '$DownloadLink'." + Say-Error "Failed to extract the version from download link '$AkaMsDownloadLink'." } #retrieve effective (product) version - $EffectiveVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -PackageDownloadLink $DownloadLink + # TODO don't use feed for this, get the address from aka.ms link + $EffectiveVersion = Get-Product-Version -AzureFeed $feed -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink Say-Verbose "Product version: '$EffectiveVersion'." - } -} - -if ([string]::IsNullOrEmpty($DownloadLink)) { - $SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile - $DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture - $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture -} + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} -$InstallRoot = Resolve-Installation-Path $InstallDir -Say-Verbose "InstallRoot: $InstallRoot" -$ScriptName = $MyInvocation.MyCommand.Name - -if ($DryRun) { - PrintDryRunOutput - return + if ($DryRun) { + PrintDryRunOutput + return + } + } } -if ($Runtime -eq "dotnet") { - $assetName = ".NET Core Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App" -} -elseif ($Runtime -eq "aspnetcore") { - $assetName = "ASP.NET Core Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App" -} -elseif ($Runtime -eq "windowsdesktop") { - $assetName = ".NET Core Windows Desktop Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App" -} -elseif (-not $Runtime) { - $assetName = ".NET Core SDK" - $dotnetPackageRelativePath = "sdk" -} -else { - throw "Invalid value for `$Runtime" +foreach ($feed in $feeds) { + try { + $SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $feed -Channel $Channel -Version $Version -JSonFile $JSonFile + $DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='primary'} + + if (-not [string]::IsNullOrEmpty($LegacyDownloadLink)) { + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$LegacyDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='legacy'} + } + + if ($DryRun) { + PrintDryRunOutput + return + } + break + } + catch + { + Say-Verbose "Failed to acquire download links from feed $feed. Exception: $_" + } } if ($SpecificVersion -ne $EffectiveVersion) @@ -1103,20 +1146,7 @@ if ($isAssetInstalled) { return } -New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null - -$installDrive = $((Get-Item $InstallRoot -Force).PSDrive.Name); -$diskInfo = $null -try{ - $diskInfo = Get-PSDrive -Name $installDrive -} -catch{ - Say-Warning "Failed to check the disk space. Installation will continue, but it may fail if you do not have enough disk space." -} - -if ( ($diskInfo -ne $null) -and ($diskInfo.Free / 1MB -le 100)) { - throw "There is not enough disk space on drive ${installDrive}:" -} +Prepare-Install-Directory $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) Say-Verbose "Zip path: $ZipPath" From 0ce4a0e97f34b214787430c60c5c9e9dbbd09f22 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 22 Nov 2021 15:39:12 +0100 Subject: [PATCH 12/28] Use generated links to download --- src/dotnet-install.ps1 | 89 ++++++++++++------------------------------ 1 file changed, 25 insertions(+), 64 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 7b8eee1b4e..9a0fb28867 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -1095,8 +1095,7 @@ if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { } #retrieve effective (product) version - # TODO don't use feed for this, get the address from aka.ms link - $EffectiveVersion = Get-Product-Version -AzureFeed $feed -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink + $EffectiveVersion = Get-Product-Version -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink Say-Verbose "Product version: '$EffectiveVersion'." $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} @@ -1151,87 +1150,49 @@ Prepare-Install-Directory $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) Say-Verbose "Zip path: $ZipPath" -$DownloadFailed = $true +$DownloadSucceeded = $false -$PrimaryDownloadStatusCode = 0 -$LegacyDownloadStatusCode = 0 +$ErrorMessages = @() -$PrimaryDownloadFailedMsg = "" -$LegacyDownloadFailedMsg = "" - -Say "Downloading primary link $DownloadLink" -try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath - $DownloadFailed = $false -} -catch { - if ($PSItem.Exception.Data.Contains("StatusCode")) { - $PrimaryDownloadStatusCode = $PSItem.Exception.Data["StatusCode"] - } - - if ($PSItem.Exception.Data.Contains("ErrorMessage")) { - $PrimaryDownloadFailedMsg = $PSItem.Exception.Data["ErrorMessage"] - } else { - $PrimaryDownloadFailedMsg = $PSItem.Exception.Message - } - - if ($PrimaryDownloadStatusCode -eq 404) { - Say "The resource at $DownloadLink is not available." - } else { - Say $PSItem.Exception.Message - } - - SafeRemoveFile -Path $ZipPath -} +foreach ($link in $DownloadLinks) +{ + Say "Downloading `"$($link.type)`" link $($link.downloadLink)" -# Downloading primary link has failed. But we still have a legacy link to try. -if ($DownloadFailed -and $LegacyDownloadLink) { - $DownloadLink = $LegacyDownloadLink - $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) - Say-Verbose "Legacy zip path: $ZipPath" - Say "Downloading legacy link $DownloadLink" try { DownloadFile -Source $DownloadLink -OutPath $ZipPath - $DownloadFailed = $false + $DownloadSucceeded = $true + break } catch { + $StatusCode = $null + $ErrorMessage = $null + if ($PSItem.Exception.Data.Contains("StatusCode")) { - $LegacyDownloadStatusCode = $PSItem.Exception.Data["StatusCode"] + $StatusCode = $PSItem.Exception.Data["StatusCode"] } - + if ($PSItem.Exception.Data.Contains("ErrorMessage")) { - $LegacyDownloadFailedMsg = $PSItem.Exception.Data["ErrorMessage"] - } else { - $LegacyDownloadFailedMsg = $PSItem.Exception.Message - } - - if ($LegacyDownloadStatusCode -eq 404) { - Say "The resource at $DownloadLink is not available." + $ErrorMessage = $PSItem.Exception.Data["ErrorMessage"] } else { - Say $PSItem.Exception.Message + $ErrorMessage = $PSItem.Exception.Message } - SafeRemoveFile -Path $ZipPath + $ErrorMessages += "Downloading from `"$($link.type)`" link has failed with error:`nUri: $($link.downloadLink)`nStatusCode: $StatusCode`nError: $ErrorMessage" } + + # This link failed. Clean up before trying the next one. + SafeRemoveFile -Path $ZipPath } -if ($DownloadFailed) { - if (($PrimaryDownloadStatusCode -eq 404) -and ((-not $LegacyDownloadLink) -or ($LegacyDownloadStatusCode -eq 404))) { - throw "Could not find `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" - } else { - # 404-NotFound is an expected response if it goes from only one of the links, do not show that error. - # If primary path is available (not 404-NotFound) then show the primary error else show the legacy error. - if ($PrimaryDownloadStatusCode -ne 404) { - throw "Could not download `"$assetName`" with version = $SpecificVersion`r`n$PrimaryDownloadFailedMsg" - } - if (($LegacyDownloadLink) -and ($LegacyDownloadStatusCode -ne 404)) { - throw "Could not download `"$assetName`" with version = $SpecificVersion`r`n$LegacyDownloadFailedMsg" - } - throw "Could not download `"$assetName`" with version = $SpecificVersion" +if (-not $DownloadSucceeded) { + foreach ($ErrorMessage in $ErrorMessages) { + Say-Error $ErrorMessages } + + throw "Could not find `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" } -Say "Extracting zip from $DownloadLink" +Say "Extracting the archive." Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot # Check if the SDK version is installed; if not, fail the installation. From 1c143aa5d90dfea28b19bdcee307fb89e170ec83 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 22 Nov 2021 17:04:34 +0100 Subject: [PATCH 13/28] Error when no links were constructed --- src/dotnet-install.ps1 | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 9a0fb28867..2b98b9ac84 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -326,7 +326,11 @@ function GetHTTPResponse([Uri] $Uri, [bool]$HeaderOnly, [bool]$DisableRedirect, # Despite no proxy being explicitly specified, we may still be behind a default proxy $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy; if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) { - $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString + if ($null -ne $DefaultProxy.GetProxy($Uri)) { + $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString + } else { + $ProxyAddress = $null + } $ProxyUseDefaultCredentials = $true } } catch { @@ -455,7 +459,7 @@ function Get-Version-From-LatestVersion-File([string]$AzureFeed, [string]$Channe $Response = GetHTTPResponse -Uri $VersionFileUrl } catch { - Say-Error "Could not resolve version information." + Say-Verbose "Failed to download latest.version file." throw } $StringContent = $Response.Content.ReadAsStringAsync().Result @@ -1099,6 +1103,7 @@ if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { Say-Verbose "Product version: '$EffectiveVersion'." $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} + Say-Verbose "Generated aka.ms link $AkaMsDownloadLink with version $EffectiveVersion" if ($DryRun) { PrintDryRunOutput @@ -1114,9 +1119,11 @@ foreach ($feed in $feeds) { $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='primary'} + Say-Verbose "Generated primary link $DownloadLink with version $EffectiveVersion" if (-not [string]::IsNullOrEmpty($LegacyDownloadLink)) { $DownloadLinks += New-Object PSObject -Property @{downloadLink="$LegacyDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='legacy'} + Say-Verbose "Generated legacy link $DownloadLink with version $EffectiveVersion" } if ($DryRun) { @@ -1131,6 +1138,9 @@ foreach ($feed in $feeds) { } } +if ($DownloadLinks.count -eq 0) { + throw "Failed to resolve the exact version number." +} if ($SpecificVersion -ne $EffectiveVersion) { Say "Performing installation checks for effective version: $EffectiveVersion" @@ -1159,7 +1169,7 @@ foreach ($link in $DownloadLinks) Say "Downloading `"$($link.type)`" link $($link.downloadLink)" try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath + DownloadFile -Source $link.downloadLink -OutPath $ZipPath $DownloadSucceeded = $true break } From 913b08be189da1db5cee795d516ebf6ef8e7b8c2 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 24 Nov 2021 09:17:48 +0100 Subject: [PATCH 14/28] Update checks on already installed --- src/dotnet-install.ps1 | 49 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 2b98b9ac84..2d03bf20e4 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -1102,6 +1102,13 @@ if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { $EffectiveVersion = Get-Product-Version -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink Say-Verbose "Product version: '$EffectiveVersion'." + if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) + { + Say "$assetName version $EffectiveVersion is already installed." + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot + return + } + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} Say-Verbose "Generated aka.ms link $AkaMsDownloadLink with version $EffectiveVersion" @@ -1126,6 +1133,14 @@ foreach ($feed in $feeds) { Say-Verbose "Generated legacy link $DownloadLink with version $EffectiveVersion" } + Say-Verbose "Checking if the version $EffectiveVersion is already installed" + if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) + { + Say "$assetName version $EffeciveVersion is already installed." + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot + return + } + if ($DryRun) { PrintDryRunOutput return @@ -1141,19 +1156,6 @@ foreach ($feed in $feeds) { if ($DownloadLinks.count -eq 0) { throw "Failed to resolve the exact version number." } -if ($SpecificVersion -ne $EffectiveVersion) -{ - Say "Performing installation checks for effective version: $EffectiveVersion" - $SpecificVersion = $EffectiveVersion -} - -# Check if the SDK version is already installed. -$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion -if ($isAssetInstalled) { - Say "$assetName version $SpecificVersion is already installed." - Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot - return -} Prepare-Install-Directory @@ -1161,16 +1163,18 @@ $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO Say-Verbose "Zip path: $ZipPath" $DownloadSucceeded = $false - +$DownloadedLink = $null $ErrorMessages = @() foreach ($link in $DownloadLinks) { - Say "Downloading `"$($link.type)`" link $($link.downloadLink)" + Say-Verbose "Downloading `"$($link.type)`" link $($link.downloadLink)" try { DownloadFile -Source $link.downloadLink -OutPath $ZipPath + Say-Verbose "Download succeeded." $DownloadSucceeded = $true + $DownloadedLink = $link break } catch { @@ -1187,6 +1191,7 @@ foreach ($link in $DownloadLinks) $ErrorMessage = $PSItem.Exception.Message } + Say-Verbose "Download failed with status code $StatusCode. Error message: $ErrorMessage" $ErrorMessages += "Downloading from `"$($link.type)`" link has failed with error:`nUri: $($link.downloadLink)`nStatusCode: $StatusCode`nError: $ErrorMessage" } @@ -1199,7 +1204,7 @@ if (-not $DownloadSucceeded) { Say-Error $ErrorMessages } - throw "Could not find `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" + throw "Could not find `"$assetName`" with version = $($DownloadLinks[0].effectiveVersion)`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET support" } Say "Extracting the archive." @@ -1209,22 +1214,22 @@ Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot $isAssetInstalled = $false # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. -if ($SpecificVersion -Match "rtm" -or $SpecificVersion -Match "servicing") { - $ReleaseVersion = $SpecificVersion.Split("-")[0] +if ($DownloadedLink.effectiveVersion -Match "rtm" -or $DownloadedLink.effectiveVersion -Match "servicing") { + $ReleaseVersion = $DownloadedLink.effectiveVersion.Split("-")[0] Say-Verbose "Checking installation: version = $ReleaseVersion" $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion } # Check if the SDK version is installed. if (!$isAssetInstalled) { - Say-Verbose "Checking installation: version = $SpecificVersion" - $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion + Say-Verbose "Checking installation: version = $($DownloadedLink.effectiveVersion)" + $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $DownloadedLink.effectiveVersion } # Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. if (!$isAssetInstalled) { - Say-Error "Failed to verify the version of installed `"$assetName`".`nInstallation source: $DownloadLink.`nInstallation location: $InstallRoot.`nReport the bug at https://github.com/dotnet/install-scripts/issues." - throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error." + Say-Error "Failed to verify the version of installed `"$assetName`".`nInstallation source: $($DownloadedLink.downloadLink).`nInstallation location: $InstallRoot.`nReport the bug at https://github.com/dotnet/install-scripts/issues." + throw "`"$assetName`" with version = $($DownloadedLink.effectiveVersion) failed to install with an unknown error." } SafeRemoveFile -Path $ZipPath From ee49cbc0d72c1344b99e110ea3c82d32931e339c Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 24 Nov 2021 13:06:07 +0100 Subject: [PATCH 15/28] DryRun should happen before installation check --- src/dotnet-install.ps1 | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 2d03bf20e4..794d2ab791 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -867,7 +867,7 @@ function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot) { } } -function PrintDryRunOutput() +function PrintDryRunOutput($Invocation, [string] $DownloadLink, [string] $LegacyDownloadLink, [string] $SpecificVersion, [string] $EffectiveVersion) { Say "Payload URLs:" Say "Primary named payload URL: ${DownloadLink}" @@ -882,17 +882,12 @@ function PrintDryRunOutput() $RepeatableCommand+=" -Runtime `"aspnetcore`"" } - if (-not [string]::IsNullOrEmpty($NormalizedQuality)) - { - $RepeatableCommand+=" -Quality `"$NormalizedQuality`"" - } - - foreach ($key in $MyInvocation.BoundParameters.Keys) { + foreach ($key in $Invocation.BoundParameters.Keys) { if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version","Quality","FeedCredential") -contains $key)) { - $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`"" + $RepeatableCommand+=" -$key `"$($Invocation.BoundParameters[$key])`"" } } - if ($MyInvocation.BoundParameters.Keys -contains "FeedCredential") { + if ($Invocation.BoundParameters.Keys -contains "FeedCredential") { $RepeatableCommand+=" -FeedCredential `"`"" } Say "Repeatable invocation: $RepeatableCommand" @@ -1101,21 +1096,22 @@ if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { #retrieve effective (product) version $EffectiveVersion = Get-Product-Version -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink Say-Verbose "Product version: '$EffectiveVersion'." + + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} + Say-Verbose "Generated aka.ms link $AkaMsDownloadLink with version $EffectiveVersion" + + if ($DryRun) { + PrintDryRunOutput $MyInvocation $AkaMsDownloadLink $null $SpecificVersion $EffectiveVersion + return + } + Say-Verbose "Checking if the version $EffectiveVersion is already installed" if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) { Say "$assetName version $EffectiveVersion is already installed." Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot return } - - $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} - Say-Verbose "Generated aka.ms link $AkaMsDownloadLink with version $EffectiveVersion" - - if ($DryRun) { - PrintDryRunOutput - return - } } } @@ -1133,6 +1129,11 @@ foreach ($feed in $feeds) { Say-Verbose "Generated legacy link $DownloadLink with version $EffectiveVersion" } + if ($DryRun) { + PrintDryRunOutput $MyInvocation $DownloadLink $LegacyDownloadLink $SpecificVersion $EffectiveVersion + return + } + Say-Verbose "Checking if the version $EffectiveVersion is already installed" if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) { @@ -1140,12 +1141,6 @@ foreach ($feed in $feeds) { Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot return } - - if ($DryRun) { - PrintDryRunOutput - return - } - break } catch { From f1712a67ba33f62b5f2a5323a92a5ca4c04c8455 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 24 Nov 2021 13:22:12 +0100 Subject: [PATCH 16/28] Fix credential validation --- src/dotnet-install.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 794d2ab791..4833a91490 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -218,7 +218,7 @@ function Get-CLIArchitecture-From-Architecture([string]$Architecture) { } } -function ValidateFeedCredential() +function ValidateFeedCredential([string] $FeedCredential) { if ($Internal -and [string]::IsNullOrWhitespace($FeedCredential)) { $message = "Provide credentials via -FeedCredential parameter." @@ -234,6 +234,8 @@ function ValidateFeedCredential() if ((![string]::IsNullOrWhitespace($FeedCredential)) -and ($FeedCredential[0] -ne '?')) { $FeedCredential = "?" + $FeedCredential } + + return $FeedCredential } function Get-NormalizedQuality([string]$Quality) { Say-Invocation $MyInvocation @@ -1055,7 +1057,7 @@ $NormalizedChannel = Get-NormalizedChannel $Channel Say-Verbose "Normalized channel: '$NormalizedChannel'" $NormalizedProduct = Get-NormalizedProduct $Runtime Say-Verbose "Normalized product: '$NormalizedProduct'" -ValidateFeedCredential +$FeedCredential = ValidateFeedCredential $FeedCredential $InstallRoot = Resolve-Installation-Path $InstallDir Say-Verbose "InstallRoot: $InstallRoot" From f23c9ea235303d750ccd30f5b756966d5f8a1a3b Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 24 Nov 2021 19:26:08 +0100 Subject: [PATCH 17/28] Fix wrong variable usage --- src/dotnet-install.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 4833a91490..089d5fdda0 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -980,7 +980,7 @@ function Get-Feeds-To-Use() ) if (-not [string]::IsNullOrEmpty($AzureFeed)) { - $feeds = @($AddSource) + $feeds = @($AzureFeed) } if ($NoCdn) { From 6527604519b99f3fd7e8b32c1fb5eac2ffd0a4da Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Wed, 24 Nov 2021 23:33:01 +0100 Subject: [PATCH 18/28] Move aka.ms stuff into function --- src/dotnet-install.ps1 | 137 +++++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 61 deletions(-) diff --git a/src/dotnet-install.ps1 b/src/dotnet-install.ps1 index 089d5fdda0..913cbcb3c9 100644 --- a/src/dotnet-install.ps1 +++ b/src/dotnet-install.ps1 @@ -972,6 +972,42 @@ function Get-AkaMSDownloadLink([string]$Channel, [string]$Quality, [bool]$Intern } +function Get-AkaMsLink-And-Version([string] $NormalizedChannel, [string] $NormalizedQuality, [bool] $Internal, [string] $ProductName, [string] $Architecture) { + $AkaMsDownloadLink = Get-AkaMSDownloadLink -Channel $NormalizedChannel -Quality $NormalizedQuality -Internal $Internal -Product $ProductName -Architecture $Architecture + + if ([string]::IsNullOrEmpty($AkaMsDownloadLink)){ + if (-not [string]::IsNullOrEmpty($NormalizedQuality)) { + # if quality is specified - exit with error - there is no fallback approach + Say-Error "Failed to locate the latest version in the channel '$NormalizedChannel' with '$NormalizedQuality' quality for '$ProductName', os: 'win', architecture: '$Architecture'." + Say-Error "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." + throw "aka.ms link resolution failure" + } + Say-Verbose "Falling back to latest.version file approach." + return ($null, $null, $null) + } + else { + Say-Verbose "Retrieved primary named payload URL from aka.ms link: '$AkaMsDownloadLink'." + Say-Verbose "Downloading using legacy url will not be attempted." + + #get version from the path + $pathParts = $AkaMsDownloadLink.Split('/') + if ($pathParts.Length -ge 2) { + $SpecificVersion = $pathParts[$pathParts.Length - 2] + Say-Verbose "Version: '$SpecificVersion'." + } + else { + Say-Error "Failed to extract the version from download link '$AkaMsDownloadLink'." + return ($null, $null, $null) + } + + #retrieve effective (product) version + $EffectiveVersion = Get-Product-Version -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink + Say-Verbose "Product version: '$EffectiveVersion'." + + return ($AkaMsDownloadLink, $SpecificVersion, $EffectiveVersion); + } +} + function Get-Feeds-To-Use() { $feeds = @( @@ -1067,43 +1103,16 @@ $ScriptName = $MyInvocation.MyCommand.Name $feeds = Get-Feeds-To-Use $DownloadLinks = @() -#try to get download location from aka.ms link -#not applicable when exact version is specified via command or json file +# aka.ms links can only be used if the user did not request a specific version via the command line or a global.json file. if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { - $AkaMsDownloadLink = Get-AkaMSDownloadLink -Channel $NormalizedChannel -Quality $NormalizedQuality -Internal $Internal -Product $NormalizedProduct -Architecture $CLIArchitecture - - if ([string]::IsNullOrEmpty($AkaMsDownloadLink)){ - if (-not [string]::IsNullOrEmpty($NormalizedQuality)) { - # if quality is specified - exit with error - there is no fallback approach - Say-Error "Failed to locate the latest version in the channel '$NormalizedChannel' with '$NormalizedQuality' quality for '$NormalizedProduct', os: 'win', architecture: '$CLIArchitecture'." - Say-Error "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." - throw "aka.ms link resolution failure" - } - Say-Verbose "Falling back to latest.version file approach." - } - else { - Say-Verbose "Retrieved primary named payload URL from aka.ms link: '$AkaMsDownloadLink'." - Say-Verbose "Downloading using legacy url will not be attempted." - - #get version from the path - $pathParts = $AkaMsDownloadLink.Split('/') - if ($pathParts.Length -ge 2) { - $SpecificVersion = $pathParts[$pathParts.Length - 2] - Say-Verbose "Version: '$SpecificVersion'." - } - else { - Say-Error "Failed to extract the version from download link '$AkaMsDownloadLink'." - } - - #retrieve effective (product) version - $EffectiveVersion = Get-Product-Version -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink - Say-Verbose "Product version: '$EffectiveVersion'." - - $DownloadLinks += New-Object PSObject -Property @{downloadLink="$AkaMsDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} - Say-Verbose "Generated aka.ms link $AkaMsDownloadLink with version $EffectiveVersion" + ($DownloadLink, $SpecificVersion, $EffectiveVersion) = Get-AkaMsLink-And-Version $NormalizedChannel $NormalizedQuality $Internal $NormalizedProduct $CLIArchitecture + + if ($null -ne $DownloadLink) { + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} + Say-Verbose "Generated aka.ms link $DownloadLink with version $EffectiveVersion" if ($DryRun) { - PrintDryRunOutput $MyInvocation $AkaMsDownloadLink $null $SpecificVersion $EffectiveVersion + PrintDryRunOutput $MyInvocation $DownloadLink $null $SpecificVersion $EffectiveVersion return } @@ -1117,43 +1126,49 @@ if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { } } -foreach ($feed in $feeds) { - try { - $SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $feed -Channel $Channel -Version $Version -JSonFile $JSonFile - $DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture - $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture - - $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='primary'} - Say-Verbose "Generated primary link $DownloadLink with version $EffectiveVersion" - - if (-not [string]::IsNullOrEmpty($LegacyDownloadLink)) { - $DownloadLinks += New-Object PSObject -Property @{downloadLink="$LegacyDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='legacy'} - Say-Verbose "Generated legacy link $DownloadLink with version $EffectiveVersion" - } - - if ($DryRun) { - PrintDryRunOutput $MyInvocation $DownloadLink $LegacyDownloadLink $SpecificVersion $EffectiveVersion - return +# Primary and legacy links cannot be used if a quality was specified. +# If we already have an aka.ms link, no need to search the blob feeds. +if ([string]::IsNullOrEmpty($NormalizedQuality) -and 0 -eq $DownloadLinks.count) +{ + foreach ($feed in $feeds) { + try { + $SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $feed -Channel $Channel -Version $Version -JSonFile $JSonFile + $DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='primary'} + Say-Verbose "Generated primary link $DownloadLink with version $EffectiveVersion" + + if (-not [string]::IsNullOrEmpty($LegacyDownloadLink)) { + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$LegacyDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='legacy'} + Say-Verbose "Generated legacy link $DownloadLink with version $EffectiveVersion" + } + + if ($DryRun) { + PrintDryRunOutput $MyInvocation $DownloadLink $LegacyDownloadLink $SpecificVersion $EffectiveVersion + return + } + + Say-Verbose "Checking if the version $EffectiveVersion is already installed" + if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) + { + Say "$assetName version $EffeciveVersion is already installed." + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot + return + } } - - Say-Verbose "Checking if the version $EffectiveVersion is already installed" - if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) + catch { - Say "$assetName version $EffeciveVersion is already installed." - Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot - return + Say-Verbose "Failed to acquire download links from feed $feed. Exception: $_" } } - catch - { - Say-Verbose "Failed to acquire download links from feed $feed. Exception: $_" - } } if ($DownloadLinks.count -eq 0) { throw "Failed to resolve the exact version number." } + Prepare-Install-Directory $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) From fba8e4b377ce8f32812ed78b12eb0b72916a6176 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 3 Jan 2022 16:07:09 +0100 Subject: [PATCH 19/28] Clean up --- src/dotnet-install.sh | 118 +++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 9b46690241..470221be16 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -135,6 +135,31 @@ get_legacy_os_name_from_platform() { return 1 } +get_legacy_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ -n "$runtime_id" ]; then + echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") + if [ -n "$os" ]; then + echo "$os" + return 0 + fi + fi + fi + + say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" + return 1 +} + get_linux_platform_name() { eval $invocation @@ -196,31 +221,6 @@ get_current_os_name() { return 1 } -get_legacy_os_name() { - eval $invocation - - local uname=$(uname) - if [ "$uname" = "Darwin" ]; then - echo "osx" - return 0 - elif [ -n "$runtime_id" ]; then - echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") - return 0 - else - if [ -e /etc/os-release ]; then - . /etc/os-release - os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") - if [ -n "$os" ]; then - echo "$os" - return 0 - fi - fi - fi - - say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" - return 1 -} - machine_has() { eval $invocation @@ -228,7 +228,6 @@ machine_has() { return $? } - check_min_reqs() { local hasMinimum=false if machine_has "curl"; then @@ -321,11 +320,13 @@ get_normalized_architecture_from_architecture() { eval $invocation local architecture="$(to_lowercase "$1")" + + if [[ $architecture == \ ]]; then + echo "$(get_machine_architecture)" + return 0 + fi + case "$architecture" in - \) - echo "$(get_normalized_architecture_from_architecture "$(get_machine_architecture)")" - return 0 - ;; amd64|x64) echo "x64" return 0 @@ -446,7 +447,7 @@ get_normalized_product() { # args: # version_text - stdin -get_version_from_version_info() { +get_version_from_latestversion_file_content() { eval $invocation cat | tail -n 1 | sed 's/\r$//' @@ -478,7 +479,7 @@ is_dotnet_package_installed() { # azure_feed - $1 # channel - $2 # normalized_architecture - $3 -get_latest_version_info() { +get_version_from_latestversion_file() { eval $invocation local azure_feed="$1" @@ -496,7 +497,7 @@ get_latest_version_info() { say_err "Invalid value for \$runtime" return 1 fi - say_verbose "get_latest_version_info: latest url: $version_file_url" + say_verbose "get_version_from_latestversion_file: latest url: $version_file_url" download "$version_file_url" || return $? return 0 @@ -504,7 +505,7 @@ get_latest_version_info() { # args: # json_file - $1 -parse_jsonfile_for_version() { +parse_globaljson_file_for_version() { eval $invocation local json_file="$1" @@ -560,9 +561,9 @@ get_specific_version_from_version() { if [ -z "$json_file" ]; then if [[ "$version" == "latest" ]]; then local version_info - version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 + version_info="$(get_version_from_latestversion_file "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 say_verbose "get_specific_version_from_version: version_info=$version_info" - echo "$version_info" | get_version_from_version_info + echo "$version_info" | get_version_from_latestversion_file_content return 0 else echo "$version" @@ -570,7 +571,7 @@ get_specific_version_from_version() { fi else local version_info - version_info="$(parse_jsonfile_for_version "$json_file")" || return 1 + version_info="$(parse_globaljson_file_for_version "$json_file")" || return 1 echo "$version_info" return 0 fi @@ -1119,6 +1120,33 @@ get_download_link_from_aka_ms() { fi } +get-feeds-to-use() +{ + feeds=( + "https://dotnetcli.azureedge.net/dotnet" + "https://dotnetbuilds.azureedge.net/public" + ) + + if [[ -n "$azure_feed" && "$azure_feed" != "" ]]; then + feeds=("$azure_feed") + fi + + if [[ "$no_cdn" == "true" ]]; then + feeds=( + "https://dotnetcli.blob.core.windows.net/dotnet", + "https://dotnetbuilds.blob.core.windows.net/public" + ) + + if [[ -n "$uncached_feed" && "$uncached_feed" != "" ]]; then + feeds=("$uncached_feed") + fi + fi +} + +generate_urls() { + +} + calculate_vars() { eval $invocation valid_legacy_download_link=true @@ -1500,8 +1528,12 @@ do echo " --dry-run,-DryRun Do not perform installation. Display download link." echo " --no-path, -NoPath Do not set PATH for the current process." echo " --verbose,-Verbose Display diagnostics information." - echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user." - echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user." + echo " --azure-feed,-AzureFeed For internal use only." + echo " Allows using a different storage to download SDK archives from." + echo " This parameter is only used if --no-cdn is false." + echo " --uncached-feed,-UncachedFeed For internal use only." + echo " Allows using a different storage to download SDK archives from." + echo " This parameter is only used if --no-cdn is true." echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." echo " -SkipNonVersionedFiles" echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." @@ -1509,14 +1541,6 @@ do echo " Note: global.json must have a value for 'SDK:Version'" echo " -?,--?,-h,--help,-Help Shows this help message" echo "" - echo "Obsolete parameters:" - echo " --shared-runtime The recommended alternative is '--runtime dotnet'." - echo " This parameter is obsolete and may be removed in a future version of this script." - echo " Installs just the shared runtime bits, not the entire SDK." - echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)." - echo " -RuntimeId" The parameter is obsolete and may be removed in a future version of this script. Should be used only for versions below 2.1. - echo " For primary links to override OS or/and architecture, use --os and --architecture option instead." - echo "" echo "Install Location:" echo " Location is chosen in following order:" echo " - --install-dir option" From fb1935371088adcae99a7dcb1b1d15d251698de6 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 3 Jan 2022 16:09:33 +0100 Subject: [PATCH 20/28] Determine installation root earlier --- src/dotnet-install.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 470221be16..3b80ba05ab 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -1162,6 +1162,8 @@ calculate_vars() { say_verbose "Normalized channel: '$normalized_channel'." normalized_product="$(get_normalized_product "$runtime")" say_verbose "Normalized product: '$normalized_product'." + install_root="$(resolve_installation_path "$install_dir")" + say_verbose "InstallRoot: '$install_root'." #try to get download location from aka.ms link #not applicable when exact version is specified via command or json file @@ -1197,9 +1199,6 @@ calculate_vars() { #Retrieve product specific version specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version" "$download_link")" say_verbose "Product specific version: '$specific_product_version'." - - install_root="$(resolve_installation_path "$install_dir")" - say_verbose "InstallRoot: '$install_root'." return fi fi @@ -1224,9 +1223,6 @@ calculate_vars() { else say_verbose "Cound not construct a legacy_download_link; omitting..." fi - - install_root="$(resolve_installation_path "$install_dir")" - say_verbose "InstallRoot: $install_root" } install_dotnet() { From 9370cd37bc02d4ca6554071604279688d1a69fba Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Thu, 6 Jan 2022 12:56:11 +0100 Subject: [PATCH 21/28] Try multiple feeds --- src/dotnet-install.sh | 371 +++++++++++++++++++++++------------------- 1 file changed, 207 insertions(+), 164 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 3b80ba05ab..bb173eeb78 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -426,6 +426,7 @@ get_normalized_channel() { get_normalized_product() { eval $invocation + local product="" local runtime="$(to_lowercase "$1")" if [[ "$runtime" == "dotnet" ]]; then product="dotnet-runtime" @@ -1067,7 +1068,7 @@ downloadwget() { return 0 } -get_download_link_from_aka_ms() { +get_download_link_from_aka_ms() { eval $invocation #quality is not supported for LTS or current channel @@ -1120,7 +1121,7 @@ get_download_link_from_aka_ms() { fi } -get-feeds-to-use() +get_feeds_to_use() { feeds=( "https://dotnetcli.azureedge.net/dotnet" @@ -1143,93 +1144,194 @@ get-feeds-to-use() fi } -generate_urls() { +generate_download_links() { -} + download_links=() + specific_versions=() + effective_versions=() + link_types=() -calculate_vars() { - eval $invocation - valid_legacy_download_link=true + # If generate_akams_links returns false, no fallback to old links. Just terminate. + generate_akams_links || return - #normalize input variables - normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" - say_verbose "Normalized architecture: '$normalized_architecture'." - normalized_os="$(get_normalized_os "$user_defined_os")" - say_verbose "Normalized OS: '$normalized_os'." - normalized_quality="$(get_normalized_quality "$quality")" - say_verbose "Normalized quality: '$normalized_quality'." - normalized_channel="$(get_normalized_channel "$channel")" - say_verbose "Normalized channel: '$normalized_channel'." - normalized_product="$(get_normalized_product "$runtime")" - say_verbose "Normalized product: '$normalized_product'." - install_root="$(resolve_installation_path "$install_dir")" - say_verbose "InstallRoot: '$install_root'." + # Check other feeds only if we haven't been able to find an aka.ms link. + if [[ "${#download_links[@]}" -lt 1 ]]; then + for feed in ${feeds[@]} + do + generate_regular_links $feed || return + done + fi + + for link in ${download_links[@]} + do + echo "Link: $link" + done +} + +# returns: +# 0 - if operation succeded and the execution should continue as normal +# 1 - if the script has reached a concluding state and the execution should stop. +generate_akams_links() { + local valid_aka_ms_link=true; - #try to get download location from aka.ms link - #not applicable when exact version is specified via command or json file normalized_version="$(to_lowercase "$version")" - if [[ -z "$json_file" && "$normalized_version" == "latest" ]]; then - - valid_aka_ms_link=true; - get_download_link_from_aka_ms || valid_aka_ms_link=false - - if [ "$valid_aka_ms_link" == false ]; then - # if quality is specified - exit with error - there is no fallback approach - if [ ! -z "$normalized_quality" ]; then - say_err "Failed to locate the latest version in the channel '$normalized_channel' with '$normalized_quality' quality for '$normalized_product', os: '$normalized_os', architecture: '$normalized_architecture'." - say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." - return 1 - fi - say_verbose "Falling back to latest.version file approach." - else - say_verbose "Retrieved primary payload URL from aka.ms link: '$aka_ms_download_link'." - download_link=$aka_ms_download_link - - say_verbose "Downloading using legacy url will not be attempted." - valid_legacy_download_link=false - - #get version from the path - IFS='/' - read -ra pathElems <<< "$download_link" - count=${#pathElems[@]} - specific_version="${pathElems[count-2]}" - unset IFS; - say_verbose "Version: '$specific_version'." - - #Retrieve product specific version - specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version" "$download_link")" - say_verbose "Product specific version: '$specific_product_version'." - return - fi + if [[ -n "$json_file" || "$normalized_version" != "latest" ]]; then + # aka.ms links are not needed when exact version is specified via command or json file + return fi - specific_version=$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file") || specific_version='0' + get_download_link_from_aka_ms || valid_aka_ms_link=false - if [[ "$specific_version" == '0' ]]; then - say_err "Could not resolve version information." + if [[ "$valid_aka_ms_link" == true ]]; then + say_verbose "Retrieved primary payload URL from aka.ms link: '$aka_ms_download_link'." + download_link=$aka_ms_download_link + + #get version from the path + IFS='/' + read -ra pathElems <<< "$download_link" + count=${#pathElems[@]} + specific_version="${pathElems[count-2]}" + unset IFS; + say_verbose "Version: '$specific_version'." + + #Retrieve effective version + effective_version="$(get_specific_product_version "$azure_feed" "$specific_version" "$download_link")" + + # Add link info to arrays + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+="aka.ms" + + if [[ "$dry_run" == true ]]; then + print_dry_run "$download_link" "" "$effective_version" + return 1 + fi + + # Check if the SDK version is already installed. + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "$asset_name with version '$effective_version' is already installed." + return 1 + fi + + return 0 + fi + + # if quality is specified - exit with error - there is no fallback approach + if [ ! -z "$normalized_quality" ]; then + say_err "Failed to locate the latest version in the channel '$normalized_channel' with '$normalized_quality' quality for '$normalized_product', os: '$normalized_os', architecture: '$normalized_architecture'." + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." return 1 fi + say_verbose "Falling back to latest.version file approach." +} + +# args: +# feed - $1 +# returns: +# 0 - if operation succeded and the execution should continue as normal +# 1 - if the script has reached a concluding state and the execution should stop +generate_regular_links() { + local feed="$1" + local valid_legacy_download_link=true + + specific_version=$(get_specific_version_from_version "$feed" "$channel" "$normalized_architecture" "$version" "$json_file") || specific_version='0' + + if [[ "$specific_version" == '0' ]]; then + say_verbose "Failed to resolve the specific version number using feed '$feed'" + return + fi - specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")" + effective_version="$(get_specific_product_version "$feed" "$specific_version")" say_verbose "specific_version=$specific_version" - download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version" "$normalized_os")" + download_link="$(construct_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version" "$normalized_os")" say_verbose "Constructed primary named payload URL: $download_link" - legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false + # Add link info to arrays + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+="primary" + + legacy_download_link="$(construct_legacy_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false if [ "$valid_legacy_download_link" = true ]; then say_verbose "Constructed legacy named payload URL: $legacy_download_link" + + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+="legacy" else + legacy_download_link="" say_verbose "Cound not construct a legacy_download_link; omitting..." fi + + if [[ "$dry_run" == true ]]; then + print_dry_run "$download_link" "$legacy_download_link" "$effective_version" + return 1 + fi + + # Check if the SDK version is already installed. + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "$asset_name with version '$effective_version' is already installed." + return 1 + fi } -install_dotnet() { +# args: +# download_link - $1 +# legacy_download_link - $2 - can be empty +# specific_version +print_dry_run() { + local download_link="$1" + local legacy_download_link="$2" + local specific_version="$3" + + say "Payload URLs:" + say "Primary named payload URL: ${download_link}" + if [ -n "$legacy_download_link" ]; then + say "Legacy named payload URL: ${legacy_download_link}" + fi + repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\"" --os "\""$normalized_os"\""" + + if [ ! -z "$normalized_quality" ]; then + repeatable_command+=" --quality "\""$normalized_quality"\""" + fi + + if [[ "$runtime" == "dotnet" ]]; then + repeatable_command+=" --runtime "\""dotnet"\""" + elif [[ "$runtime" == "aspnetcore" ]]; then + repeatable_command+=" --runtime "\""aspnetcore"\""" + fi + + repeatable_command+="$non_dynamic_parameters" + + if [ -n "$feed_credential" ]; then + repeatable_command+=" --feed-credential "\"""\""" + fi + + say "Repeatable invocation: $repeatable_command" + exit 0 +} + +calculate_vars() { eval $invocation - local download_failed=false - local asset_name='' - local asset_relative_path='' + + script_name=$(basename "$0") + normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" + say_verbose "Normalized architecture: '$normalized_architecture'." + normalized_os="$(get_normalized_os "$user_defined_os")" + say_verbose "Normalized OS: '$normalized_os'." + normalized_quality="$(get_normalized_quality "$quality")" + say_verbose "Normalized quality: '$normalized_quality'." + normalized_channel="$(get_normalized_channel "$channel")" + say_verbose "Normalized channel: '$normalized_channel'." + normalized_product="$(get_normalized_product "$runtime")" + say_verbose "Normalized product: '$normalized_product'." + install_root="$(resolve_installation_path "$install_dir")" + say_verbose "InstallRoot: '$install_root'." if [[ "$runtime" == "dotnet" ]]; then asset_relative_path="shared/Microsoft.NETCore.App" @@ -1240,84 +1342,51 @@ install_dotnet() { elif [ -z "$runtime" ]; then asset_relative_path="sdk" asset_name=".NET Core SDK" - else - say_err "Invalid value for \$runtime" - return 1 fi - # Check if the SDK version is already installed. - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then - say "$asset_name version $specific_version is already installed." - return 0 - fi + get_feeds_to_use +} + +install_dotnet() { + eval $invocation + local download_failed=false + local download_completed=false mkdir -p "$install_root" zip_path="$(mktemp "$temporary_file_template")" say_verbose "Zip path: $zip_path" + for link_index in "${!download_links[@]}" + do + download_link="${download_links[$link_index]}" + specific_version="${specific_versions[$link_index]}" + effective_version="${effective_versions[$link_index]}" + link_type="${link_types[$link_index]}" - # Failures are normal in the non-legacy case for ultimately legacy downloads. - # Do not output to stderr, since output to stderr is considered an error. - say "Downloading primary link $download_link" + say "Attempting to download using $link_type link $download_link" - # The download function will set variables $http_code and $download_error_msg in case of failure. - download "$download_link" "$zip_path" 2>&1 || download_failed=true + # The download function will set variables $http_code and $download_error_msg in case of failure. + download "$download_link" "$zip_path" 2>&1 || download_failed=true - # if the download fails, download the legacy_download_link - if [ "$download_failed" = true ]; then - primary_path_http_code="$http_code"; primary_path_download_error_msg="$download_error_msg" - case $primary_path_http_code in - 404) - say "The resource at $download_link is not available." - ;; - *) - say "$primary_path_download_error_msg" - ;; - esac - rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" - if [ "$valid_legacy_download_link" = true ]; then - download_failed=false - download_link="$legacy_download_link" - zip_path="$(mktemp "$temporary_file_template")" - say_verbose "Legacy zip path: $zip_path" - - say "Downloading legacy link $download_link" - - # The download function will set variables $http_code and $download_error_msg in case of failure. - download "$download_link" "$zip_path" 2>&1 || download_failed=true - - if [ "$download_failed" = true ]; then - legacy_path_http_code="$http_code"; legacy_path_download_error_msg="$download_error_msg" - case $legacy_path_http_code in - 404) - say "The resource at $download_link is not available." - ;; - *) - say "$legacy_path_download_error_msg" - ;; - esac - rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" - fi - fi - fi - - if [ "$download_failed" = true ]; then - if [[ "$primary_path_http_code" = "404" && ( "$valid_legacy_download_link" = false || "$legacy_path_http_code" = "404") ]]; then - say_err "Could not find \`$asset_name\` with version = $specific_version" - say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" + if [ "$download_failed" = true ]; then + case $http_code in + 404) + say "The resource at $link_type link '$download_link' is not available." + ;; + *) + say "Failed to download $link_type link '$download_link': $download_error_msg" + ;; + esac + rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" else - say_err "Could not download: \`$asset_name\` with version = $specific_version" - # 404-NotFound is an expected response if it goes from only one of the links, do not show that error. - # If primary path is available (not 404-NotFound) then show the primary error else show the legacy error. - if [ "$primary_path_http_code" != "404" ]; then - say_err "$primary_path_download_error_msg" - return 1 - fi - if [[ "$valid_legacy_download_link" = true && "$legacy_path_http_code" != "404" ]]; then - say_err "$legacy_path_download_error_msg" - return 1 - fi + download_completed=true + break fi + done + + if [[ "$download_completed" == false ]]; then + say_err "Could not find \`$asset_name\` with version = $specific_version" + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" return 1 fi @@ -1345,7 +1414,7 @@ install_dotnet() { # Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. say_err "Failed to verify the version of installed \`$asset_name\`.\nInstallation source: $download_link.\nInstallation location: $install_root.\nReport the bug at https://github.com/dotnet/install-scripts/issues." - say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error." + say_err "\`$asset_name\` with version = $specific_product_version failed to install with an error." return 1 } @@ -1363,8 +1432,8 @@ architecture="" dry_run=false no_path=false no_cdn=false -azure_feed="https://dotnetcli.azureedge.net/dotnet" -uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" +azure_feed="" +uncached_feed="" feed_credential="" verbose=false runtime="" @@ -1553,10 +1622,6 @@ do shift done -if [ "$no_cdn" = true ]; then - azure_feed="$uncached_feed" -fi - say "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" say "- The SDK needs to be installed without user interaction and without admin rights." say "- The SDK installation doesn't need to persist across multiple CI runs." @@ -1574,34 +1639,12 @@ fi check_min_reqs calculate_vars -script_name=$(basename "$0") +generate_download_links -if [ "$dry_run" = true ]; then - say "Payload URLs:" - say "Primary named payload URL: ${download_link}" - if [ "$valid_legacy_download_link" = true ]; then - say "Legacy named payload URL: ${legacy_download_link}" - fi - repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\"" --os "\""$normalized_os"\""" - - if [ ! -z "$normalized_quality" ]; then - repeatable_command+=" --quality "\""$normalized_quality"\""" - fi - - if [[ "$runtime" == "dotnet" ]]; then - repeatable_command+=" --runtime "\""dotnet"\""" - elif [[ "$runtime" == "aspnetcore" ]]; then - repeatable_command+=" --runtime "\""aspnetcore"\""" - fi - repeatable_command+="$non_dynamic_parameters" - - if [ -n "$feed_credential" ]; then - repeatable_command+=" --feed-credential "\"""\""" - fi - - say "Repeatable invocation: $repeatable_command" - exit 0 +if [ "$dry_run" = true ]; then + # Don't continue to installation step in dry_run mode. + return fi install_dotnet From 7d94d2df602550b06ea9efd41083fb4813f98c14 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Thu, 6 Jan 2022 18:34:23 +0100 Subject: [PATCH 22/28] Small fixes --- src/dotnet-install.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index bb173eeb78..9194c9af81 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -489,11 +489,11 @@ get_version_from_latestversion_file() { local version_file_url=null if [[ "$runtime" == "dotnet" ]]; then - version_file_url="$uncached_feed/Runtime/$channel/latest.version" + version_file_url="$azure_feed/Runtime/$channel/latest.version" elif [[ "$runtime" == "aspnetcore" ]]; then - version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version" + version_file_url="$azure_feed/aspnetcore/Runtime/$channel/latest.version" elif [ -z "$runtime" ]; then - version_file_url="$uncached_feed/Sdk/$channel/latest.version" + version_file_url="$azure_feed/Sdk/$channel/latest.version" else say_err "Invalid value for \$runtime" return 1 @@ -1201,7 +1201,7 @@ generate_akams_links() { download_links+=($download_link) specific_versions+=($specific_version) effective_versions+=($effective_version) - link_types+="aka.ms" + link_types+=("aka.ms") if [[ "$dry_run" == true ]]; then print_dry_run "$download_link" "" "$effective_version" @@ -1252,7 +1252,7 @@ generate_regular_links() { download_links+=($download_link) specific_versions+=($specific_version) effective_versions+=($effective_version) - link_types+="primary" + link_types+=("primary") legacy_download_link="$(construct_legacy_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false @@ -1262,7 +1262,7 @@ generate_regular_links() { download_links+=($download_link) specific_versions+=($specific_version) effective_versions+=($effective_version) - link_types+="legacy" + link_types+=("legacy") else legacy_download_link="" say_verbose "Cound not construct a legacy_download_link; omitting..." @@ -1366,6 +1366,7 @@ install_dotnet() { say "Attempting to download using $link_type link $download_link" # The download function will set variables $http_code and $download_error_msg in case of failure. + download_failed=false download "$download_link" "$zip_path" 2>&1 || download_failed=true if [ "$download_failed" = true ]; then @@ -1407,14 +1408,14 @@ install_dotnet() { fi # Check if the standard SDK version is installed. - say_verbose "Checking installation: version = $specific_product_version" - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then + say_verbose "Checking installation: version = $effective_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then return 0 fi # Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. say_err "Failed to verify the version of installed \`$asset_name\`.\nInstallation source: $download_link.\nInstallation location: $install_root.\nReport the bug at https://github.com/dotnet/install-scripts/issues." - say_err "\`$asset_name\` with version = $specific_product_version failed to install with an error." + say_err "\`$asset_name\` with version = $effective_version failed to install with an error." return 1 } From 1562795dc0e69653b7047109843061e8e5e335cc Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Thu, 6 Jan 2022 20:05:30 +0100 Subject: [PATCH 23/28] Failures during download shouldn't write to stderr Some download errors are expected. They dont signal a problem. --- src/dotnet-install.sh | 12 ++++++------ .../Utils/CommandResultAssertions.cs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 9194c9af81..91098c242b 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -1001,9 +1001,9 @@ downloadcurl() { local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " local failed=false if [ -z "$out_path" ]; then - curl $curl_options "$remote_path_with_credential" || failed=true + curl $curl_options "$remote_path_with_credential" 2>&1 || failed=true else - curl $curl_options -o "$out_path" "$remote_path_with_credential" || failed=true + curl $curl_options -o "$out_path" "$remote_path_with_credential" 2>&1 || failed=true fi if [ "$failed" = true ]; then local disable_feed_credential=false @@ -1035,20 +1035,20 @@ downloadwget() { local wget_result='' if [ -z "$out_path" ]; then - wget -q $wget_options $wget_options_extra -O - "$remote_path_with_credential" + wget -q $wget_options $wget_options_extra -O - "$remote_path_with_credential" 2>&1 wget_result=$? else - wget $wget_options $wget_options_extra -O "$out_path" "$remote_path_with_credential" + wget $wget_options $wget_options_extra -O "$out_path" "$remote_path_with_credential" 2>&1 wget_result=$? fi if [[ $wget_result == 2 ]]; then # Parsing of the command has failed. Exclude potentially unrecognized options and retry. if [ -z "$out_path" ]; then - wget -q $wget_options -O - "$remote_path_with_credential" + wget -q $wget_options -O - "$remote_path_with_credential" 2>&1 wget_result=$? else - wget $wget_options -O "$out_path" "$remote_path_with_credential" + wget $wget_options -O "$out_path" "$remote_path_with_credential" 2>&1 wget_result=$? fi fi diff --git a/tests/Install-Scripts.Test/Utils/CommandResultAssertions.cs b/tests/Install-Scripts.Test/Utils/CommandResultAssertions.cs index 221eee0791..45b54e2dc4 100644 --- a/tests/Install-Scripts.Test/Utils/CommandResultAssertions.cs +++ b/tests/Install-Scripts.Test/Utils/CommandResultAssertions.cs @@ -151,14 +151,14 @@ public AndConstraint HaveStdErrMatching(string pattern, public AndConstraint NotHaveStdOut() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWith(AppendDiagnosticsTo($"Expected command to not output to stdout but it was not:")); + .FailWith(AppendDiagnosticsTo($"Expected command to not output to stdout but it did:")); return new AndConstraint(this); } public AndConstraint NotHaveStdErr() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo("Expected command to not output to stderr but it was not:")); + .FailWith(AppendDiagnosticsTo("Expected command to not output to stderr but it did:")); return new AndConstraint(this); } From 0fe0fb30880ac101b167bbbc3bc1815fd40cbcda Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Fri, 7 Jan 2022 01:46:57 +0100 Subject: [PATCH 24/28] Bug and test fixes --- src/dotnet-install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 91098c242b..2ad8326d4e 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -1134,7 +1134,7 @@ get_feeds_to_use() if [[ "$no_cdn" == "true" ]]; then feeds=( - "https://dotnetcli.blob.core.windows.net/dotnet", + "https://dotnetcli.blob.core.windows.net/dotnet" "https://dotnetbuilds.blob.core.windows.net/public" ) @@ -1184,6 +1184,8 @@ generate_akams_links() { if [[ "$valid_aka_ms_link" == true ]]; then say_verbose "Retrieved primary payload URL from aka.ms link: '$aka_ms_download_link'." + say_verbose "Downloading using legacy url will not be attempted." + download_link=$aka_ms_download_link #get version from the path From 2d66ca54896d0bd9ef38c90b13066db1c725d801 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Fri, 7 Jan 2022 10:20:05 +0100 Subject: [PATCH 25/28] curl shouldn't output to std err --- src/dotnet-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 2ad8326d4e..c82b807f47 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -637,7 +637,7 @@ get_specific_product_version() { if machine_has "curl" then - specific_product_version=$(curl -s --fail "${download_link}${feed_credential}") + specific_product_version=$(curl -s --fail "${download_link}${feed_credential}" 2>&1) if [ $? = 0 ]; then echo "${specific_product_version//[$'\t\r\n']}" return 0 @@ -909,7 +909,7 @@ get_http_header_curl() { fi curl_options="-I -sSL --retry 5 --retry-delay 2 --connect-timeout 15 " - curl $curl_options "$remote_path_with_credential" || return 1 + curl $curl_options "$remote_path_with_credential" 2>&1 || return 1 return 0 } From e7a29da1f026442eff976313e0c2f8426c7d57cc Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Fri, 7 Jan 2022 15:28:08 +0100 Subject: [PATCH 26/28] Print generated links for debugging --- src/dotnet-install.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index c82b807f47..6c42a4266a 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -1162,9 +1162,15 @@ generate_download_links() { done fi - for link in ${download_links[@]} + if [[ "${#download_links[@]}" -eq 0 ]]; then + say_err "Failed to resolve the exact version number." + return 1 + fi + + say_verbose "Generated ${#download_links[@]} links." + for link_index in ${!download_links[@]} do - echo "Link: $link" + say_verbose "Link $link_index: ${link_types[$link_index]}, ${effective_versions[$link_index]}, ${download_links[$link_index]}" done } From 20f669d505bf335eb1554935a95736689b648ec3 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 10 Jan 2022 17:56:00 +0100 Subject: [PATCH 27/28] Added tests --- src/dotnet-install.sh | 6 +- ...ivenThatIWantToInstallDotnetFromAScript.cs | 143 +++++++++++++++++- 2 files changed, 145 insertions(+), 4 deletions(-) diff --git a/src/dotnet-install.sh b/src/dotnet-install.sh index 6c42a4266a..b80ff63934 100755 --- a/src/dotnet-install.sh +++ b/src/dotnet-install.sh @@ -1128,7 +1128,7 @@ get_feeds_to_use() "https://dotnetbuilds.azureedge.net/public" ) - if [[ -n "$azure_feed" && "$azure_feed" != "" ]]; then + if [[ -n "$azure_feed" ]]; then feeds=("$azure_feed") fi @@ -1138,7 +1138,7 @@ get_feeds_to_use() "https://dotnetbuilds.blob.core.windows.net/public" ) - if [[ -n "$uncached_feed" && "$uncached_feed" != "" ]]; then + if [[ -n "$uncached_feed" ]]; then feeds=("$uncached_feed") fi fi @@ -1175,7 +1175,7 @@ generate_download_links() { } # returns: -# 0 - if operation succeded and the execution should continue as normal +# 0 - if operation succeeded and the execution should continue as normal # 1 - if the script has reached a concluding state and the execution should stop. generate_akams_links() { local valid_aka_ms_link=true; diff --git a/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs b/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs index 6d5886fc39..2594329377 100644 --- a/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs +++ b/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs @@ -69,6 +69,7 @@ public class GivenThatIWantToInstallDotnetFromAScript : IDisposable ("6.0.1xx-preview2", "6\\.0\\.1.*", Quality.Daily | Quality.Signed), ("6.0.1xx-preview3", "6\\.0\\.1.*", Quality.Daily), ("6.0.1xx-preview4", "6\\.0\\.1.*", Quality.Daily), + ("7.0.1xx", "6\\.0\\..*", Quality.Daily), }; public static IEnumerable InstallSdkFromChannelTestCases @@ -373,6 +374,139 @@ public void WhenInstallingDotnetRuntimeWithFeedCredential(string channel, string commandResult.Should().NotHaveStdOutContainingIgnoreCase(feedCredential); } + [Theory] + [Trait("MonitoringTest", "true")] + [InlineData("5.0.404-servicing.21560.14")] + [InlineData("6.0.100-preview.6.21364.34")] + [InlineData("7.0.100-alpha.1.22054.9")] + public void WhenInstallingASpecificVersionOfTheSdk(string version) + { + // Run install script to download and install. + var args = GetInstallScriptArgs(channel:null, runtime: null, quality:null, _sdkInstallationDirectory, version: version); + + var commandResult = CreateInstallCommand(args) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + commandResult.Should().HaveStdOutContaining("Installation finished"); + + // Run dotnet to verify that the version is installed into correct folder. + var dotnetArgs = new List { "--info" }; + + var dotnetCommandResult = CreateDotnetCommand(_sdkInstallationDirectory, dotnetArgs) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + // On MacOS, installation directory has an extra /private at the beginning. + string installPathRegex = "\\[(/private)?" + Regex.Escape(Path.Combine(_sdkInstallationDirectory, "sdk")) + "\\]"; + string regex = Regex.Escape(" " + version + " ") + installPathRegex; + dotnetCommandResult.Should().HaveStdOutMatching(regex); + commandResult.Should().NotHaveStdErr(); + + TestOutputHelper.PopulateTestLoggerOutput(outputHelper, commandResult); + } + + [Theory] + [Trait("MonitoringTest", "true")] + [InlineData("5.0.13-servicing.21560.6", "5.0.13")] + [InlineData("6.0.0-preview.4.21176.7")] + [InlineData("7.0.0-alpha.1.21528.8")] + public void WhenInstallingASpecificVersionOfDotnetRuntime(string version, string? effectiveVersion = null) + { + // Run install script to download and install. + var args = GetInstallScriptArgs(channel: null, "dotnet", quality: null, _sdkInstallationDirectory, version: version); + + var commandResult = CreateInstallCommand(args) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + commandResult.Should().HaveStdOutContaining("Installation finished"); + + // Run dotnet to verify that the version is installed into correct folder. + var dotnetArgs = new List { "--info" }; + + var dotnetCommandResult = CreateDotnetCommand(_sdkInstallationDirectory, dotnetArgs) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + string lineStartRegex = Regex.Escape(" Microsoft.NETCore.App "); + string lineEndRegex = "\\ \\[(/private)?" + Regex.Escape(Path.Combine(_sdkInstallationDirectory, "shared", "Microsoft.NETCore.App")) + "\\]"; + string regex = lineStartRegex + Regex.Escape(effectiveVersion ?? version) + lineEndRegex; + dotnetCommandResult.Should().HaveStdOutMatching(regex); + commandResult.Should().NotHaveStdErr(); + + TestOutputHelper.PopulateTestLoggerOutput(outputHelper, commandResult); + } + + [Theory] + [Trait("MonitoringTest", "true")] + [InlineData("5.0.13-servicing.21552.32", "5.0.13")] + [InlineData("6.0.0-preview.4.21176.7")] + [InlineData("7.0.0-alpha.1.21528.8")] + public void WhenInstallingASpecificVersionOfAspNetCoreRuntime(string version, string? effectiveVersion = null) + { + // Run install script to download and install. + var args = GetInstallScriptArgs(channel: null, "aspnetcore", quality: null, _sdkInstallationDirectory, version: version); + + var commandResult = CreateInstallCommand(args) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + commandResult.Should().HaveStdOutContaining("Installation finished"); + + // Run dotnet to verify that the version is installed into correct folder. + var dotnetArgs = new List { "--info" }; + + var dotnetCommandResult = CreateDotnetCommand(_sdkInstallationDirectory, dotnetArgs) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + string lineStartRegex = Regex.Escape(" Microsoft.AspNetCore.App "); + string lineEndRegex = "\\ \\[(/private)?" + Regex.Escape(Path.Combine(_sdkInstallationDirectory, "shared", "Microsoft.AspNetCore.App")) + "\\]"; + string regex = lineStartRegex + Regex.Escape(effectiveVersion ?? version) + lineEndRegex; + dotnetCommandResult.Should().HaveStdOutMatching(regex); + commandResult.Should().NotHaveStdErr(); + + TestOutputHelper.PopulateTestLoggerOutput(outputHelper, commandResult); + } + + [Theory] + [Trait("MonitoringTest", "true")] + // productVersion files are broken prior to 6.0 release. + // [InlineData("5.0.14-servicing.21614.9")] + [InlineData("6.0.1-servicing.21568.2")] + [InlineData("7.0.0-alpha.1.21472.1")] + public void WhenInstallingASpecificVersionOfWindowsdesktopRuntime(string version) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // Don't install windowsdesktop if not on Windows. + return; + } + + // Run install script to download and install. + var args = GetInstallScriptArgs(channel: null, "windowsdesktop", quality: null, _sdkInstallationDirectory, version: version); + + var commandResult = CreateInstallCommand(args) + .CaptureStdOut() + .CaptureStdErr() + .Execute(); + + commandResult.Should().NotHaveStdErr(); + commandResult.Should().HaveStdOutContaining("Installation finished"); + + TestOutputHelper.PopulateTestLoggerOutput(outputHelper, commandResult); + + // Dotnet CLI is not included in the windowsdesktop runtime. Therefore, version validation cannot be tested. + // Add the validation once the becomes available in the artifacts. + } + [Theory] [InlineData(null, "2.4", "ga")] [InlineData(null, "3.9", null)] @@ -407,7 +541,8 @@ private static IEnumerable GetInstallScriptArgs( string? quality, string? installDir, string? feedCredentials = null, - bool verboseLogging = false) + bool verboseLogging = false, + string? version = null) { if (!string.IsNullOrWhiteSpace(channel)) { @@ -439,6 +574,12 @@ private static IEnumerable GetInstallScriptArgs( yield return feedCredentials; } + if (!string.IsNullOrWhiteSpace(version)) + { + yield return "-Version"; + yield return version; + } + if (verboseLogging) { yield return "-Verbose"; From 41e6d6b885df79c99684d3b916da27f255f387e4 Mon Sep 17 00:00:00 2001 From: Bekir Ozturk Date: Mon, 10 Jan 2022 18:29:03 +0100 Subject: [PATCH 28/28] Test Fixes --- .../GivenThatIWantToInstallDotnetFromAScript.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs b/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs index 2594329377..28312762fc 100644 --- a/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs +++ b/tests/Install-Scripts.Test/GivenThatIWantToInstallDotnetFromAScript.cs @@ -69,7 +69,7 @@ public class GivenThatIWantToInstallDotnetFromAScript : IDisposable ("6.0.1xx-preview2", "6\\.0\\.1.*", Quality.Daily | Quality.Signed), ("6.0.1xx-preview3", "6\\.0\\.1.*", Quality.Daily), ("6.0.1xx-preview4", "6\\.0\\.1.*", Quality.Daily), - ("7.0.1xx", "6\\.0\\..*", Quality.Daily), + ("7.0.1xx", "7\\.0\\..*", Quality.Daily), }; public static IEnumerable InstallSdkFromChannelTestCases @@ -376,10 +376,10 @@ public void WhenInstallingDotnetRuntimeWithFeedCredential(string channel, string [Theory] [Trait("MonitoringTest", "true")] - [InlineData("5.0.404-servicing.21560.14")] + [InlineData("5.0.404-servicing.21560.14", "5.0.404")] [InlineData("6.0.100-preview.6.21364.34")] [InlineData("7.0.100-alpha.1.22054.9")] - public void WhenInstallingASpecificVersionOfTheSdk(string version) + public void WhenInstallingASpecificVersionOfTheSdk(string version, string? effectiveVersion = null) { // Run install script to download and install. var args = GetInstallScriptArgs(channel:null, runtime: null, quality:null, _sdkInstallationDirectory, version: version); @@ -401,7 +401,7 @@ public void WhenInstallingASpecificVersionOfTheSdk(string version) // On MacOS, installation directory has an extra /private at the beginning. string installPathRegex = "\\[(/private)?" + Regex.Escape(Path.Combine(_sdkInstallationDirectory, "sdk")) + "\\]"; - string regex = Regex.Escape(" " + version + " ") + installPathRegex; + string regex = Regex.Escape(" " + (effectiveVersion ?? version) + " ") + installPathRegex; dotnetCommandResult.Should().HaveStdOutMatching(regex); commandResult.Should().NotHaveStdErr(); @@ -446,7 +446,7 @@ public void WhenInstallingASpecificVersionOfDotnetRuntime(string version, string [Trait("MonitoringTest", "true")] [InlineData("5.0.13-servicing.21552.32", "5.0.13")] [InlineData("6.0.0-preview.4.21176.7")] - [InlineData("7.0.0-alpha.1.21528.8")] + [InlineData("7.0.0-alpha.1.21567.15")] public void WhenInstallingASpecificVersionOfAspNetCoreRuntime(string version, string? effectiveVersion = null) { // Run install script to download and install.