diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index bbf6b586670b..92b8d9aae5ee 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -621,17 +621,18 @@ stages: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log + - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) displayName: Build shared fx - - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -noBuildNative -projects src/Grpc/**/*.csproj + - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -noBuildNative -projects src/Grpc/**/*.csproj $(_InternalRuntimeDownloadArgs) displayName: Restore interop projects - script: ./build.cmd -ci -nobl -noBuildRepoTasks -noRestore -test -all -noBuildNative -projects eng\helix\helix.proj - /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true + /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true $(_InternalRuntimeDownloadArgs) /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + artifacts: - name: Helix_logs path: artifacts/log/ @@ -665,7 +666,7 @@ stages: arguments: $(Build.SourcesDirectory)/NuGet.config $Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: ./eng/scripts/ci-source-build.sh --ci --nobl --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false + - script: ./eng/scripts/ci-source-build.sh --ci --nobl --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false $(_InternalRuntimeDownloadArgs) displayName: Run ci-source-build.sh - task: PublishBuildArtifacts@1 displayName: Upload logs diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 1bf215525b34..daca90c0b63b 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -240,8 +240,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) { return $installScript } -function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') { - InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey +function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) { + InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath } function InstallDotNet([string] $dotnetRoot, @@ -250,7 +250,8 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtime = '', [bool] $skipNonVersionedFiles = $false, [string] $runtimeSourceFeed = '', - [string] $runtimeSourceFeedKey = '') { + [string] $runtimeSourceFeedKey = '', + [switch] $noPath) { $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ @@ -261,6 +262,7 @@ function InstallDotNet([string] $dotnetRoot, if ($architecture) { $installParameters.Architecture = $architecture } if ($runtime) { $installParameters.Runtime = $runtime } if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + if ($noPath) { $installParameters.NoPath = $True } try { & $installScript @installParameters diff --git a/eng/helix/content/InstallDotNet.ps1 b/eng/helix/content/InstallDotNet.ps1 deleted file mode 100644 index 778e24dbd24e..000000000000 --- a/eng/helix/content/InstallDotNet.ps1 +++ /dev/null @@ -1,35 +0,0 @@ - <# - .SYNOPSIS - Installs dotnet sdk and runtime using https://dot.net/v1/dotnet-install.ps1 - .DESCRIPTION - Installs dotnet sdk and runtime using https://dot.net/v1/dotnet-install.ps1 -.PARAMETER arch - The architecture to install. -.PARAMETER sdkVersion - The sdk version to install -.PARAMETER runtimeVersion - The runtime version to install -.PARAMETER installDir - The directory to install to -#> -param( - [Parameter(Mandatory = $true)] - $arch, - - [Parameter(Mandatory = $true)] - $sdkVersion, - - [Parameter(Mandatory = $true)] - $runtimeVersion, - - [Parameter(Mandatory = $true)] - $installDir -) - -& $PSScriptRoot\Download.ps1 "https://dot.net/v1/dotnet-install.ps1" $PSScriptRoot\dotnet-install.ps1 -Write-Host "Download of dotnet-install.ps1 complete..." -Write-Host "Installing SDK...& $PSScriptRoot\dotnet-install.ps1 -Architecture $arch -Version $sdkVersion -InstallDir $installDir -NoPath" -Invoke-Expression "& $PSScriptRoot\dotnet-install.ps1 -Architecture $arch -Version $sdkVersion -InstallDir $installDir -NoPath" -Write-Host "Installing Runtime...& $PSScriptRoot\dotnet-install.ps1 -Architecture $arch -Runtime dotnet -Version $runtimeVersion -InstallDir $installDir -NoPath" -Invoke-Expression "& $PSScriptRoot\dotnet-install.ps1 -Architecture $arch -Runtime dotnet -Version $runtimeVersion -InstallDir $installDir -NoPath" -Write-Host "InstallDotNet.ps1 complete..." diff --git a/eng/helix/content/runtests.cmd b/eng/helix/content/runtests.cmd index af984dd88274..4dca87d067d3 100644 --- a/eng/helix/content/runtests.cmd +++ b/eng/helix/content/runtests.cmd @@ -17,6 +17,8 @@ shift set $aspnetref=%9 shift set $helixTimeout=%9 +shift +set $feedCred=%9 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk set DOTNET_ROOT=%DOTNET_HOME%\%$arch% @@ -26,10 +28,16 @@ set DOTNET_CLI_HOME=%HELIX_CORRELATION_PAYLOAD%\home set PATH=%DOTNET_ROOT%;!PATH!;%HELIX_CORRELATION_PAYLOAD%\node\bin echo Set path to: %PATH% -echo "Invoking InstallDotNet.ps1 %$arch% %$sdkVersion% %$runtimeVersion% %DOTNET_ROOT%" -powershell.exe -NoProfile -ExecutionPolicy unrestricted -file InstallDotNet.ps1 %$arch% %$sdkVersion% %$runtimeVersion% %DOTNET_ROOT% + +powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true '' '' $true" +IF [%$feedCred%] == [] ( + powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true '' '' $true" +) else ( + powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true https://dotnetclimsrc.blob.core.windows.net/dotnet %$feedCred% $true" +) set exit_code=0 + echo "Restore: dotnet restore RunTests\RunTests.csproj --source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json --source https://api.nuget.org/v3/index.json --ignore-failed-sources..." dotnet restore RunTests\RunTests.csproj --source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json --source https://api.nuget.org/v3/index.json --ignore-failed-sources diff --git a/eng/helix/content/runtests.sh b/eng/helix/content/runtests.sh index 79f0edb53503..ecc58322fe5a 100644 --- a/eng/helix/content/runtests.sh +++ b/eng/helix/content/runtests.sh @@ -30,49 +30,24 @@ RED="\033[0;31m" YELLOW="\033[0;33m" MAGENTA="\033[0;95m" -curl -o dotnet-install.sh -sSL https://dot.net/v1/dotnet-install.sh -if [ $? -ne 0 ]; then - download_retries=3 - while [ $download_retries -gt 0 ]; do - curl -o dotnet-install.sh -sSL https://dot.net/v1/dotnet-install.sh - if [ $? -ne 0 ]; then - let download_retries=download_retries-1 - echo -e "${YELLOW}Failed to download dotnet-install.sh. Retries left: $download_retries.${RESET}" - else - download_retries=0 - fi - done -fi - -# Call "sync" between "chmod" and execution to prevent "text file busy" error in Docker (aufs) -chmod +x "dotnet-install.sh"; sync - -./dotnet-install.sh --version $dotnet_sdk_version --install-dir "$DOTNET_ROOT" -if [ $? -ne 0 ]; then - sdk_retries=3 - while [ $sdk_retries -gt 0 ]; do - ./dotnet-install.sh --version $dotnet_sdk_version --install-dir "$DOTNET_ROOT" - if [ $? -ne 0 ]; then - let sdk_retries=sdk_retries-1 - echo -e "${YELLOW}Failed to install .NET Core SDK $version. Retries left: $sdk_retries.${RESET}" - else - sdk_retries=0 - fi - done -fi - -./dotnet-install.sh --runtime dotnet --version $dotnet_runtime_version --install-dir "$DOTNET_ROOT" -if [ $? -ne 0 ]; then - runtime_retries=3 - while [ $runtime_retries -gt 0 ]; do - ./dotnet-install.sh --runtime dotnet --version $dotnet_runtime_version --install-dir "$DOTNET_ROOT" - if [ $? -ne 0 ]; then - let runtime_retries=runtime_retries-1 - echo -e "${YELLOW}Failed to install .NET Core runtime $version. Retries left: $runtime_retries.${RESET}" - else - runtime_retries=0 - fi - done +. eng/common/tools.sh +InstallDotNet $DOTNET_ROOT $dotnet_sdk_version "" "" true || { + exit_code=$? + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} +if [[ -z "${11:-}" ]]; then + InstallDotNet $DOTNET_ROOT $dotnet_runtime_version "" dotnet true || { + exit_code=$? + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code + } +else + InstallDotNet $DOTNET_ROOT $dotnet_runtime_version "" dotnet true https://dotnetclimsrc.blob.core.windows.net/dotnet ${11} || { + exit_code=$? + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code + } fi if [ -e /proc/self/coredump_filter ]; then @@ -82,7 +57,7 @@ if [ -e /proc/self/coredump_filter ]; then echo -n 0x3F > /proc/self/coredump_filter fi -# dontet-install.sh seems to affect the Linux filesystem and causes test flakiness unless we sync the filesystem before running tests +# dotnet-install.sh seems to affect the Linux filesystem and causes test flakiness unless we sync the filesystem before running tests sync exit_code=0 diff --git a/eng/scripts/ci-source-build.sh b/eng/scripts/ci-source-build.sh index 468f74975193..d599fced8c98 100755 --- a/eng/scripts/ci-source-build.sh +++ b/eng/scripts/ci-source-build.sh @@ -33,10 +33,76 @@ reporoot="$(dirname "$(dirname "$scriptroot")")" # mv "$reporoot/global.bak.json" "$reporoot/global.json" #}" EXIT +dotnet_runtime_source_feed='' +dotnet_runtime_source_feed_key='' +other_args=() + +# +# Functions +# +__usage() { + echo "Usage: $(basename "${BASH_SOURCE[0]}") [options] [[--] ...] + +Arguments: + ... Arguments passed to the command. Variable number of arguments allowed. + + --dotnet-runtime-source-feed Additional feed that can be used when downloading .NET runtimes + --dotnet-runtime-source-feed-key Key for feed that can be used when downloading .NET runtimes + +Description: + This script is meant for testing source build by imitating some of the input parameters and conditions. +" + + if [[ "${1:-}" != '--no-exit' ]]; then + exit 2 + fi +} + +__error() { + echo -e "${RED}error: $*${RESET}" 1>&2 +} + +# +# main +# + +while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -\?|-h|-help) + __usage --no-exit + exit 0 + ;; + -dotnet-runtime-source-feed|-dotnetruntimesourcefeed) + shift + [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed" && __usage + dotnet_runtime_source_feed="${1:-}" + ;; + -dotnet-runtime-source-feed-key|-dotnetruntimesourcefeedkey) + shift + [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed-key" && __usage + dotnet_runtime_source_feed_key="${1:-}" + ;; + *) + other_args[${#other_args[*]}]="$1" + ;; + esac + shift +done + +# Set up additional runtime args +runtime_feed_args=() +if [ ! -z "$dotnet_runtime_source_feed$dotnet_runtime_source_feed_key" ]; then + runtimeFeedArg="/p:DotNetRuntimeSourceFeed=$dotnet_runtime_source_feed" + runtimeFeedKeyArg="/p:DotNetRuntimeSourceFeedKey=$dotnet_runtime_source_feed_key" + runtime_feed_args[${#runtime_feed_args[*]}]=$runtimeFeedArg + runtime_feed_args[${#runtime_feed_args[*]}]=$runtimeFeedKeyArg +fi + # Build repo tasks -"$reporoot/eng/common/build.sh" --restore --build --ci --configuration Release /p:ProjectToBuild=$reporoot/eng/tools/RepoTasks/RepoTasks.csproj +"$reporoot/eng/common/build.sh" --restore --build --ci --configuration Release /p:ProjectToBuild=$reporoot/eng/tools/RepoTasks/RepoTasks.csproj ${runtime_feed_args[@]+"${runtime_feed_args[@]}"} export DotNetBuildFromSource='true' # Build projects -"$reporoot/eng/common/build.sh" --restore --build --pack "$@" \ No newline at end of file +"$reporoot/eng/common/build.sh" --restore --build --pack ${other_args[@]+"${other_args[@]}"} ${runtime_feed_args[@]+"${runtime_feed_args[@]}"} diff --git a/eng/targets/Helix.targets b/eng/targets/Helix.targets index 24baa2b27652..706c407778f2 100644 --- a/eng/targets/Helix.targets +++ b/eng/targets/Helix.targets @@ -24,6 +24,20 @@ + + + + eng\common\%(Filename)%(Extension) + PreserveNewest + PreserveNewest + + + eng\common\dotnet-install-scripts\%(Filename)%(Extension) + PreserveNewest + PreserveNewest +