diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 209f247241a6..544aa0fb2381 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -233,8 +233,11 @@ jobs: workspace: clean: all pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: + LC_ALL: 'en_US.UTF-8' + LANG: 'en_US.UTF-8' + LANGUAGE: 'en_US.UTF-8' PB_SKIPTESTS: 'true' steps: - checkout: self @@ -294,8 +297,11 @@ jobs: workspace: clean: all pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: + LC_ALL: 'en_US.UTF-8' + LANG: 'en_US.UTF-8' + LANGUAGE: 'en_US.UTF-8' PB_SKIPTESTS: 'true' steps: - checkout: self @@ -489,8 +495,11 @@ jobs: workspace: clean: all pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: + LC_ALL: 'en_US.UTF-8' + LANG: 'en_US.UTF-8' + LANGUAGE: 'en_US.UTF-8' PB_SKIPTESTS: 'true' steps: - checkout: self diff --git a/.azure/pipelines/e2e-tests.yml b/.azure/pipelines/e2e-tests.yml index 4838bd4ed416..43aa29e60102 100644 --- a/.azure/pipelines/e2e-tests.yml +++ b/.azure/pipelines/e2e-tests.yml @@ -85,7 +85,7 @@ jobs: testResultsFiles: 'artifacts/logs/**/*.trx' - job: Host_Linux pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 strategy: maxParallel: 8 matrix: @@ -113,6 +113,10 @@ jobs: SelfContainedMacOs_Node10: Test.RuntimeIdentifier: osx-x64 Node.Version: 10.x + variables: + LC_ALL: 'en_US.UTF-8' + LANG: 'en_US.UTF-8' + LANGUAGE: 'en_US.UTF-8' steps: - task: NodeTool@0 displayName: Install Node $(Node.Version) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 41f01d7afac3..5f4f4831932b 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -75,7 +75,7 @@ jobs: ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'MacOS')) }}: vmImage: macOS-10.14 ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Linux')) }}: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Windows')) }}: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: NetCorePublic-Pool @@ -89,6 +89,9 @@ jobs: BuildConfiguration: ${{ parameters.configuration }} BuildDirectory: ${{ parameters.buildDirectory }} BinlogArg: /bl:artifacts/logs/${{ parameters.agentOs }}.binlog + LC_ALL: 'en_US.UTF-8' + LANG: 'en_US.UTF-8' + LANGUAGE: 'en_US.UTF-8' ${{ if eq(parameters.agentOs, 'Windows') }}: JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk ${{ if or(ne(parameters.codeSign, 'true'), ne(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/build/SharedFxInstaller.targets b/build/SharedFxInstaller.targets index 8ae3dac2df3d..48b409f7e69d 100644 --- a/build/SharedFxInstaller.targets +++ b/build/SharedFxInstaller.targets @@ -229,7 +229,7 @@ - ubuntu.14.04 + ubuntu.18.04 @(_DebSharedFxDependencies->'"%(Identity)": { "package_version": "%(Version)" }', ', ') diff --git a/build/dependencies.props b/build/dependencies.props index 21b767b43435..0787336ce8a1 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -2,8 +2,8 @@ - 2.1.23 - 2.1.23 + 2.1.26 + 2.1.26 diff --git a/build/docker/alpine.Dockerfile b/build/docker/alpine.Dockerfile index ae9cbab56fba..66f5d1580994 100644 --- a/build/docker/alpine.Dockerfile +++ b/build/docker/alpine.Dockerfile @@ -1,4 +1,4 @@ -FROM microsoft/dotnet:2.1.0-preview1-runtime-deps-alpine +FROM mcr.microsoft.com/dotnet/runtime-deps:2.1-alpine ARG USER ARG USER_ID ARG GROUP_ID @@ -18,10 +18,14 @@ RUN apk add --no-cache \ USER $USER_ID:$GROUP_ID +# Use a location for .dotnet/ that's not under repo root. +ENV DOTNET_HOME /code/.dotnet + # Disable the invariant mode (set in base image) ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT false ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 # Skip package initilization ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 diff --git a/build/tools/docker/ubuntu.14.04/Dockerfile b/build/tools/docker/ubuntu.18.04/Dockerfile similarity index 78% rename from build/tools/docker/ubuntu.14.04/Dockerfile rename to build/tools/docker/ubuntu.18.04/Dockerfile index f039506971dc..5c8fc714cb76 100644 --- a/build/tools/docker/ubuntu.14.04/Dockerfile +++ b/build/tools/docker/ubuntu.18.04/Dockerfile @@ -4,7 +4,7 @@ # # Dockerfile that creates a container suitable to build dotnet-cli -FROM ubuntu:14.04 +FROM ubuntu:18.04 # Misc Dependencies for build RUN apt-get update && \ @@ -15,9 +15,10 @@ RUN apt-get update && \ sudo \ libunwind8 \ libkrb5-3 \ - libicu52 \ + libicu60 \ liblttng-ust0 \ libssl1.0.0 \ + locales \ zlib1g \ libuuid1 \ debhelper \ @@ -25,21 +26,27 @@ RUN apt-get update && \ devscripts \ git \ cmake \ - clang-3.5 \ - lldb-3.6 \ + clang-3.9 \ + lldb-3.9 \ wget && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* && \ + locale-gen "en_US.UTF-8" # Use clang as c++ compiler -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 -RUN update-alternatives --set c++ /usr/bin/clang++-3.5 +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.9 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.9 # Setup User to match Host User, and give superuser permissions ARG USER_ID=0 RUN useradd -m code_executor -u ${USER_ID} -g sudo RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +# Preset well-known locale +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + # With the User Change, we need to change permissions on these directories RUN chmod -R a+rwx /usr/local RUN chmod -R a+rwx /home diff --git a/dockerbuild.sh b/dockerbuild.sh index 20df1a5c396e..6ba130fa4494 100755 --- a/dockerbuild.sh +++ b/dockerbuild.sh @@ -90,6 +90,9 @@ fi dockerfile="$DIR/build/docker/$image.Dockerfile" tagname="aspnetcore-build-$image" +# Use a location for .dotnet/ that's not under repo root in the container and don't reuse host's folder. +mkdir "$(dirname $DIR)/.dotnet-$image" + docker build "$(dirname "$dockerfile")" \ --build-arg "USER=$(whoami)" \ --build-arg "USER_ID=$(id -u)" \ @@ -114,6 +117,7 @@ docker run \ -e PB_ASSETROOTURL \ -e PRODUCTBUILDID \ -v "$DIR:/code/build" \ + -v "$(dirname $DIR)/.dotnet-$image:/code/.dotnet" \ ${docker_args[@]+"${docker_args[@]}"} \ $tagname \ ./build.sh \ diff --git a/eng/PatchConfig.props b/eng/PatchConfig.props index 04811d69264e..659f8439a502 100644 --- a/eng/PatchConfig.props +++ b/eng/PatchConfig.props @@ -100,4 +100,8 @@ Later on, this will be checked using this condition: @aspnet/signalr-protocol-msgpack; + + + + diff --git a/modules/EntityFrameworkCore b/modules/EntityFrameworkCore index 68dec9469e94..1892d878c050 160000 --- a/modules/EntityFrameworkCore +++ b/modules/EntityFrameworkCore @@ -1 +1 @@ -Subproject commit 68dec9469e94a05fe8fe01ed3faba76914254c47 +Subproject commit 1892d878c050ee4d74b8473907a26c8756f6b89c diff --git a/src/Identity/test/Identity.Test/CdnScriptTaghelperTests.cs b/src/Identity/test/Identity.Test/CdnScriptTaghelperTests.cs index a2509d52a1cb..6cbf76cca851 100644 --- a/src/Identity/test/Identity.Test/CdnScriptTaghelperTests.cs +++ b/src/Identity/test/Identity.Test/CdnScriptTaghelperTests.cs @@ -9,6 +9,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing.xunit; using Xunit; using Xunit.Abstractions; @@ -23,7 +24,9 @@ public CdnScriptTagTests(ITestOutputHelper output) _output = output; } - [Fact] + // Because this test runs on .NET Core and seems reliable there, likely not worth running on .NET Framework. + [ConditionalFact] + [FrameworkSkipCondition(RuntimeFrameworks.CLR, SkipReason = "At least flaky on .NET Framework.")] public async Task IdentityUI_ScriptTags_SubresourceIntegrityCheck() { var slnDir = GetSolutionDir(); @@ -39,7 +42,7 @@ public async Task IdentityUI_ScriptTags_SubresourceIntegrityCheck() Assert.NotEmpty(scriptTags); var shasum = new Dictionary(StringComparer.OrdinalIgnoreCase); - using (var client = new HttpClient(new RetryHandler(new HttpClientHandler() { }))) + using (var client = new HttpClient(new RetryHandler(new HttpClientHandler(), _output))) { foreach (var script in scriptTags) { @@ -63,22 +66,52 @@ public async Task IdentityUI_ScriptTags_SubresourceIntegrityCheck() }); } - class RetryHandler : DelegatingHandler + private class RetryHandler : DelegatingHandler { - public RetryHandler(HttpMessageHandler innerHandler) : base(innerHandler) { } + private readonly ITestOutputHelper _output; + + public RetryHandler(HttpMessageHandler innerHandler, ITestOutputHelper output) : base(innerHandler) + { + _output = output; + } + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { HttpResponseMessage result = null; - for (var i = 0; i < 10; i++) + var method = request.Method; + var url = request.RequestUri; + var waitIntervalBeforeRetry = 1; + + // Try 6 times with 1, 2, 4, 8, 16 seconds between attempts. Last attempt may throw or report + // error to caller. + for (var i = 0; i < 5; i++) { - result = await base.SendAsync(request, cancellationToken); - if (result.IsSuccessStatusCode) + try { - return result; + _output.WriteLine($"Sending request '{method} - {url}' {i+1} attempt."); + result = await base.SendAsync(request, cancellationToken); + if (result.IsSuccessStatusCode) + { + return result; + } + else + { + _output.WriteLine($"Request '{method} - {url}' failed with {result.StatusCode}."); + } + } + catch (Exception e) + { + _output.WriteLine($"Request '{method} - {url}' failed with {e.ToString()}"); + } + finally + { + await Task.Delay(TimeSpan.FromSeconds(waitIntervalBeforeRetry), cancellationToken); + waitIntervalBeforeRetry = waitIntervalBeforeRetry * 2; } - await Task.Delay(1000); } - return result; + + // Try one last time to show the actual error. + return await base.SendAsync(request, cancellationToken); } } diff --git a/src/Mvc/build/Key.snk b/src/Mvc/build/Key.snk deleted file mode 100644 index e10e4889c125..000000000000 Binary files a/src/Mvc/build/Key.snk and /dev/null differ diff --git a/src/Mvc/build/buildpipeline/linux.groovy b/src/Mvc/build/buildpipeline/linux.groovy deleted file mode 100644 index 903f218bb817..000000000000 --- a/src/Mvc/build/buildpipeline/linux.groovy +++ /dev/null @@ -1,10 +0,0 @@ -@Library('dotnet-ci') _ - -simpleNode('Ubuntu16.04', 'latest-or-auto-docker') { - stage ('Checking out source') { - checkout scm - } - stage ('Build') { - sh './build.sh --ci' - } -} diff --git a/src/Mvc/build/buildpipeline/osx.groovy b/src/Mvc/build/buildpipeline/osx.groovy deleted file mode 100644 index aaac63686b53..000000000000 --- a/src/Mvc/build/buildpipeline/osx.groovy +++ /dev/null @@ -1,10 +0,0 @@ -@Library('dotnet-ci') _ - -simpleNode('OSX10.12','latest') { - stage ('Checking out source') { - checkout scm - } - stage ('Build') { - sh './build.sh --ci' - } -} diff --git a/src/Mvc/build/buildpipeline/pipeline.groovy b/src/Mvc/build/buildpipeline/pipeline.groovy deleted file mode 100644 index e915cadae124..000000000000 --- a/src/Mvc/build/buildpipeline/pipeline.groovy +++ /dev/null @@ -1,18 +0,0 @@ -import org.dotnet.ci.pipelines.Pipeline - -def windowsPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows.groovy') -def linuxPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/linux.groovy') -def osxPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/osx.groovy') -String configuration = 'Release' -def parameters = [ - 'Configuration': configuration -] - -windowsPipeline.triggerPipelineOnEveryGithubPR("Windows ${configuration} x64 Build", parameters) -windowsPipeline.triggerPipelineOnGithubPush(parameters) - -linuxPipeline.triggerPipelineOnEveryGithubPR("Ubuntu 16.04 ${configuration} Build", parameters) -linuxPipeline.triggerPipelineOnGithubPush(parameters) - -osxPipeline.triggerPipelineOnEveryGithubPR("OSX 10.12 ${configuration} Build", parameters) -osxPipeline.triggerPipelineOnGithubPush(parameters) diff --git a/src/Mvc/build/buildpipeline/windows.groovy b/src/Mvc/build/buildpipeline/windows.groovy deleted file mode 100644 index 8d26f313d497..000000000000 --- a/src/Mvc/build/buildpipeline/windows.groovy +++ /dev/null @@ -1,12 +0,0 @@ -@Library('dotnet-ci') _ - -// 'node' indicates to Jenkins that the enclosed block runs on a node that matches -// the label 'windows-with-vs' -simpleNode('Windows_NT','latest') { - stage ('Checking out source') { - checkout scm - } - stage ('Build') { - bat '.\\run.cmd -CI default-build' - } -} diff --git a/src/PackageArchive/Archive.targets b/src/PackageArchive/Archive.targets index 9775f7414898..c331251f4a8f 100644 --- a/src/PackageArchive/Archive.targets +++ b/src/PackageArchive/Archive.targets @@ -84,6 +84,8 @@ $(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion).$([MSBuild]::Subtract($(AspNetCorePatchVersion), 1)) + $(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion).$([MSBuild]::Subtract($(AspNetCorePatchVersion), 2)) SubresourceIntegrityCheckData @@ -96,6 +97,55 @@ public async Task FallbackSrcContent_Matches_CDNContent(ScriptTag scriptTag) Assert.Equal(RemoveLineEndings(cdnContent), RemoveLineEndings(fallbackSrcContent)); } + private class RetryHandler : DelegatingHandler + { + private readonly ITestOutputHelper _output; + + public RetryHandler(HttpMessageHandler innerHandler, ITestOutputHelper output) : base(innerHandler) + { + _output = output; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + HttpResponseMessage result = null; + var method = request.Method; + var url = request.RequestUri; + var waitIntervalBeforeRetry = 1; + + // Try 6 times with 1, 2, 4, 8, 16 seconds between attempts. Last attempt may throw or report + // error to caller. + for (var i = 0; i < 5; i++) + { + try + { + _output.WriteLine($"Sending request '{method} - {url}' {i+1} attempt."); + result = await base.SendAsync(request, cancellationToken); + if (result.IsSuccessStatusCode) + { + return result; + } + else + { + _output.WriteLine($"Request '{method} - {url}' failed with {result.StatusCode}."); + } + } + catch (Exception e) + { + _output.WriteLine($"Request '{method} - {url}' failed with {e.ToString()}"); + } + finally + { + await Task.Delay(TimeSpan.FromSeconds(waitIntervalBeforeRetry), cancellationToken); + waitIntervalBeforeRetry = waitIntervalBeforeRetry * 2; + } + } + + // Try one last time to show the actual error. + return await base.SendAsync(request, cancellationToken); + } + } + public struct ScriptTag { public string Src; diff --git a/test/SharedFx.UnitTests/RetryHelper.cs b/test/SharedFx.UnitTests/RetryHelper.cs index 2d92113e3656..8d3400fc06c9 100644 --- a/test/SharedFx.UnitTests/RetryHelper.cs +++ b/test/SharedFx.UnitTests/RetryHelper.cs @@ -14,13 +14,6 @@ internal static class RetryHelpers /// private const int MaxRetryMinutes = 15; - private static int TotalRetriesUsed; - - public static int GetTotalRetriesUsed() - { - return TotalRetriesUsed; - } - public static async Task RetryAsync(Func action, IReporter reporter) { await RetryAsync( @@ -36,7 +29,7 @@ public static async Task RetryAsync(Func> action, IReporter report { Exception firstException = null; - var retriesRemaining = 10; + var retriesRemaining = 5; var retryDelayInMinutes = 1; while (retriesRemaining > 0) @@ -49,14 +42,15 @@ public static async Task RetryAsync(Func> action, IReporter report { firstException = firstException ?? e; reporter.Output($"Exception thrown! {e.Message}"); + + retriesRemaining--; reporter.Output($"Waiting {retryDelayInMinutes} minute(s) to retry ({retriesRemaining} left)..."); + await Task.Delay(retryDelayInMinutes * 60 * 1000); - // Do exponential back-off, but limit it (1, 2, 4, 8, 15, 15, 15, ...) - // With MaxRetryMinutes=15 and MaxRetries=10, this will delay a maximum of 105 minutes + // Do exponential back-off, but limit it (1, 2, 4, 8, 15) + // With MaxRetryMinutes=15 and MaxRetries=5, this will delay a maximum of 30 minutes (1/3 job timeout). retryDelayInMinutes = Math.Min(2 * retryDelayInMinutes, MaxRetryMinutes); - retriesRemaining--; - TotalRetriesUsed++; } } throw new InvalidOperationException("Max exception retries reached, giving up.", firstException); diff --git a/test/SharedFx.UnitTests/SharedFx.UnitTests.csproj b/test/SharedFx.UnitTests/SharedFx.UnitTests.csproj index e439e2842bc9..9366f526e04e 100644 --- a/test/SharedFx.UnitTests/SharedFx.UnitTests.csproj +++ b/test/SharedFx.UnitTests/SharedFx.UnitTests.csproj @@ -30,14 +30,14 @@ <_Parameter1>PreviousAspNetCoreReleaseVersion <_Parameter2>$(PreviousAspNetCoreReleaseVersion) + + <_Parameter1>ValidateBaseline + <_Parameter2>$(ValidateBaseline) + - - - - - + diff --git a/test/SharedFx.UnitTests/SharedFxTests.cs b/test/SharedFx.UnitTests/SharedFxTests.cs index de57b07f7055..0319997f97e5 100644 --- a/test/SharedFx.UnitTests/SharedFxTests.cs +++ b/test/SharedFx.UnitTests/SharedFxTests.cs @@ -17,11 +17,16 @@ namespace Microsoft.AspNetCore { public class SharedFxTests { - [Theory] [MemberData(nameof(GetSharedFxConfig))] public async Task BaselineTest(SharedFxConfig config) { + if (!TestData.GetValidateBaseline()) + { + // Inability to validate the package baselines indicates dotnetcli is not up-to-date. + return; + } + var previousVersion = TestData.GetPreviousAspNetCoreReleaseVersion(); var url = new Uri($"https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/" + previousVersion + "/aspnetcore-runtime-internal-" + previousVersion + "-win-x64.zip"); var zipName = "assemblies.zip"; diff --git a/test/SharedFx.UnitTests/TestData.cs b/test/SharedFx.UnitTests/TestData.cs index dd024ae3bdc8..8001e1b38f58 100644 --- a/test/SharedFx.UnitTests/TestData.cs +++ b/test/SharedFx.UnitTests/TestData.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Linq; using System.Reflection; @@ -20,6 +21,9 @@ public class TestData public static string GetSharedFxRuntimeIdentifier() => GetTestDataValue("SharedFxRuntimeIdentifier"); + public static bool GetValidateBaseline() => + string.Equals(GetTestDataValue("ValidateBaseline"), "true", StringComparison.OrdinalIgnoreCase); + private static string GetTestDataValue(string key) => typeof(TestData).Assembly.GetCustomAttributes().Single(d => d.Key == key).Value; } diff --git a/version.props b/version.props index 0d65929671e4..fd255dd72ee9 100644 --- a/version.props +++ b/version.props @@ -2,8 +2,8 @@ 2 1 - 27 - true + 28 + false servicing Servicing