Playwright 🎭 🛠️ (#45682)

* Version Bumps

* Run with `Test: Windows Server x64`

* Install Playwright in CI

* Add running playwright locally instructions

* Test Cleanup

* Remove Selenium

* Prebuild for Playwright install

* Update default-build.yml

* Update default-build.yml

* Update default-build.yml

* Update default-build.yml

* Update default-build.yml

* Playwright install in test

* New template test pipeline

* Update components-e2e-tests.yml

* Update components-template-tests.yml

* Update components-e2e-tests.yml

* Update components-e2e-tests.yml

* Update components-e2e-tests.yml

* Update components-template-tests.yml

* Update components-e2e-tests.yml

* Update components-e2e-tests.yml

* Update components-e2e-tests.yml

* Revert new pipeline changes

* RestoreAdditionalProjectSources

* Update Templates.Blazor.Tests.csproj

* PR Feedback

* RunBlazorPlaywrightTemplateTests

* Socket logic

* Fixed socket wait logic

Fixes #30761

* Remove redundant process wait for exit

We're already in the `Exited` Event:

There are two ways of being notified when the associated process exits: synchronously and asynchronously. Synchronous notification means calling the WaitForExit method to block the current thread until the process exits. Asynchronous notification uses the Exited event, which allows the calling thread to continue execution in the meantime. In the latter case, EnableRaisingEvents must be set to true for the calling application to receive the Exited event.

https://learn.microsoft.com/dotnet/api/system.diagnostics.process.exited?view=net-7.0

* Update BlazorWasmTemplateTest.cs

* Update Templates.Blazor.Tests.csproj

* Revert App.Ref, add src in props

* Config updates

* Update RequiresDelayedBuildProjects.props

* Update Templates.Blazor.Tests.csproj

* Skip failing tests

* Add #46430

* Update Templates.Blazor.Tests.csproj

* Update BlazorWasmTemplateTest.cs

* Update BlazorWasmTemplateTest.cs

---------

Co-authored-by: William Godbe <[email protected]>

Author: TanayParikh

.azure/pipelines/ci.yml

@@ -698,7 +698,7 @@ stages:
           # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job.
           cancelTimeoutInMinutes: 30
           buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true
-                     /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false
+                     /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true
                      $(_InternalRuntimeDownloadArgs)
           beforeBuild:
           - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"

eng/RequiresDelayedBuildProjects.props

@@ -16,5 +16,6 @@
     <RequiresDelayedBuild Include="$(RepoRoot)src\Grpc\JsonTranscoding\test\Microsoft.AspNetCore.Grpc.Swagger.Tests\Microsoft.AspNetCore.Grpc.Swagger.Tests.csproj" />
     <RequiresDelayedBuild Include="$(RepoRoot)src\Grpc\JsonTranscoding\test\testassets\IntegrationTestsWebsite\IntegrationTestsWebsite.csproj" />
     <RequiresDelayedBuild Include="$(RepoRoot)src\Grpc\JsonTranscoding\test\testassets\Sandbox\Sandbox.csproj" />
+    <RequiresDelayedBuild Include="$(RepoRoot)src\ProjectTemplates\test\Templates.Blazor.Tests\Templates.Blazor.Tests.csproj" />
   </ItemGroup>
 </Project>

eng/Versions.props

@@ -280,7 +280,7 @@
     <NewtonsoftJsonVersion>13.0.1</NewtonsoftJsonVersion>
     <NSwagApiDescriptionClientVersion>13.0.4</NSwagApiDescriptionClientVersion>
     <PhotinoNETVersion>1.1.6</PhotinoNETVersion>
-    <MicrosoftPlaywrightVersion>1.17.3</MicrosoftPlaywrightVersion>
+    <MicrosoftPlaywrightVersion>1.28.0</MicrosoftPlaywrightVersion>
     <PollyExtensionsHttpVersion>3.0.0</PollyExtensionsHttpVersion>
     <PollyVersion>7.2.3</PollyVersion>
     <SeleniumSupportVersion>4.8.0</SeleniumSupportVersion>

eng/tools/GenerateFiles/Directory.Build.props.in

@@ -1,6 +1,7 @@
 <Project>
   <PropertyGroup>
     <DefaultNetCoreTargetFramework>${DefaultNetCoreTargetFramework}</DefaultNetCoreTargetFramework>
+    <ArtifactsShippingPackagesDir>${ArtifactsShippingPackagesDir}</ArtifactsShippingPackagesDir>
     <TreatWarningsAsErrors Condition="'$(BuildingInsideVisualStudio)' != 'true'">true</TreatWarningsAsErrors>
     <LibNetHostAppPackVersion Condition=" '$(LibNetHostAppPackVersion)' == '' ">${LibNetHostAppPackVersion}</LibNetHostAppPackVersion>
   </PropertyGroup>

eng/tools/GenerateFiles/GenerateFiles.csproj

@@ -26,7 +26,8 @@
         MicrosoftNETCoreAppRuntimeVersion=$(MicrosoftNETCoreAppRuntimeVersion);
         MicrosoftPlaywrightCLIVersion=$(MicrosoftPlaywrightCLIVersion);
         LibNetHostAppPackVersion=$(BundledNETCoreAppPackageVersion);
-        SupportedRuntimeIdentifiers=$(SupportedRuntimeIdentifiers.Trim())
+        SupportedRuntimeIdentifiers=$(SupportedRuntimeIdentifiers.Trim());
+        ArtifactsShippingPackagesDir=$(ArtifactsShippingPackagesDir)
       </_TemplateProperties>
     </PropertyGroup>
 

src/ProjectTemplates/README.md

@@ -71,6 +71,18 @@ Then, use one of:
 previous step, it is NOT advised that you install templates created on your local machine using just
 `dotnet new -i [nupkgPath]`.
 
+#### Running Blazor Playwright Template Tests
+
+1. From the root of the repo, build the templates: `.\eng\build.cmd -all -pack`
+2. `cd .\src\ProjectTemplates\test\Templates.Blazor.Tests`
+3. `dotnet test .\Templates.Blazor.Tests.csproj` with optional `--filter` arg to run a specific test.
+
+The requisite browsers should be automatically installed. If you encounter browser errors, the browsers can be manually installed via the following script, replacing `[TFM]` with the current target TFM (ex. `net8.0`).
+
+```cmd
+.\bin\Debug\[TFM]\playwright.ps1 install
+```
+
 #### Conditional tests & skipping test platforms
 
 Individual test methods can be decorated with attributes to configure them to not run ("skip running") on certain platforms. The `[ConditionalFact]` and `[ConditionalTheory]` attributes must be used on tests using the skip attributes in order for them to actually be skipped:

src/ProjectTemplates/Shared/TemplatePackageInstaller.cs

@@ -99,6 +99,11 @@ private static async Task InstallTemplatePackages(ITestOutputHelper output)
             .Where(p => _templatePackages.Any(t => Path.GetFileName(p).StartsWith(t, StringComparison.OrdinalIgnoreCase)))
             .ToArray();
 
+        if (builtPackages.Length == 0)
+        {
+            throw new InvalidOperationException($"Failed to find required templates in {packagesDir}. Please ensure the *Templates*.nupkg have been built.");
+        }
+
         Assert.Equal(4, builtPackages.Length);
 
         await VerifyCannotFindTemplateAsync(output, "web");

src/ProjectTemplates/TestInfrastructure/Directory.Build.props.in

@@ -7,4 +7,7 @@
 
   <Import Project="${ArtifactsBinDir}GenerateFiles\Directory.Build.props" />
 
+  <PropertyGroup>
+    <RestoreAdditionalProjectSources>$(RestoreAdditionalProjectSources);$(ArtifactsShippingPackagesDir)</RestoreAdditionalProjectSources>
+  </PropertyGroup>
 </Project>

src/ProjectTemplates/test/Templates.Blazor.Tests/BlazorServerTemplateTest.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.BrowserTesting;
 using Microsoft.AspNetCore.Testing;
@@ -25,7 +26,7 @@ public BlazorServerTemplateTest(ProjectFactoryFixture projectFactory)
 
     public override string ProjectType { get; } = "blazorserver";
 
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30761")]
+    [Theory]
     [InlineData(BrowserKind.Chromium)]
     public async Task BlazorServerTemplateWorks_NoAuth(BrowserKind browserKind)
     {
@@ -77,11 +78,8 @@ await BrowserManager.GetBrowserInstance(browserKind, BrowserContextInfo) :
         }
     }
 
-    public static IEnumerable<object[]> BlazorServerTemplateWorks_IndividualAuthData =>
-            BrowserManager.WithBrowsers(new[] { BrowserKind.Chromium }, true, false);
-
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
-    [MemberData(nameof(BlazorServerTemplateWorks_IndividualAuthData))]
+    [InlineData(BrowserKind.Chromium)]
+    [Theory]
     [SkipOnHelix("https://github.com/dotnet/aspnetcore/issues/30825", Queues = "All.OSX")]
     public async Task BlazorServerTemplateWorks_IndividualAuth(BrowserKind browserKind)
     {
@@ -134,29 +132,9 @@ public async Task BlazorServerTemplateWorks_IndividualAuth(BrowserKind browserKi
 
     private async Task TestBasicNavigation(IPage page)
     {
-        var socket = await page.WaitForWebSocketAsync();
-
-        var framesReceived = 0;
-        var framesSent = 0;
-
-        void FrameReceived(object sender, IWebSocketFrame frame) { framesReceived++; }
-        void FrameSent(object sender, IWebSocketFrame frame) { framesSent++; }
-
-        socket.FrameReceived += FrameReceived;
-        socket.FrameSent += FrameSent;
+        // Wait for the page to load, and the connection to idle for >500ms
+        await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new() { Timeout = 60_000 });
 
-        // Receive render batch
-        await page.WaitForWebSocketAsync(new() { Predicate = (s) => framesReceived == 1 });
-        await page.WaitForWebSocketAsync(new() { Predicate = (s) => framesSent == 1 });
-
-        // JS interop call to intercept navigation
-        await page.WaitForWebSocketAsync(new() { Predicate = (s) => framesReceived == 2 });
-        await page.WaitForWebSocketAsync(new() { Predicate = (s) => framesSent == 2 });
-
-        socket.FrameReceived -= FrameReceived;
-        socket.FrameSent -= FrameSent;
-
-        await page.WaitForSelectorAsync("nav");
         // <title> element gets project ID injected into it during template execution
         Assert.Equal("Index", (await page.TitleAsync()).Trim());
 
@@ -168,10 +146,13 @@ private async Task TestBasicNavigation(IPage page)
         await page.WaitForSelectorAsync("h1+p >> text=Current count: 0");
 
         // Clicking the counter button works
-        await page.ClickAsync("p+button >> text=Click me");
-        await page.WaitForSelectorAsync("h1+p >> text=Current count: 1");
+        for (var i = 1; i <= 3; i++)
+        {
+            await page.ClickAsync("p+button >> text=Click me");
+            await page.WaitForSelectorAsync($"h1+p >> text=Current count: {i}");
+        }
 
-        // Can navigate to the 'fetch data' page
+        // Can navigate to the 'Fetch Data' page
         await page.ClickAsync("a[href=fetchdata] >> text=Fetch data");
         await page.WaitForSelectorAsync("h1 >> text=Weather forecast");
 
@@ -180,7 +161,7 @@ private async Task TestBasicNavigation(IPage page)
         Assert.Equal(5, await page.Locator("p+table>tbody>tr").CountAsync());
     }
 
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [Theory(Skip="https://github.com/dotnet/aspnetcore/issues/46430")]
     [InlineData("IndividualB2C", null)]
     [InlineData("IndividualB2C", new [] { "--called-api-url \"https://graph.microsoft.com\"", "--called-api-scopes user.readwrite" })]
     [InlineData("SingleOrg", null)]

src/ProjectTemplates/test/Templates.Blazor.Tests/BlazorTemplateTest.cs

@@ -18,6 +18,7 @@ public abstract class BlazorTemplateTest : BrowserTestBase
     public BlazorTemplateTest(ProjectFactoryFixture projectFactory)
     {
         ProjectFactory = projectFactory;
+        Microsoft.Playwright.Program.Main(new[] { "install" });
     }
 
     public ProjectFactoryFixture ProjectFactory { get; set; }

src/ProjectTemplates/test/Templates.Blazor.Tests/BlazorWasmTemplateTest.cs

@@ -23,7 +23,7 @@ public BlazorWasmTemplateTest(ProjectFactoryFixture projectFactory)
 
     public override string ProjectType { get; } = "blazorwasm";
 
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [Theory]
     [InlineData(BrowserKind.Chromium)]
     public async Task BlazorWasmStandaloneTemplate_Works(BrowserKind browserKind)
     {
@@ -59,7 +59,7 @@ private static async Task<IPage> NavigateToPage(IBrowserContext browser, string
         return page;
     }
 
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [Theory(Skip="https://github.com/dotnet/aspnetcore/issues/46430")]
     [InlineData(BrowserKind.Chromium)]
     public async Task BlazorWasmHostedTemplate_Works(BrowserKind browserKind)
     {
@@ -107,7 +107,7 @@ private static async Task AssertCompressionFormat(AspNetProcess aspNetProcess, s
         Assert.Equal(expectedEncoding, response.Content.Headers.ContentEncoding.Single());
     }
 
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/45736")]
     [InlineData(BrowserKind.Chromium)]
     public async Task BlazorWasmStandalonePwaTemplate_Works(BrowserKind browserKind)
     {
@@ -142,7 +142,7 @@ public async Task BlazorWasmStandalonePwaTemplate_Works(BrowserKind browserKind)
         }
     }
 
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/45736")]
     [InlineData(BrowserKind.Chromium)]
     public async Task BlazorWasmHostedPwaTemplate_Works(BrowserKind browserKind)
     {
@@ -212,15 +212,15 @@ private static void ValidatePublishedServiceWorker(Project project)
         Assert.True(serviceWorkerContents.Contains($"/* Manifest version: {serviceWorkerAssetsManifestVersion} */", StringComparison.Ordinal));
     }
 
-    [ConditionalTheory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [ConditionalTheory(Skip="https://github.com/dotnet/aspnetcore/issues/46430")]
     [InlineData(BrowserKind.Chromium)]
     // LocalDB doesn't work on non Windows platforms
     [OSSkipCondition(OperatingSystems.Linux | OperatingSystems.MacOSX)]
     public Task BlazorWasmHostedTemplate_IndividualAuth_Works_WithLocalDB(BrowserKind browserKind)
         => BlazorWasmHostedTemplate_IndividualAuth_Works(browserKind, true);
 
     // This test depends on BlazorWasmTemplate_CreateBuildPublish_IndividualAuthNoLocalDb running first
-    [Theory(Skip = "https://github.com/dotnet/aspnetcore/issues/30882")]
+    [Theory(Skip="https://github.com/dotnet/aspnetcore/issues/46430")]
     [InlineData(BrowserKind.Chromium)]
     [SkipOnHelix("https://github.com/dotnet/aspnetcore/issues/30825", Queues = "All.OSX")]
     public Task BlazorWasmHostedTemplate_IndividualAuth_Works_WithOutLocalDB(BrowserKind browserKind)

src/ProjectTemplates/test/Templates.Blazor.Tests/Templates.Blazor.Tests.csproj

@@ -4,8 +4,12 @@
     <TestGroupName>ProjectTemplates.Blazor.E2ETests</TestGroupName>
     <DefineConstants>$(DefineConstants);XPLAT</DefineConstants>
 
+    <RequiresDelayedBuild>true</RequiresDelayedBuild>
+
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''">true</RunTemplateTests>
-    <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
+    <RunBlazorPlaywrightTemplateTests Condition="'$(RunBlazorPlaywrightTemplateTests)' == ''">$(RunTemplateTests)</RunBlazorPlaywrightTemplateTests>
+    <SkipTests Condition="'$(RunBlazorPlaywrightTemplateTests)' != 'true'">true</SkipTests>
+
     <BaseOutputPath />
     <OutputPath />
 
@@ -47,6 +51,7 @@
     <Reference Include="Microsoft.Extensions.Configuration.Json" />
     <Reference Include="AngleSharp" />
     <Reference Include="System.Net.Http" />
+    <Reference Include="Microsoft.AspNetCore.Components.WebAssembly" />
     <Reference Include="Microsoft.Playwright" Condition="'$(IsPlaywrightAvailable)' == 'true'" />
     <Reference Include="Microsoft.Playwright" ExcludeAssets="build" Condition="'$(IsPlaywrightAvailable)' != 'true'" />
     <ProjectReference Include="$(RepoRoot)src\Framework\App.Runtime\src\Microsoft.AspNetCore.App.Runtime.csproj"