E2e Test Fixes

Makefile

@@ -617,8 +617,6 @@ playwright: $(PLAYWRIGHT_DIR)
 
 .PHONY: test-e2e%
 test-e2e%: TEST_TYPE ?= e2e
-	# Clear display env variable. Otherwise, chromium tests can fail.
-	DISPLAY=
 
 .PHONY: test-e2e
 test-e2e: test-e2e-sqlite

playwright.config.js

@@ -64,13 +64,12 @@ export default {
       },
     },
 
-    // disabled because of https://github.com/go-gitea/gitea/issues/21355
-    // {
-    //   name: 'firefox',
-    //   use: {
-    //     ...devices['Desktop Firefox'],
-    //   },
-    // },
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+      },
+    },
 
     {
       name: 'webkit',

tests/e2e/README.md

@@ -4,11 +4,11 @@ E2e tests largely follow the same syntax as [integration tests](../integration).
 Whereas integration tests are intended to mock and stress the back-end, server-side code, e2e tests the interface between front-end and back-end, as well as visual regressions with both assertions and visual comparisons.
 They can be run with make commands for the appropriate backends, namely:
 ```shell
-make test-sqlite
-make test-pgsql
-make test-mysql
-make test-mysql8
-make test-mssql
+make test-e2e-sqlite
+make test-e2e-pgsql
+make test-e2e-mysql
+make test-e2e-mysql8
+make test-e2e-mssql
 ```
 
 Make sure to perform a clean front-end build before running tests:
@@ -64,6 +64,10 @@ Start tests based on the database container
 TEST_MSSQL_HOST=localhost:1433 TEST_MSSQL_DBNAME=gitea_test TEST_MSSQL_USERNAME=sa TEST_MSSQL_PASSWORD=MwantsaSecurePassword1 make test-e2e-mssql
 ```
 
+## Debug
+
+Set the environment variable `PLAYWRIGHT_DEBUG` to enable [visual debugging](https://playwright.dev/docs/debug#run-in-debug-mode-1).
+
 ## Running individual tests
 
 Example command to run `example.test.e2e.js` test file:

tests/e2e/e2e_test.go

@@ -75,6 +75,8 @@ func TestMain(m *testing.M) {
 
 // TestE2e should be the only test e2e necessary. It will collect all "*.test.e2e.js" files in this directory and build a test for each.
 func TestE2e(t *testing.T) {
+	browsers := []string{"chromium", "firefox", "webkit", "Mobile Chrome", "Mobile Safari"}
+
 	// Find the paths of all e2e test files in test test directory.
 	searchGlob := filepath.Join(filepath.Dir(setting.AppPath), "tests", "e2e", "*.test.e2e.js")
 	paths, err := filepath.Glob(searchGlob)
@@ -91,30 +93,40 @@ func TestE2e(t *testing.T) {
 		runArgs = append(runArgs, "--update-snapshots")
 	}
 
+	// If debug flag is set
+	if _, set := os.LookupEnv("PLAYWRIGHT_DEBUG"); set {
+		runArgs = append(runArgs, "--debug")
+	}
+
 	// Create new test for each input file
 	for _, path := range paths {
-		_, filename := filepath.Split(path)
-		testname := filename[:len(filename)-len(filepath.Ext(path))]
-
-		t.Run(testname, func(t *testing.T) {
-			// Default 2 minute timeout
-			onGiteaRun(t, func(*testing.T, *url.URL) {
-				cmd := exec.Command(runArgs[0], runArgs...)
-				cmd.Env = os.Environ()
-				cmd.Env = append(cmd.Env, fmt.Sprintf("GITEA_URL=%s", setting.AppURL))
-				var stdout, stderr bytes.Buffer
-				cmd.Stdout = &stdout
-				cmd.Stderr = &stderr
-				err := cmd.Run()
-				if err != nil {
-					// Currently colored output is conflicting. Using Printf until that is resolved.
-					fmt.Printf("%v", stdout.String())
-					fmt.Printf("%v", stderr.String())
-					log.Fatal("Playwright Failed: %s", err)
-				} else {
-					fmt.Printf("%v", stdout.String())
-				}
+		// Iterate each browser serially to reset the fixtures
+		// TODO: parallel tests with separate environments?
+		for _, browser := range browsers {
+			_, filename := filepath.Split(path)
+			testname := filename[:len(filename)-len(filepath.Ext(path))]
+
+			t.Run(testname+"/"+browser, func(t *testing.T) {
+				// Default 2 minute timeout
+				onGiteaRun(t, func(*testing.T, *url.URL) {
+					// Finally, append the testname to only run the current test
+					cmd := exec.Command(runArgs[0], append(runArgs[1:], "--project="+browser, testname)...)
+					cmd.Env = os.Environ()
+					cmd.Env = append(cmd.Env, fmt.Sprintf("GITEA_URL=%s", setting.AppURL))
+					var stdout, stderr bytes.Buffer
+					cmd.Stdout = &stdout
+					cmd.Stderr = &stderr
+					err := cmd.Run()
+					if err != nil {
+						// Currently colored output is conflicting. Using Printf until that is resolved.
+						fmt.Printf("%v", stdout.String())
+						fmt.Printf("%v", stderr.String())
+						log.Fatal("Playwright Failed: %s", err)
+					} else {
+						fmt.Printf("%v", stdout.String())
+					}
+				})
 			})
-		})
+		}
 	}
 }

tests/e2e/example.test.e2e.js

@@ -16,32 +16,30 @@ test('Load Homepage', async ({page}) => {
 test('Test Register Form', async ({page}, workerInfo) => {
   const response = await page.goto('/user/sign_up');
   await expect(response?.status()).toBe(200); // Status OK
-  await page.type('input[name=user_name]', `e2e-test-${workerInfo.workerIndex}`);
-  await page.type('input[name=email]', `e2e-test-${workerInfo.workerIndex}@test.com`);
-  await page.type('input[name=password]', 'test123');
-  await page.type('input[name=retype]', 'test123');
-  await page.click('form button.ui.green.button:visible');
+  await page.locator('input#user_name').fill(`e2e-test-${workerInfo.workerIndex}`);
+  await page.locator('input#email').fill(`e2e-test-${workerInfo.workerIndex}@test.com`);
+  await page.locator('input#password').fill('test123');
+  await page.locator('input#retype').fill('test123');
+  await page.locator('form button.ui.green.button:visible').click();
   // Make sure we routed to the home page. Else login failed.
   await expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`);
   await expect(page.locator('.dashboard-navbar span>img.ui.avatar')).toBeVisible();
   await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created.');
 
-  save_visual(page);
+  await save_visual(page);
 });
 
 test('Test Login Form', async ({page}, workerInfo) => {
   const response = await page.goto('/user/login');
   await expect(response?.status()).toBe(200); // Status OK
 
-  await page.type('input[name=user_name]', `user2`);
-  await page.type('input[name=password]', `password`);
-  await page.click('form button.ui.green.button:visible');
-
-  await page.waitForLoadState('networkidle');
+  await page.locator('input#user_name').fill('user2');
+  await page.locator('input#password').fill('password');
+  await page.locator('form button.ui.green.button:visible').click();
 
   await expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`);
 
-  save_visual(page);
+  await save_visual(page);
 });
 
 test('Test Logged In User', async ({browser}, workerInfo) => {
@@ -53,5 +51,5 @@ test('Test Logged In User', async ({browser}, workerInfo) => {
   // Make sure we routed to the home page. Else login failed.
   await expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`);
 
-  save_visual(page);
+  await save_visual(page);
 });

tests/e2e/issue.test.e2e.js

@@ -0,0 +1,56 @@
+// @ts-check
+import {test, expect} from '@playwright/test';
+import {login_user, save_visual, load_logged_in_context} from './utils_e2e.js';
+
+test.beforeAll(async ({browser}, workerInfo) => {
+  await login_user(browser, workerInfo, 'user2');
+});
+
+test('Test New Issue', async ({browser}, workerInfo) => {
+  const context = await load_logged_in_context(browser, workerInfo, 'user2');
+
+  const page = await context.newPage();
+
+  let response = await page.goto('/user2/repo2/issues');
+  await expect(response?.status()).toBe(200); // Status OK
+
+  // Click New Issue
+  await page.getByRole('link', {name: 'New Issue'}).click();
+
+  await expect(page).toHaveURL(`${workerInfo.project.use.baseURL}/user2/repo2/issues/new`);
+
+  await page.locator('[name=title]').fill(`New Issue: ${workerInfo.title}`);
+  await page.locator('[name=content]').fill(`
+# Test Header
+
+- [ ] Unchecked list item
+- [ ] Second unchecked list item
+- [x] Checked list item
+`);
+
+  // Switch to preview
+  const previewButton = page.getByText('Preview');
+  await previewButton.click();
+  await expect(previewButton).toHaveClass(/(^|\W)active($|\W)/);
+  await expect(page.locator('[data-tab-panel=markdown-previewer]')).toBeVisible();
+  await expect(page.getByRole('heading', {name: 'Test Header'})).toBeVisible();
+
+  // Create issue
+  await page.getByRole('button', {name: 'Create Issue'}).click();
+  await expect(page).toHaveURL(`${workerInfo.project.use.baseURL}/user2/repo2/issues/3`);
+
+  await expect(page.getByRole('heading', {name: 'Test Header'})).toBeVisible();
+
+  // Test checkboxes
+  const checkboxes = page.locator('.task-list-item > [type=checkbox]');
+  await expect(checkboxes).toHaveCount(3);
+  await expect(checkboxes.first()).not.toBeChecked();
+  const checkboxPostPromise = page.waitForResponse(`${workerInfo.project.use.baseURL}/user2/repo2/issues/3/content`);
+  await checkboxes.first().click(); // Toggle checkbox
+  await expect(checkboxes.first()).toBeChecked();
+  expect((await checkboxPostPromise).status()).toBe(200); // Wait for successful content post response
+  response = await page.reload(); // Reload page to check consistency
+  await expect(checkboxes.first()).toBeChecked();
+
+  await save_visual(page);
+});

tests/e2e/utils_e2e.js

@@ -8,22 +8,16 @@ const LOGIN_PASSWORD = 'password';
 export async function login_user(browser, workerInfo, user) {
   // Set up a new context
   const context = await browser.newContext();
-  const page = await context.newPage();
 
   // Route to login page
   // Note: this could probably be done more quickly with a POST
-  const response = await page.goto('/user/login');
-  await expect(response?.status()).toBe(200); // Status OK
-
-  // Fill out form
-  await page.type('input[name=user_name]', user);
-  await page.type('input[name=password]', LOGIN_PASSWORD);
-  await page.click('form button.ui.green.button:visible');
-
-  await page.waitForLoadState('networkidle');
-
-  await expect(page.url(), {message: `Failed to login user ${user}`}).toBe(`${workerInfo.project.use.baseURL}/`);
-
+  const response = await context.request.post('/user/login', {
+    form: {
+      'user_name': user,
+      'password': LOGIN_PASSWORD
+    }
+  });
+  expect(response).toBeOK();
   // Save state
   await context.storageState({path: `${ARTIFACTS_PATH}/state-${user}-${workerInfo.workerIndex}.json`});
 
@@ -53,7 +47,7 @@ export async function save_visual(page) {
       timeout: 20000,
       mask: [
         page.locator('.dashboard-navbar span>img.ui.avatar'),
-        page.locator('.ui.dropdown.jump.item span>img.ui.avatar'),
+        page.locator('.ui.dropdown.jump.item.tooltip span>img.ui.avatar'),
       ],
     });
   }