tests/e2e: adjust failure threshold of snapshot tests

We had increased the failure threshold to prevent test flake
but it is too high to capture diffs on very large snapshot images.
This change decreases the failure threshold and page sizes to
ensure we catch diffs. Test flake is avoided by waiting for style
transitions on the unit page and waiting for network requests to
finish before proceeding with snapshots.

Change-Id: I517027caf31064e4f70dcf121853e34457849d87
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/336710
Trust: Jamal Carvalho <[email protected]>
Run-TryBot: Jamal Carvalho <[email protected]>
TryBot-Result: kokoro <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>

Author: jamalc

tests/e2e/__image_snapshots__/basic-desktop--404-with-fetch-button-screenshot-snap.png

@@ -9,7 +9,7 @@ import { Page } from 'puppeteer';
 
 import './global-types';
 import * as pg from './helpers/page';
-import * as golangxtools from './helpers/golang-x-tools.page.ts';
+import * as golangxtools from './helpers/golang-x-tools.page';
 
 let page: Page;
 
@@ -25,6 +25,8 @@ afterAll(async () => {
 
 test('fixed header appears after scrolling', async () => {
   await page.evaluate(() => window.scrollTo({ top: 250 }));
+  // Wait for header transition
+  await page.evaluate(() => new Promise(r => setTimeout(r, 250)));
   const image = await page.screenshot();
   expect(image).toMatchImageSnapshot();
   await page.evaluate(() => window.scrollTo({ top: 0 }));
@@ -33,14 +35,14 @@ test('fixed header appears after scrolling', async () => {
 describe('readme', () => {
   test('expands', async () => {
     await page.click(pg.select('readme-expand'));
-    await page.evaluate(() => window.scrollTo({ top: 0 }));
+    await scrollTop(page);
     const expanded = await page.screenshot({ fullPage: true });
     expect(expanded).toMatchImageSnapshot();
   });
 
   test('collapses', async () => {
     await page.click(pg.select('readme-collapse'));
-    await page.evaluate(() => window.scrollTo({ top: 0 }));
+    await scrollTop(page);
     const collapsed = await page.screenshot({ fullPage: true });
     expect(collapsed).toMatchImageSnapshot();
   });
@@ -49,14 +51,14 @@ describe('readme', () => {
 describe('directories', () => {
   test('expand', async () => {
     await page.click(pg.select('directories-toggle'));
-    await page.evaluate(() => window.scrollTo({ top: 0 }));
+    await scrollTop(page);
     const expanded = await page.screenshot({ fullPage: true });
     expect(expanded).toMatchImageSnapshot();
   });
 
   test('collapse', async () => {
     await page.click(pg.select('directories-toggle'));
-    await page.evaluate(() => window.scrollTo({ top: 0 }));
+    await scrollTop(page);
     const collapsed = await page.screenshot({ fullPage: true });
     expect(collapsed).toMatchImageSnapshot();
   });
@@ -65,14 +67,14 @@ describe('directories', () => {
 describe('jump to modal', () => {
   test('opens', async () => {
     await page.click(pg.select('jump-to-button'));
-    await page.evaluate(() => window.scrollTo({ top: 0 }));
+    await scrollTop(page);
     const expanded = await page.screenshot();
     expect(expanded).toMatchImageSnapshot();
   });
 
   test('closes', async () => {
     await page.click(pg.select('close-dialog'));
-    await page.evaluate(() => window.scrollTo({ top: 0 }));
+    await scrollTop(page);
     const collapsed = await page.screenshot();
     expect(collapsed).toMatchImageSnapshot();
   });
@@ -81,3 +83,15 @@ describe('jump to modal', () => {
 test('no page errors', () => {
   expect(pageErrors).toHaveLength(0);
 });
+
+/**
+ * scrollTop scrolls to the top of a given page and waits
+ * a short amount of time for any style transitions to
+ * complete. Used to make sure the documentation page
+ * header has completed transitioning.
+ * @param page the page to scroll.
+ */
+async function scrollTop(page: Page): Promise<void> {
+  await page.evaluate(() => window.scrollTo({ top: 0 }));
+  await page.evaluate(() => new Promise(r => setTimeout(r, 250)));
+}

tests/e2e/__image_snapshots__/basic-desktop--badge-screenshot-snap.png

@@ -26,6 +26,8 @@ afterAll(async () => {
 
 test('fixed header appears after scrolling', async () => {
   await page.evaluate(() => window.scrollTo({ top: 250 }));
+  // Wait for header transition
+  await page.evaluate(() => new Promise(r => setTimeout(r, 250)));
   const image = await page.screenshot();
   expect(image).toMatchImageSnapshot();
 });

tests/e2e/__image_snapshots__/basic-desktop--error-screenshot-snap.png

@@ -44,7 +44,7 @@ export async function newPage(): Promise<Page> {
   });
   const go = page.goto;
   page.goto = (path: string, opts?: DirectNavigationOptions) =>
-    go.call(page, GO_DISCOVERY_E2E_BASE_URL + path, opts);
+    go.call(page, GO_DISCOVERY_E2E_BASE_URL + path, { waitUntil: 'networkidle0', ...opts });
   return page;
 }
 

tests/e2e/__image_snapshots__/basic-desktop--license-policy-screenshot-snap.png

@@ -14,12 +14,16 @@ github.com/tendermint/[email protected]
 [email protected]
 golang.org/x/[email protected]
 golang.org/x/[email protected]
+golang.org/x/[email protected]
 k8s.io/[email protected]
 mvdan.cc/sh/[email protected]
 mvdan.cc/[email protected]+incompatible
 golang.org/x/tools/[email protected]
 github.com/jackc/[email protected]+incompatible
+[email protected]
+[email protected]
 [email protected]
+[email protected]
 
 # This module is expected to return a 404, so that  the path will redirect
 # to github.com/jackc/pgx/v4/pgxpool.

tests/e2e/__image_snapshots__/basic-desktop--search-help-screenshot-snap.png

@@ -9,7 +9,7 @@ import { configureToMatchImageSnapshot } from 'jest-image-snapshot';
 
 // Extends jest to compare image snapshots.
 const toMatchImageSnapshot = configureToMatchImageSnapshot({
-  failureThreshold: 0.1,
+  failureThreshold: 0.001,
   failureThresholdType: 'percent',
   customSnapshotIdentifier: ({ defaultIdentifier, counter }) => {
     return defaultIdentifier.replace('test-ts', '').replace(`-${counter}`, '');

tests/e2e/__image_snapshots__/basic-mobile--404-with-fetch-button-screenshot-snap.png

@@ -16,7 +16,7 @@ let page: Page;
 
 beforeAll(async () => {
   page = await pg.newPage();
-  await page.setViewport({ height: 2732, width: 1366 });
+  await page.setViewport({ height: 1366, width: 1366 });
 });
 
 afterAll(async () => {

tests/e2e/__image_snapshots__/basic-mobile--badge-screenshot-snap.png

@@ -16,7 +16,7 @@ let page: Page;
 
 beforeAll(async () => {
   page = await pg.newPage();
-  await page.setViewport({ height: 2732, width: 500 });
+  await page.setViewport({ height: 1000, width: 500 });
 });
 
 afterAll(async () => {