Skip to content

Commit 724f974

Browse files
Separated a11y and vis tests and made vis test be triggered by /vistest (#7402)
Running the vis tests cost money and we don't need them run on each merge to `main`. We should eventually move off of storybook and run real visual regression testing but until then let's make the vis tests be triggered by a comment `/vistest` and go from there. TODO: - [x] We need add documentation for this new comment. I assume into the [`CONTRIBUTING.md`](https://github.com/Shopify/polaris/blob/00449becb15df24d22d7d062e2e20cda7c14d954/.github/CONTRIBUTING.md) file? - [ ] Test the workflow. I moved things and merged things in. in theory this should work but I have not tested it yet
1 parent 141746b commit 724f974

File tree

3 files changed

+77
-40
lines changed

3 files changed

+77
-40
lines changed

.github/CONTRIBUTING.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ We recommend running tests as well as trying your build of a package in a real p
155155

156156
- `yarn test` will run tests for all packages
157157
- `yarn turbo run test --filter="[HEAD^1]"` will test any package that has changed in the last commit
158+
- Run visual testing via adding the `/vistest` comment in a PR
158159

159160
You can test your changes in another project locally by creating a [snapshot release](https://github.com/Shopify/polaris/blob/main/documentation/Releasing.md#snapshot-release).
160161

.github/workflows/ci-a11y.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Accessibility Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- next
8+
9+
jobs:
10+
accessibility_test:
11+
name: 'Accessibility test'
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout branch
15+
uses: actions/checkout@v3
16+
17+
- name: Setup Node with v16.13.0
18+
uses: actions/setup-node@v3
19+
with:
20+
node-version: 16.13.0
21+
cache: yarn
22+
23+
- name: Restore cache
24+
uses: actions/cache@v3
25+
with:
26+
path: |
27+
**/.turbo
28+
node_modules/.cache/turbo
29+
polaris-react/build-internal/cache
30+
key: ${{ runner.os }}-accessibility-test-v1-${{ github.sha }}
31+
restore-keys: |
32+
${{ runner.os }}-accessibility-test-v1-
33+
34+
- name: Install dependencies
35+
run: yarn --frozen-lockfile
36+
37+
- name: Build packages
38+
run: yarn turbo run build --filter=@shopify/polaris
39+
40+
- name: Build Storybook
41+
run: yarn workspace @shopify/polaris run storybook:build --quiet
42+
43+
- name: Run accessibility test
44+
run: node ./polaris-react/scripts/accessibility-check.js
45+
env:
46+
STORYBOOK_DISABLE_DOCS: 1

.github/workflows/ci-a11y-vrt.yml renamed to .github/workflows/vistest.yml

Lines changed: 30 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,15 @@
1-
name: Accessibility and Visual Regression Tests
1+
name: Visual Regression Tests
22

33
on:
4-
push:
5-
branches:
6-
- main
7-
- next
8-
paths:
9-
- 'polaris-react/src/**'
10-
- 'polaris-react/playground/**'
11-
- 'polaris-tokens/src/**'
12-
pull_request:
13-
paths:
14-
- 'polaris-react/src/**'
15-
- 'polaris-react/playground/**'
16-
- 'polaris-tokens/src/**'
4+
issue_comment:
5+
types:
6+
- created
177

188
jobs:
19-
accessibility_test:
20-
name: 'Accessibility test'
9+
visual_regression_test:
10+
name: 'Visual regression test'
11+
if: |
12+
github.event.issue.pull_request && github.event.comment.body == '/vistest'
2113
runs-on: ubuntu-latest
2214
steps:
2315
- name: Checkout branch
@@ -26,38 +18,36 @@ jobs:
2618
- name: Setup Node with v16.13.0
2719
uses: actions/setup-node@v3
2820
with:
29-
node-version: 16.13.0
30-
cache: yarn
21+
permission: write
3122

3223
- name: Restore cache
3324
uses: actions/cache@v3
3425
with:
35-
path: |
36-
**/.turbo
37-
node_modules/.cache/turbo
38-
polaris-react/build-internal/cache
39-
key: ${{ runner.os }}-accessibility-test-v1-${{ github.sha }}
40-
restore-keys: |
41-
${{ runner.os }}-accessibility-test-v1-
26+
script: |
27+
try {
28+
const pullRequest = await github.rest.pulls.get({
29+
owner: context.repo.owner,
30+
repo: context.repo.repo,
31+
pull_number: context.issue.number,
32+
})
4233
43-
- name: Install dependencies
44-
run: yarn --frozen-lockfile
34+
// Pull request from fork
35+
if (context.payload.repository.full_name !== pullRequest.data.head.repo.full_name) {
36+
const errorMessage = '`/vistest` is not supported on pull requests from forked repositories.'
4537
46-
- name: Build packages
47-
run: yarn turbo run build --filter=@shopify/polaris
48-
49-
- name: Build Storybook
50-
run: yarn workspace @shopify/polaris run storybook:build --quiet
38+
await github.rest.issues.createComment({
39+
issue_number: context.issue.number,
40+
owner: context.repo.owner,
41+
repo: context.repo.repo,
42+
body: errorMessage,
43+
})
5144
52-
- name: Run accessibility test
53-
run: node ./polaris-react/scripts/accessibility-check.js
54-
env:
55-
STORYBOOK_DISABLE_DOCS: 1
45+
core.setFailed(errorMessage)
46+
}
47+
} catch (err) {
48+
core.setFailed(`Request failed with error ${err}`)
49+
}
5650
57-
visual_regression_test:
58-
name: 'Visual regression test'
59-
runs-on: ubuntu-latest
60-
steps:
6151
- name: Checkout branch
6252
uses: actions/checkout@v3
6353
with:

0 commit comments

Comments
 (0)