diff --git a/.github/workflows/lighthouserc.json b/.github/workflows/lighthouserc.json new file mode 100644 index 0000000000..409ce756fe --- /dev/null +++ b/.github/workflows/lighthouserc.json @@ -0,0 +1,18 @@ +{ + "ci": { + "collect": { + "staticDistDir": "./docs/_build/html", + "settings": { + "skipAudits": ["canonical"] + } + }, + "assert": { + "assertions": { + "categories:performance": ["error", { "minScore": 0.1 }], + "categories:accessibility": ["error", { "minScore": 0.96 }], + "categories:best-practices": ["error", { "minScore": 0.85 }], + "categories:seo": ["error", { "minScore": 0.8 }] + } + } + } +} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc5d0011bc..268cf3527c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -106,54 +106,34 @@ jobs: run: | sphinx-build -b html docs/ docs/_build/html - # Serve the docs and wait to be ready + # The lighthouse audit runs directly on the HTML files, no serving needed + - name: Audit with Lighthouse + uses: treosh/lighthouse-ci-action@v8 + with: + configPath: ".github/workflows/lighthouserc.json" + temporaryPublicStorage: true + uploadArtifacts: true + runs: 3 # Multiple runs to reduce variance + urls: | + demo/kitchen-sink/paragraph-markup.html + demo/example_pandas.html + demo/theme-elements.html + + # Serve the docs for auditing with pa11y - name: Serve the built site run: | nohup python docs/serve.py --port=${PORT} --host=${HOST} & curl --retry 10 --retry-connrefused --retry-max-time 60 ${URL}/index.html - # Run the audit - # TODO: use the hosted API with a secret? would allow for comparison over time... - - name: Make folder for Lighthouse reports - run: mkdir -p /tmp/lighthouse/lighthouse-${{ github.run_number }} - - - name: Run Lighthouse on Site - id: lighthouse - uses: foo-software/lighthouse-check-action@v2.0.0 - with: - # TODO: generate this list to audit all html pages - urls: >- - ${{ env.URL }}/index.html, - ${{ env.URL }}/demo/api.html, - ${{ env.URL }}/demo/kitchen-sink/paragraph-markup.html, - ${{ env.URL }}/demo/kitchen-sink/lists-and-tables.html, - ${{ env.URL }}/demo/example_pandas.html, - ${{ env.URL }}/user_guide/accessibility.html - outputDirectory: /tmp/lighthouse/lighthouse-${{ github.run_number }} - verbose: true - - - name: Run the accessibility audit + - name: Audit with pa11y run: python docs/scripts/a11y.py --no-serve - # Check the audit for threshold values - # TODO: write this someplace after a PR is merged, and load? - - name: Assess Lighthouse Check results - uses: foo-software/lighthouse-check-status-action@v1.0.1 - with: - lighthouseCheckResults: ${{ steps.lighthouse.outputs.lighthouseCheckResults }} - minAccessibilityScore: "96" - minBestPracticesScore: "85" - minPerformanceScore: "10" - minSeoScore: "80" - if: always() - - - name: Publish Audit reports + - name: Publish pa11y report uses: actions/upload-artifact@v2 with: - name: Pa11y and Lighthouse ${{ github.run_number }} + name: Pa11y ${{ github.run_number }} path: | /tmp/pa11y - /tmp/lighthouse if: always() publish: