diff --git a/tests/test_a11y.py b/tests/test_a11y.py index d9d9dbbfb1..9797f88528 100644 --- a/tests/test_a11y.py +++ b/tests/test_a11y.py @@ -70,20 +70,69 @@ def url_base(): process.wait() -def fingerprint_violations(accessibility_page_scan_violations): - """Create a fingerprint of the Axe violations array. +def filter_ignored_violations(violations, url_pathname): + """Filter out ignored axe-core violations. - https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues + In some tests, we wish to ignore certain accessibility violations that we + won't ever fix or that we don't plan to fix soon. """ - return [ - { - "id": violation["id"], - "help": violation["help"], - "helpUrl": violation["helpUrl"], - "targets": [node["target"] for node in violation["nodes"]], - } - for violation in accessibility_page_scan_violations - ] + # we allow empty table headers + # https://dequeuniversity.com/rules/axe/4.8/empty-table-header?application=RuleDescription + if url_pathname == "/examples/pydata.html": + return [v for v in violations if v["id"] != "empty-table-header"] + elif url_pathname in [ + "/examples/kitchen-sink/generic.html", + "/user_guide/theme-elements.html", + ]: + filtered = [] + for violation in violations: + # TODO: eventually fix this rule violation. See + # https://github.com/pydata/pydata-sphinx-theme/issues/1479. + if violation["id"] == "landmark-unique": + # Ignore landmark-unique only for .sidebar targets. Don't ignore + # it for other targets because then the test might fail to catch + # a change that violates the rule in some other way. + unexpected_nodes = [] + for node in violation["nodes"]: + # If some target is not .sidebar then we've found a rule + # violation we weren't expecting + if not all([".sidebar" in target for target in node["target"]]): + unexpected_nodes.append(node) + if unexpected_nodes: + violation["nodes"] = unexpected_nodes + filtered.append(violation) + else: + filtered.append(violation) + return filtered + else: + return violations + + +def format_violations(violations): + """Return a pretty string representation of Axe-core violations.""" + result = f""" + + Found {len(violations)} accessibility violation(s): + """ + + for violation in violations: + result += f""" + + - Rule violated: + {violation["id"]} - {violation["help"]} + - URL: {violation["helpUrl"]} + - Impact: {violation["impact"]} + - Tags: {" ".join(violation["tags"])} + - Targets:""" + + for node in violation["nodes"]: + for target in node["target"]: + result += f""" + - {target}""" + + result += "\n\n" + + return result @pytest.mark.a11y @@ -122,7 +171,7 @@ def fingerprint_violations(accessibility_page_scan_violations): "/examples/kitchen-sink/typography.html", "#typography", ), - ("/examples/pydata.html", "#pydata-library-styles"), + ("/examples/pydata.html", "#PyData-Library-Styles"), ( "/user_guide/theme-elements.html", "#theme-specific-elements", @@ -142,16 +191,18 @@ def fingerprint_violations(accessibility_page_scan_violations): # Using one of the simplest pages on the site, select the whole page for # testing in order to effectively test repeated website elements like # nav, sidebars, breadcrumbs, footer - ("/user_guide/page-toc.html", ""), + ( + "/user_guide/page-toc.html", + "", # select whole page + ), ], ) def test_axe_core( - data_regression, - theme: str, + page: Page, url_base: str, + theme: str, url_pathname: str, selector: str, - page: Page, ): """Should have no Axe-core violations at the provided theme and page section.""" # Load the page at the provided path @@ -164,14 +215,13 @@ def test_axe_core( # Inject the Axe-core JavaScript library into the page page.add_script_tag(path="node_modules/axe-core/axe.min.js") - # Run the Axe-core library against a section of the page. (Don't run it - # against the whole page because in this test we're not trying to find - # accessibility violations in the nav, sidebar, footer, or other parts of - # the PyData Sphinx Theme documentation website.) + # Run the Axe-core library against a section of the page (unless the + # selector is empty, then run against the whole page) results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')") # Check found violations against known violations that we do not plan to fix - data_regression.check(fingerprint_violations(results["violations"])) + filtered_violations = filter_ignored_violations(results["violations"], url_pathname) + assert len(filtered_violations) == 0, format_violations(filtered_violations) def test_version_switcher_highlighting(page: Page, url_base: str) -> None: diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml deleted file mode 100644 index b84d7335f1..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml +++ /dev/null @@ -1,5 +0,0 @@ -- help: Table header text should not be empty - helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI - id: empty-table-header - targets: - - - thead > tr > th:nth-child(1) diff --git a/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml deleted file mode 100644 index b84d7335f1..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml +++ /dev/null @@ -1,5 +0,0 @@ -- help: Table header text should not be empty - helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI - id: empty-table-header - targets: - - - thead > tr > th:nth-child(1) diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml deleted file mode 100644 index fe51488c70..0000000000 --- a/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml +++ /dev/null @@ -1 +0,0 @@ -[]