diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..ea24f417 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,58 @@ +name: Docs + +on: + push: + branches: + - main + tags: + - v[0-9]+.[0-9]+.[0-9]+ + pull_request: + +permissions: + contents: write + +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + - run: python -m pip install pip --upgrade + - run: python -m pip install -e ".[test,docs]" + - run: | + cd docs + make html + - uses: actions/upload-artifact@v3 + with: + name: docs-html + path: | + docs/_build/html + retention-days: 5 + + deploy: + if: ${{ github.event_name != 'pull_request' }} + needs: [build] + environment: + name: github-pages + runs-on: ubuntu-latest + steps: + - name: Download doc build + uses: actions/download-artifact@v3 + with: + name: docs-html + path: ./docs/_build/html + - name: Upload to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/_build/html + exclude_assets: '.buildinfo,_static/jquery-*.js,_static/underscore-*.js' + destination_dir: ./latest + keep_files: false + full_commit_message: Deploy latest to GitHub Pages diff --git a/.gitignore b/.gitignore index 627f8278..6b2c3631 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ __pycache__ build dist .tmp +.DS_Store +/docs/sample +/docs/_build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55f8e7ae..8d468221 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,6 +74,7 @@ repos: hooks: - id: isort args: ["--sp", "setup.cfg"] + exclude: ".*(docs/conf.py)$" - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_templates/mpl_third_party_sidebar.html b/docs/_templates/mpl_third_party_sidebar.html new file mode 100644 index 00000000..43068887 --- /dev/null +++ b/docs/_templates/mpl_third_party_sidebar.html @@ -0,0 +1,8 @@ + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..cb7e341d --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,74 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import datetime +from packaging.version import Version + +from pytest_mpl import __version__ + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'pytest-mpl' +author = 'Thomas Robitaille' +copyright = '{}, {}'.format(datetime.datetime.now().year, author) + +release = __version__ +pytest_mpl_version = Version(__version__) +is_release = not (pytest_mpl_version.is_prerelease or pytest_mpl_version.is_devrelease) + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sample_summaries', + 'sphinx_design', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "mpl_sphinx_theme" +html_theme_options = { + "navbar_links": "absolute", + "show_prev_next": False, + "logo": {"link": "https://matplotlib.org/stable/", + "image_light": "images/logo2.svg", + "image_dark": "images/logo_dark.svg"}, + "collapse_navigation": False, +} +html_sidebars = { + "**": ["mpl_third_party_sidebar.html", "sidebar-nav-bs.html"] +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 00000000..7bc41a13 --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,187 @@ +.. title:: Configuration + +############# +Configuration +############# + +Tolerance +^^^^^^^^^ + +The RMS tolerance for the image comparison (which defaults to 2) can be +specified in the ``mpl_image_compare`` decorator with the ``tolerance`` +argument: + +.. code:: python + + @pytest.mark.mpl_image_compare(tolerance=20) + def test_image(): + ... + +Savefig options +^^^^^^^^^^^^^^^ + +You can pass keyword arguments to ``savefig`` by using +``savefig_kwargs`` in the ``mpl_image_compare`` decorator: + +.. code:: python + + @pytest.mark.mpl_image_compare(savefig_kwargs={'dpi':300}) + def test_image(): + ... + +Baseline images +^^^^^^^^^^^^^^^ + +The baseline directory (which defaults to ``baseline`` ) and the +filename of the plot (which defaults to the name of the test with a +``.png`` suffix) can be customized with the ``baseline_dir`` and +``filename`` arguments in the ``mpl_image_compare`` decorator: + +.. code:: python + + @pytest.mark.mpl_image_compare(baseline_dir='baseline_images', + filename='other_name.png') + def test_image(): + ... + +The baseline directory in the decorator above will be interpreted as +being relative to the test file. Note that the baseline directory can +also be a URL (which should start with ``http://`` or ``https://`` and +end in a slash). If you want to specify mirrors, set ``baseline_dir`` to +a comma-separated list of URLs (real commas in the URL should be encoded +as ``%2C``). + +Finally, you can also set a custom baseline directory globally when +running tests by running ``pytest`` with:: + + pytest --mpl --mpl-baseline-path=baseline_images + +This directory will be interpreted as being relative to where pytest +is run. However, if the ``--mpl-baseline-relative`` option is also +included, this directory will be interpreted as being relative to +the current test directory. +In addition, if both this option and the ``baseline_dir`` +option in the ``mpl_image_compare`` decorator are used, the one in the +decorator takes precedence. + +Results always +^^^^^^^^^^^^^^ + +By default, result images are only saved for tests that fail. +Passing ``--mpl-results-always`` to pytest will force result images +to be saved for all tests, even for tests that pass. + +When in **hybrid mode**, even if a test passes hash comparison, +a comparison to the baseline image will also be carried out, +with the baseline image and diff image (if image comparison fails) +saved for all tests. This secondary comparison will not affect +the success status of the test. + +This option is useful for always *comparing* the result images against +the baseline images, while only *assessing* the tests against the +hash library. +If you only update your baseline images after merging a PR, this +option means that the generated summary will always show how the +PR affects the baseline images, with the success status of each +test (based on the hash library) also shown in the generated +summary. This option is applied automatically when generating +a HTML summary. + +When the ``--mpl-results-always`` option is active, and some hash +comparison tests are performed, a hash library containing all the +result hashes will also be saved to the root of the results directory. +The filename will be extracted from ``--mpl-generate-hash-library``, +``--mpl-hash-library`` or ``hash_library=`` in that order. + +Base style +^^^^^^^^^^ + +By default, tests will be run using the Matplotlib 'classic' style +(ignoring any locally defined RC parameters). This can be overridden by +using the ``style`` argument: + +.. code:: python + + @pytest.mark.mpl_image_compare(style='fivethirtyeight') + def test_image(): + ... + +Package version dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Different versions of Matplotlib and FreeType may result in slightly +different images. When testing on multiple platforms or as part of a +pipeline, it is important to ensure that the versions of these +packages match the versions used to generate the images used for +comparison. It can be useful to pin versions of Matplotlib and FreeType +so as to avoid automatic updates that fail tests. + +Removing text +^^^^^^^^^^^^^ + +If you are running a test for which you are not interested in comparing +the text labels, you can use the ``remove_text`` argument to the +decorator: + +.. code:: python + + @pytest.mark.mpl_image_compare(remove_text=True) + def test_image(): + ... + +This will make the test insensitive to changes in e.g. the freetype +library. + +Test failure example +-------------------- + +If the images produced by the tests are correct, then the test will +pass, but if they are not, the test will fail with a message similar to +the following:: + + E Exception: Error: Image files did not match. + E RMS Value: 142.2287807767823 + E Expected: + E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/baseline-coords_overlay_auto_coord_meta.png + E Actual: + E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/coords_overlay_auto_coord_meta.png + E Difference: + E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/coords_overlay_auto_coord_meta-failed-diff.png + E Tolerance: + E 10 + +The image paths included in the exception are then available for +inspection: + ++----------------+----------------+-------------+ +| Expected | Actual | Difference | ++================+================+=============+ +| |expected| | |actual| | |diff| | ++----------------+----------------+-------------+ + +In this case, the differences are very clear, while in some cases it may +be necessary to use the difference image, or blink the expected and +actual images, in order to see what changed. + +The default tolerance is 2, which is very strict. In some cases, you may +want to relax this to account for differences in fonts across different +systems. + +By default, the expected, actual and difference files are written to a +temporary directory with a non-deterministic path. If you want to instead +write them to a specific directory, you can use:: + + pytest --mpl --mpl-results-path=results + +The ``results`` directory will then contain one sub-directory per test, and each +sub-directory will contain the three files mentioned above. If you are using a +continuous integration service, you can then use the option to upload artifacts +to upload these results to somewhere where you can view them. For more +information, see: + +* `Uploading artifacts on Travis-CI `_ +* `Build Artifacts (CircleCI) `_ +* `Packaging Artifacts (AppVeyor) `_ + +.. |expected| image:: images/baseline-coords_overlay_auto_coord_meta.png +.. |actual| image:: images/coords_overlay_auto_coord_meta.png +.. |diff| image:: images/coords_overlay_auto_coord_meta-failed-diff.png diff --git a/docs/images/baseline-coords_overlay_auto_coord_meta.png b/docs/images/baseline-coords_overlay_auto_coord_meta.png new file mode 100644 index 00000000..06970ddd Binary files /dev/null and b/docs/images/baseline-coords_overlay_auto_coord_meta.png differ diff --git a/docs/images/coords_overlay_auto_coord_meta-failed-diff.png b/docs/images/coords_overlay_auto_coord_meta-failed-diff.png new file mode 100644 index 00000000..0d1b7e2d Binary files /dev/null and b/docs/images/coords_overlay_auto_coord_meta-failed-diff.png differ diff --git a/docs/images/coords_overlay_auto_coord_meta.png b/docs/images/coords_overlay_auto_coord_meta.png new file mode 100644 index 00000000..b6c1a5bc Binary files /dev/null and b/docs/images/coords_overlay_auto_coord_meta.png differ diff --git a/docs/images/html_all.png b/docs/images/html_all.png new file mode 100644 index 00000000..82e3eec4 Binary files /dev/null and b/docs/images/html_all.png differ diff --git a/docs/images/html_filter.png b/docs/images/html_filter.png new file mode 100644 index 00000000..9fc6f998 Binary files /dev/null and b/docs/images/html_filter.png differ diff --git a/docs/images/html_result.png b/docs/images/html_result.png new file mode 100644 index 00000000..d8ef5c44 Binary files /dev/null and b/docs/images/html_result.png differ diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..c55ab66f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,90 @@ +.. title:: pytest-mpl documentation + +.. module:: pytest-mpl + +.. toctree:: + :hidden: + + installing + usage + configuration + summaries + +################################## +pytest-mpl |release| documentation +################################## + +This is a plugin to facilitate image comparison for Matplotlib figures in pytest. + +************ +Installation +************ + +.. grid:: 1 1 2 2 + + .. grid-item:: + + Install using `pip `__: + + .. code-block:: bash + + pip install pytest-mpl + + .. grid-item:: + + Install from `conda-forge `__ using `conda `__: + + .. code-block:: bash + + conda install pytest-mpl + +Further details are available in the :doc:`Installation Guide `. + + +****************** +Learning resources +****************** + +.. grid:: 1 1 2 2 + + .. grid-item-card:: + :padding: 2 + + Tutorials + ^^^ + + - :doc:`Basic usage ` + + .. grid-item-card:: + :padding: 2 + + How-tos + ^^^ + + + .. grid-item-card:: + :padding: 2 + + Understand how pytest-mpl works + ^^^ + + + .. grid-item-card:: + :padding: 2 + + Reference + ^^^ + + - :doc:`Configuration ` + - :doc:`Summary Reports ` + + +************ +Contributing +************ + +pytest-mpl is a community project maintained for and by its users. +There are many ways you can help! + +- Report a bug or request a feature `on GitHub `__ +- Improve the documentation or code diff --git a/docs/installing.rst b/docs/installing.rst new file mode 100644 index 00000000..9bd0279c --- /dev/null +++ b/docs/installing.rst @@ -0,0 +1,86 @@ +.. title:: Installation Guide + +################## +Installation Guide +################## + +This plugin is compatible with Python 3.6 and later, and +requires `pytest `__ and +`matplotlib `__ to be installed. + +Using pip +========= + +``pytest-mpl`` can be installed with ``pip``: + +.. code-block:: bash + + pip install pytest-mpl + + +Using conda +=========== + +Installing ``pytest-mpl`` from the ``conda-forge`` channel can be achieved by adding ``conda-forge`` to your channels with: + +.. code-block:: bash + + conda config --add channels conda-forge + conda config --set channel_priority strict + +Once the ``conda-forge`` channel has been enabled, ``pytest-mpl`` can be installed with ``conda``: + +.. code-block:: bash + + conda install pytest-mpl + +or with ``mamba``: + +.. code-block:: bash + + mamba install pytest-mpl + +It is possible to list all of the versions of ``pytest-mpl`` available on your platform with ``conda``: + +.. code-block:: bash + + conda search pytest-mpl --channel conda-forge + +or with ``mamba``: + +.. code-block:: bash + + mamba search pytest-mpl --channel conda-forge + +Alternatively, ``mamba repoquery`` may provide more information: + +.. code-block:: bash + + # Search all versions available on your platform: + mamba repoquery search pytest-mpl --channel conda-forge + + # List packages depending on pytest-mpl: + mamba repoquery whoneeds pytest-mpl --channel conda-forge + + # List dependencies of pytest-mpl: + mamba repoquery depends pytest-mpl --channel conda-forge + +Installing the development version +================================== + +Clone the `pytest-mpl GitHub repository `__, or your own fork of it. +Then install ``pytest-mpl`` using ``pip`` from the root directory of the repo: + +.. code-block:: bash + + pip install -e ".[test,docs]" + + +Troubleshooting +=============== + +To check that ``pytest-mpl`` has been installed correctly and is recognised by ``pytest``, run: + +.. code-block:: bash + + pytest --trace-config diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/sample_summaries.py b/docs/sample_summaries.py new file mode 100644 index 00000000..0c4f9d8d --- /dev/null +++ b/docs/sample_summaries.py @@ -0,0 +1,112 @@ +import os +import shutil +import pathlib +import tempfile +import subprocess + +from docutils import nodes +from sphinx.util.docutils import SphinxRole +from sphinx.util.osutil import canon_path + +REPO_ROOT = pathlib.Path(__file__).parent.parent +TEST_FILE = REPO_ROOT / "tests" / "subtests" / "test_subtest.py" +SAMPLE_DIR = "sample" + + +def run_pytest(test_name): + + # Create generated samples directory + sample_dir_abs = pathlib.Path(__file__).parent / SAMPLE_DIR + if not sample_dir_abs.exists(): + os.mkdir(sample_dir_abs) + + # Form path to current sample + dest = sample_dir_abs / test_name + if dest.exists(): + return dest # skip if already generated + + # Generate the current sample + tmp_dir = tempfile.mkdtemp() + command = f"python -m pytest {TEST_FILE}::{test_name} -v --mpl --basetemp={tmp_dir}" + subprocess.run(command, shell=True, check=True) + + # Find the name of the directory the sample is within + # (directory name is sometimes truncated) + src = next(filter( + lambda x: x.name[:-1] in test_name, + pathlib.Path(tmp_dir).glob("*0") + )) / "results" + os.replace(src, dest) + + return dest + + +class SummaryButtons(nodes.General, nodes.Inline, nodes.TextElement): + pass + + +class SummaryRole(SphinxRole): + def run(self): + node = SummaryButtons(name=self.text) + return [node], [] + + +def move_summaries(app, *args, **kwargs): + gen_sample_dir = pathlib.Path(__file__).parent / SAMPLE_DIR + out_sample_dir = pathlib.Path(app.outdir) / SAMPLE_DIR + if out_sample_dir.exists(): + shutil.rmtree(out_sample_dir) + shutil.copytree(gen_sample_dir, out_sample_dir) + + +def html_visit_summary(self, node): + + test_name = str(node["name"]) + out = run_pytest(test_name) + + classes = ( + "sd-sphinx-override sd-btn sd-text-wrap sd-btn-{importance} " + "sd-shadow-sm sd-me-2 reference internal" + ) + button = ( + '{label}' + ) + + summary_types = { + "HTML": "fig_comparison.html", + "Basic HTML": "fig_comparison_basic.html", + "JSON": "results.json", + } + + current_filename = self.builder.current_docname + self.builder.out_suffix + current_dir = pathlib.PurePath(current_filename).parent + first_button = True + for label, file in summary_types.items(): + if (out / file).exists(): + importance = "primary" if first_button else "secondary" + self.body.append(button.format( + classes=classes.format(importance=importance), + href=canon_path((current_dir / SAMPLE_DIR / test_name / file).as_posix()), + label=label, + )) + first_button = False + + raise nodes.SkipNode + + +def skip(self, node): + raise nodes.SkipNode + + +def setup(app): + app.connect("build-finished", move_summaries) + app.add_node( + SummaryButtons, + html=(html_visit_summary, None), + latex=(skip, None), + text=(skip, None), + man=(skip, None), + texinfo=(skip, None), + ) + app.add_role("summary", SummaryRole()) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/summaries.rst b/docs/summaries.rst new file mode 100644 index 00000000..add3e6c9 --- /dev/null +++ b/docs/summaries.rst @@ -0,0 +1,95 @@ +.. title:: Summary Reports + +############### +Summary Reports +############### + +Generating a Test Summary +^^^^^^^^^^^^^^^^^^^^^^^^^ + +By specifying the ``--mpl-generate-summary=html`` CLI argument, a HTML summary +page will be generated showing the test result, log entry and generated result +image. When in the (default) image comparison mode, the baseline image, diff +image and RMS (if any), and tolerance of each test will also be shown. +When in the hash comparison mode, the baseline hash and result hash will +also be shown. When in hybrid mode, all of these are included. + +When generating a HTML summary, the ``--mpl-results-always`` option is +automatically applied (see section below). Therefore images for passing +tests will also be shown. + ++---------------+---------------+---------------+ +| |html all| | |html filter| | |html result| | ++---------------+---------------+---------------+ + +As well as ``html``, ``basic-html`` can be specified for an alternative HTML +summary which does not rely on JavaScript or external resources. A ``json`` +summary can also be saved. Multiple options can be specified comma-separated. + +.. card:: Image comparison only + + .. code-block:: bash + + pytest --mpl --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html_images_only` + +.. card:: Hash comparison only + + .. code-block:: bash + + pytest --mpl --mpl-hash-library=mpl35_ft261.json --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html_hashes_only` + +.. card:: Hybrid mode: hash and image comparison + + .. code-block:: bash + + pytest --mpl --mpl-hash-library=mpl35_ft261.json --mpl-baseline-path=baseline --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html` + +.. card:: Generating baseline images and hashes (With no testing) + + .. code-block:: bash + + pytest --mpl --mpl-generate-path=baseline --mpl-generate-hash-library=test_hashes.json --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html_generate` + +.. card:: Generating baseline images (With no testing) + + .. code-block:: bash + + pytest --mpl --mpl-generate-path=baseline --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html_generate_images_only` + +.. card:: Generating baseline hashes (With image comparison) + + .. code-block:: bash + + pytest --mpl --mpl-generate-hash-library=test_hashes.json --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html_generate_hashes_only` + +.. card:: Generating baseline hashes (With hash comparison) + + .. code-block:: bash + + pytest --mpl --mpl-generate-hash-library=test_hashes.json --mpl-hash-library=mpl35_ft261.json --mpl-results-path=results --mpl-generate-summary=html,json + + :summary:`test_html_run_generate_hashes_only` + +.. card:: Hybrid mode: hash and image comparison + + .. code-block:: bash + + pytest --mpl --mpl-hash-library=mpl35_ft261.json --mpl-baseline-path=baseline --mpl-results-path=results --mpl-generate-summary=basic-html,json + + :summary:`test_basic_html` + +.. |html all| image:: images/html_all.png +.. |html filter| image:: images/html_filter.png +.. |html result| image:: images/html_result.png diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 00000000..66c7e979 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,88 @@ +.. title:: Basic Usage + +########### +Basic Usage +########### + +For each figure to test, the reference image is subtracted from the generated image, and the RMS of the residual is compared to a user-specified tolerance. If the residual is too large, the test will fail (this is implemented using helper functions from ``matplotlib.testing``). + +With Baseline Images +^^^^^^^^^^^^^^^^^^^^ + +To use, you simply need to mark the function where you want to compare +images using ``@pytest.mark.mpl_image_compare``, and make sure that the +function returns a Matplotlib figure (or any figure object that has a +``savefig`` method): + +.. code:: python + + import pytest + import matplotlib.pyplot as plt + + @pytest.mark.mpl_image_compare + def test_succeeds(): + fig = plt.figure() + ax = fig.add_subplot(1,1,1) + ax.plot([1,2,3]) + return fig + +To generate the baseline images, run the tests with the +``--mpl-generate-path`` option with the name of the directory where the +generated images should be placed:: + + pytest --mpl-generate-path=baseline + +If the directory does not exist, it will be created. The directory will +be interpreted as being relative to where you are running ``pytest``. +Once you are happy with the generated images, you should move them to a +sub-directory called ``baseline`` relative to the test files (this name +is configurable, see below). You can also generate the baseline image +directly in the right directory. + +With a Hash Library +^^^^^^^^^^^^^^^^^^^ + +Instead of comparing to baseline images, you can instead compare against a JSON +library of SHA-256 hashes. This has the advantage of not having to check baseline +images into the repository with the tests, or download them from a remote +source. + +The hash library can be generated with +``--mpl-generate-hash-library=path_to_file.json``. The hash library to be used +can either be specified via the ``--mpl-hash-library=`` command line argument, +or via the ``hash_library=`` keyword argument to the +``@pytest.mark.mpl_image_compare`` decorator. + +When generating a hash library, the tests will also be run as usual against the +existing hash library specified by ``--mpl-hash-library`` or the keyword argument. +However, generating baseline images will always result in the tests being skipped. + + +Hybrid Mode: Hashes and Images +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is possible to configure both hashes and baseline images. In this scenario +only the hash comparison can determine the test result. If the hash comparison +fails, the test will fail, however a comparison to the baseline image will be +carried out so the actual difference can be seen. If the hash comparison passes, +the comparison to the baseline image is skipped (unless **results always** is +configured). + +This is especially useful if the baseline images are external to the repository +containing the tests, and are accessed via HTTP. In this situation, if the hashes +match, the baseline images won't be retrieved, saving time and bandwidth. Also, it +allows the tests to be modified and the hashes updated to reflect the changes +without having to modify the external images. + + +Running Tests +^^^^^^^^^^^^^ + +Once tests are written with baseline images, a hash library, or both to compare +against, the tests can be run with:: + + pytest --mpl + +and the tests will pass if the images are the same. If you omit the +``--mpl`` option, the tests will run but will only check that the code +runs, without checking the output images. diff --git a/setup.cfg b/setup.cfg index b9eb7133..dc9f7b23 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,10 @@ pytest11 = [options.extras_require] test = pytest-cov +docs = + sphinx + mpl_sphinx_theme>=3.6.0.dev0 + sphinx_design [tool:pytest] testpaths = "tests"