pytest gallery tests (#4792)

* pytest parametrize gallery tests

* two context managers as fixtures

* use pathlib instead of os

* docstrings

* use pytest.mark.filterwarnings

* show_replaced_by_check_graphic as fixture

* Revert previous (more readable without)

This reverts commit bdca27c.

* gallery_dir function to variable

* add setup/teardown fixture

* tweak docstring

* remove gallerytests from Makefile

* update dev guide

* doc tweaks

* use monkeypatch to simplify

* move stuff about

* monkeypatch for sys.path; comments and param ids

* revert assert False

* revert deliberate fails

* import_patches fixture

* post rebase check_graphic fix

* update test references

* whatsnew

* test data version

Author: rcomer

.github/workflows/benchmark.yml

@@ -15,7 +15,7 @@ jobs:
     env:
       IRIS_TEST_DATA_LOC_PATH: benchmarks
       IRIS_TEST_DATA_PATH: benchmarks/iris-test-data
-      IRIS_TEST_DATA_VERSION: "2.14"
+      IRIS_TEST_DATA_VERSION: "2.15"
       # Lets us manually bump the cache to rebuild
       ENV_CACHE_BUILD: "0"
       TEST_DATA_CACHE_BUILD: "2"

.github/workflows/ci-tests.yml

@@ -46,7 +46,7 @@ jobs:
             session: "tests"
 
     env:
-      IRIS_TEST_DATA_VERSION: "2.14"
+      IRIS_TEST_DATA_VERSION: "2.15"
       ENV_NAME: "ci-tests"
 
     steps:

docs/Makefile

@@ -55,8 +55,3 @@ linkcheck:
 		echo "Running linkcheck in $$i..."; \
 		(cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) linkcheck); done
 
-gallerytest:
-	@echo
-	@echo "Running \"gallery\" tests..."
-	@echo
-	python -m unittest discover -v -t . 

docs/gallery_tests/conftest.py

@@ -0,0 +1,67 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+
+"""Pytest fixtures for the gallery tests."""
+
+import pathlib
+
+import matplotlib.pyplot as plt
+import pytest
+
+import iris
+
+CURRENT_DIR = pathlib.Path(__file__).resolve()
+GALLERY_DIR = CURRENT_DIR.parents[1] / "gallery_code"
+
+
+@pytest.fixture
+def image_setup_teardown():
+    """
+    Setup and teardown fixture.
+
+    Ensures all figures are closed before and after test to prevent one test
+    polluting another if it fails with a figure unclosed.
+
+    """
+    plt.close("all")
+    yield
+    plt.close("all")
+
+
+@pytest.fixture
+def import_patches(monkeypatch):
+    """
+    Replace plt.show() with a function that does nothing, also add all the
+    gallery examples to sys.path.
+
+    """
+
+    def no_show():
+        pass
+
+    monkeypatch.setattr(plt, "show", no_show)
+
+    for example_dir in GALLERY_DIR.iterdir():
+        if example_dir.is_dir():
+            monkeypatch.syspath_prepend(example_dir)
+
+    yield
+
+
+@pytest.fixture
+def iris_future_defaults():
+    """
+    Create a fixture which resets all the iris.FUTURE settings to the defaults,
+    as otherwise changes made in one test can affect subsequent ones.
+
+    """
+    # Run with all default settings in iris.FUTURE.
+    default_future_kwargs = iris.Future().__dict__.copy()
+    for dead_option in iris.Future.deprecated_options:
+        # Avoid a warning when setting these !
+        del default_future_kwargs[dead_option]
+    with iris.FUTURE.context(**default_future_kwargs):
+        yield

docs/gallery_tests/gallerytest_util.py

@@ -0,0 +1,44 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+
+import importlib
+
+import matplotlib.pyplot as plt
+import pytest
+
+from iris.tests import _RESULT_PATH
+from iris.tests.graphics import check_graphic
+
+from .conftest import GALLERY_DIR
+
+
+def gallery_examples():
+    """Generator to yield all current gallery examples."""
+
+    for example_file in GALLERY_DIR.glob("*/plot*.py"):
+        yield example_file.stem
+
+
+@pytest.mark.filterwarnings("error::iris.IrisDeprecation")
+@pytest.mark.parametrize("example", gallery_examples())
+def test_plot_example(
+    example,
+    image_setup_teardown,
+    import_patches,
+    iris_future_defaults,
+):
+    """Test that all figures from example code match KGO."""
+
+    module = importlib.import_module(example)
+
+    # Run example.
+    module.main()
+    # Loop through open figures and set each to be the current figure so check_graphic
+    # will find it.
+    for fig_num in plt.get_fignums():
+        plt.figure(fig_num)
+        image_id = f"gallery_tests.test_{example}.{fig_num - 1}"
+        check_graphic(image_id, _RESULT_PATH)