{{ filter_option('image-match', 'matching images') }}
{{ filter_option('image-diff', 'differing images') }}
{{ filter_option('image-missing', 'no baseline image') }}
+ {%- endif %}
+ {% if results.warn_missing['baseline_hash'] -%}
{{ filter_option('hash-match', 'matching hashes') }}
{{ filter_option('hash-diff', 'differing hashes') }}
{{ filter_option('hash-missing', 'no baseline hash') }}
+ {%- endif %}
{{ r.name }}
+ {% if results.warn_missing['baseline_image'] -%}
{% macro image_card(file, name) -%}
@@ -21,6 +22,7 @@
{{ r.name }}
{{ image_card(r.diff_image, 'Diff') }}
{{ image_card(r.result_image, 'Result') }}
+ {%- endif %}
@@ -31,19 +33,24 @@
{{ r.name }}
+ {% macro pre_data(id, value, name) -%}
+ {% filter indent(width=8) -%}
+
+
{{ name }}
+
{{ value }}
+
+ {%- endfilter %}
+ {%- endmacro -%}
+ {% if results.warn_missing['baseline_image'] -%}
- {% macro pre_data(id, value, name) -%}
-
-
{{ name }}
-
{{ value }}
-
- {%- endmacro -%}
{{ pre_data('rms', r.rms_str, 'RMS') }}
{{ pre_data('tolerance', r.tolerance, 'Tolerance') }}
+ {%- endif %}
+ {% if results.warn_missing['baseline_hash'] -%}
@@ -51,6 +58,7 @@
{{ r.name }}
{{ pre_data('result_hash', r.result_hash, 'Result') }}
+ {%- endif %}
diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css
index 7a07169e..3effb1df 100644
--- a/pytest_mpl/summary/templates/styles.css
+++ b/pytest_mpl/summary/templates/styles.css
@@ -33,7 +33,7 @@ body.no-hash-test .mpl-hash {
display: none;
}
pre {
- white-space: pre-wrap;
+ white-space: pre-line;
}
div.result div.status-badge button img {
vertical-align: sub;
diff --git a/tests/subtests/__init__.py b/tests/subtests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/subtests/baseline/test_hdiff_idiff.png b/tests/subtests/baseline/test_hdiff_idiff.png
new file mode 100644
index 00000000..b377b071
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_idiff.png differ
diff --git a/tests/subtests/baseline/test_hdiff_idiff_tolerance.png b/tests/subtests/baseline/test_hdiff_idiff_tolerance.png
new file mode 100644
index 00000000..516801e0
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_idiff_tolerance.png differ
diff --git a/tests/subtests/baseline/test_hdiff_imatch.png b/tests/subtests/baseline/test_hdiff_imatch.png
new file mode 100644
index 00000000..c1a2a3ef
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_imatch.png differ
diff --git a/tests/subtests/baseline/test_hdiff_imatch_removetext.png b/tests/subtests/baseline/test_hdiff_imatch_removetext.png
new file mode 100644
index 00000000..6d59b1d4
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_imatch_removetext.png differ
diff --git a/tests/subtests/baseline/test_hdiff_imatch_savefig.png b/tests/subtests/baseline/test_hdiff_imatch_savefig.png
new file mode 100644
index 00000000..c791a4b1
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_imatch_savefig.png differ
diff --git a/tests/subtests/baseline/test_hdiff_imatch_style.png b/tests/subtests/baseline/test_hdiff_imatch_style.png
new file mode 100644
index 00000000..49f4adee
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_imatch_style.png differ
diff --git a/tests/subtests/baseline/test_hdiff_imatch_tolerance.png b/tests/subtests/baseline/test_hdiff_imatch_tolerance.png
new file mode 100644
index 00000000..516801e0
Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_imatch_tolerance.png differ
diff --git a/tests/subtests/baseline/test_hmatch_idiff.png b/tests/subtests/baseline/test_hmatch_idiff.png
new file mode 100644
index 00000000..028c9be4
Binary files /dev/null and b/tests/subtests/baseline/test_hmatch_idiff.png differ
diff --git a/tests/subtests/baseline/test_hmatch_imatch.png b/tests/subtests/baseline/test_hmatch_imatch.png
new file mode 100644
index 00000000..338c53ae
Binary files /dev/null and b/tests/subtests/baseline/test_hmatch_imatch.png differ
diff --git a/tests/subtests/baseline/test_hmissing_idiff.png b/tests/subtests/baseline/test_hmissing_idiff.png
new file mode 100644
index 00000000..15ea61d1
Binary files /dev/null and b/tests/subtests/baseline/test_hmissing_idiff.png differ
diff --git a/tests/subtests/baseline/test_hmissing_imatch.png b/tests/subtests/baseline/test_hmissing_imatch.png
new file mode 100644
index 00000000..03483cbc
Binary files /dev/null and b/tests/subtests/baseline/test_hmissing_imatch.png differ
diff --git a/tests/subtests/hashes/mpl33_ft261.json b/tests/subtests/hashes/mpl33_ft261.json
new file mode 100644
index 00000000..e285cd6f
--- /dev/null
+++ b/tests/subtests/hashes/mpl33_ft261.json
@@ -0,0 +1,13 @@
+{
+ "subtests.subtest.test_hmatch_imatch": "42c391b37022e2c4edb53f5fd988e94f421905b40cea1544e62ffb3c049292a8",
+ "subtests.subtest.test_hmatch_idiff": "c14ba098dbda0988e35be5724ffb15b8e666253a4b37dec6a21203607c17473d",
+ "subtests.subtest.test_hmatch_imissing": "6c07931bac1a926c88bea5d07c40c8c1ce30648712e3fc963028193863e3ae65",
+ "subtests.subtest.test_hdiff_imatch": "d1ff383721a0c395c856302be7de8a8138a2693651425dc181ede262860aef7b",
+ "subtests.subtest.test_hdiff_idiff": "d1fff55ace5ef7e45dcd9913b54e0d9970028cae59666e937ccb3586d0f76e9a",
+ "subtests.subtest.test_hdiff_imissing": "d1ff35845c5887c034230e02aa4b60e053c779c693867e4803e1d72dde9240f7",
+ "subtests.subtest.test_hdiff_imatch_tolerance": "d1ff6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",
+ "subtests.subtest.test_hdiff_idiff_tolerance": "d1ff6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",
+ "subtests.subtest.test_hdiff_imatch_savefig": "d1ff5dc3f9e8acda06b0097ee893819be62ca9adbbcca7d2300602f079a93b92",
+ "subtests.subtest.test_hdiff_imatch_style": "d1ff7692747ec72d3c8669cdb3d66468426b83ecf49a214cd918b8f5a0752a1f",
+ "subtests.subtest.test_hdiff_imatch_removetext": "d1ff0d60d794a7cdfec884463c4fe14612ab1fe7fda4bc7fa702c8f1615e1539"
+}
diff --git a/tests/subtests/hashes/mpl34_ft261.json b/tests/subtests/hashes/mpl34_ft261.json
new file mode 100644
index 00000000..a8012e51
--- /dev/null
+++ b/tests/subtests/hashes/mpl34_ft261.json
@@ -0,0 +1,13 @@
+{
+ "subtests.subtest.test_hmatch_imatch": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
+ "subtests.subtest.test_hmatch_idiff": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
+ "subtests.subtest.test_hmatch_imissing": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
+ "subtests.subtest.test_hdiff_imatch": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
+ "subtests.subtest.test_hdiff_idiff": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
+ "subtests.subtest.test_hdiff_imissing": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
+ "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "subtests.subtest.test_hdiff_imatch_savefig": "d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568",
+ "subtests.subtest.test_hdiff_imatch_style": "d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7",
+ "subtests.subtest.test_hdiff_imatch_removetext": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
+}
diff --git a/tests/subtests/hashes/mpl35_ft261.json b/tests/subtests/hashes/mpl35_ft261.json
new file mode 100644
index 00000000..2ae41a7e
--- /dev/null
+++ b/tests/subtests/hashes/mpl35_ft261.json
@@ -0,0 +1,13 @@
+{
+ "subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
+ "subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
+ "subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
+ "subtests.subtest.test_hdiff_imatch": "d1ff8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
+ "subtests.subtest.test_hdiff_idiff": "d1ff21206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
+ "subtests.subtest.test_hdiff_imissing": "d1ff11cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
+ "subtests.subtest.test_hdiff_imatch_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
+ "subtests.subtest.test_hdiff_idiff_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
+ "subtests.subtest.test_hdiff_imatch_savefig": "d1ff803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
+ "subtests.subtest.test_hdiff_imatch_style": "d1ffde36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
+ "subtests.subtest.test_hdiff_imatch_removetext": "d1ff6cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a"
+}
diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py
new file mode 100644
index 00000000..0c688843
--- /dev/null
+++ b/tests/subtests/helpers.py
@@ -0,0 +1,250 @@
+import re
+import json
+from pathlib import Path
+
+__all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex']
+
+
+class MatchError(Exception):
+ pass
+
+
+def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None):
+ """Diff a pytest-mpl summary dictionary.
+
+ Parameters
+ ----------
+ baseline : dict
+ Baseline pytest-mpl summary.
+ result : dict
+ Generated result pytest-mpl summary.
+ baseline_hash_library : Path, optional, default=None
+ Path to the baseline hash library.
+ Baseline hashes in the baseline summary are updated to these values
+ to handle different Matplotlib versions.
+ result_hash_library : Path, optional, default=None
+ Path to the "baseline" image hash library.
+ Result hashes in the baseline summary are updated to these values
+ to handle different Matplotlib versions.
+ """
+ if baseline_hash_library and baseline_hash_library.exists():
+ # Load "correct" baseline hashes
+ with open(baseline_hash_library, 'r') as f:
+ baseline_hash_library = json.load(f)
+ else:
+ baseline_hash_library = {}
+ if result_hash_library and result_hash_library.exists():
+ # Load "correct" result hashes
+ with open(result_hash_library, 'r') as f:
+ result_hash_library = json.load(f)
+ else:
+ result_hash_library = {}
+
+ # Get test names
+ baseline_tests = set(baseline.keys())
+ result_tests = set(result.keys())
+
+ # Test names must be identical
+ diff_set(baseline_tests, result_tests, error='Test names are not identical.')
+
+ item_match_errors = [] # Raise a MatchError for all mismatched values at the end
+
+ for test in baseline_tests:
+
+ # Get baseline and result summary for the specific test
+ baseline_summary = baseline[test]
+ result_summary = result[test]
+
+ # Swap the baseline and result hashes in the summary
+ # for the corresponding hashes in each hash library
+ if baseline_hash_library and test in baseline_hash_library:
+ baseline_summary = replace_hash(baseline_summary, 'baseline_hash',
+ baseline_hash_library[test])
+ if result_hash_library:
+ baseline_summary = replace_hash(baseline_summary, 'result_hash',
+ result_hash_library[test])
+
+ # Get keys of recorded items
+ baseline_keys = set(baseline_summary.keys())
+ result_keys = set(result_summary.keys())
+
+ # Summaries must have the same keys
+ diff_set(baseline_keys, result_keys, error=f'Summary for {test} is not identical.')
+
+ for key in baseline_keys:
+ error = f'Summary item {key} for {test} does not match.\n'
+ try:
+ diff_dict_item(baseline_summary[key], result_summary[key], error=error)
+ except MatchError as e:
+ item_match_errors.append(str(e))
+
+ if len(item_match_errors) > 0:
+ raise MatchError('\n\n----------\n\n'.join(item_match_errors))
+
+
+def diff_set(baseline, result, error=''):
+ """Raise and show the difference between Python sets."""
+ if baseline != result:
+ missing_from_result = baseline - result
+ missing_from_baseline = result - baseline
+ if len(missing_from_result) > 0:
+ error += f'\nKeys {sorted(missing_from_result)} missing from the result.'
+ if len(missing_from_baseline) > 0:
+ error += f'\nKeys {sorted(missing_from_baseline)} missing from the baseline.'
+ raise MatchError(error)
+
+
+def diff_dict_item(baseline, result, error=''):
+ """Diff a specific item in a pytest-mpl summary dictionary."""
+ # Comparison makes the following (good) assumptions
+ expected_types = (str, int, float, bool, type(None))
+ assert isinstance(baseline, expected_types)
+ assert isinstance(result, expected_types)
+
+ # Prepare error message
+ error += f'Baseline:\n"{baseline}"\n\n'
+ error += f'Result:\n"{result}"\n'
+
+ # Matching items must have the same type
+ if type(baseline) is not type(result):
+ raise MatchError(error + '\nTypes are not equal.\n')
+
+ # Handle regex in baseline string (so things like paths can be ignored)
+ if isinstance(baseline, str) and baseline.startswith('REGEX:'):
+ if re.fullmatch(baseline[6:], result) is not None:
+ return
+
+ # Handle bool and NoneType
+ if isinstance(baseline, (bool, type(None))) and baseline is result:
+ return
+
+ # Handle float
+ if isinstance(baseline, float) and abs(baseline - result) < 1e-4:
+ return
+
+ # Handle str and int
+ if baseline == result:
+ return
+
+ raise MatchError(error)
+
+
+def patch_summary(summary, patch_file):
+ """Replace in `summary` any items defined in `patch_file`."""
+ # By only applying patches, changes between MPL versions are more obvious.
+ with open(patch_file, 'r') as f:
+ patch = json.load(f)
+ for test, test_summary in patch.items():
+ for k, v in test_summary.items():
+ summary[test][k] = v
+ return summary
+
+
+def replace_hash(summary, hash_key, new_hash):
+ """Replace a hash in a pytest-mpl summary with a different hash.
+
+ Parameters
+ ----------
+ summary : dict
+ A single test from a pytest-mpl summary.
+ hash_key : str
+ Key of the hash. Either `baseline_hash` or `result_hash`.
+ new_hash : str
+ The new hash.
+ """
+ assert isinstance(new_hash, str)
+ old_hash = summary[hash_key]
+ if not isinstance(old_hash, str) or old_hash == new_hash:
+ return summary # Either already correct or missing
+
+ # Update the hash
+ summary[hash_key] = new_hash
+ summary['status_msg'] = summary['status_msg'].replace(old_hash, new_hash)
+
+ return summary
+
+
+def assert_existence(summary, items=('baseline_image', 'diff_image', 'result_image'), path=''):
+ """Assert that images included in a pytest-mpl summary exist.
+
+ Parameters
+ ----------
+ summary : dict
+ The pytest-mpl summary dictionary to check.
+ items : tuple or list, optional
+ The image keys to check if reported.
+ path : str or path_like, optional, default=''
+ Path to results directory. Defaults to current directory.
+ """
+ for test in summary.values():
+ for item in items:
+ if test[item] is not None:
+ assert (Path(path) / test[item]).exists()
+
+
+def _escape_regex(msg):
+ if not msg.startswith('REGEX:'):
+ msg = msg.replace('.', r'\.').replace('(', r'\(').replace(')', r'\)')
+ msg = 'REGEX:' + msg
+ return msg
+
+
+def _escape_path(msg, path):
+ pattern = (rf"({path}[A-Za-z0-9_\-\/.\\]*)" +
+ r"(baseline\\.png|result-failed-diff\\.png|result\\.png|\\.json)")
+ msg = re.sub(pattern, r".*\2", msg)
+ pattern = rf"({path}[A-Za-z0-9_\-\/.\\]*)"
+ msg = re.sub(pattern, r".*", msg)
+ return msg
+
+
+def _escape_float(msg, key):
+ pattern = rf"({key}[0-9]+\\\.[0-9]{{1}})([0-9]+)"
+ msg = re.sub(pattern, r"\1[0-9]*", msg)
+ return msg
+
+
+def apply_regex(file, regex_paths, regex_strs):
+ """Convert all `status_msg` entries in JSON summary file to regex.
+
+ Use in your own script to assist with updating baseline summaries.
+
+ Parameters
+ ----------
+ file : Path
+ JSON summary file to convert `status_msg` to regex in. Overwritten.
+ regex_paths : list of str
+ List of path beginnings to identify paths that need to be converted to regex.
+ E.g. `['/home/user/']`
+ Does: `aaa /home/user/pytest/tmp/result\\.png bbb` -> `aaa .*result\\.png bbb`
+ regex_strs : list of str
+ List of keys to convert following floats to 1 d.p.
+ E.g. ['RMS Value: ']
+ Does: `aaa RMS Value: 12\\.432644 bbb` -> `aaa RMS Value: 12\\.4[0-9]* bbb`
+ """
+
+ with open(file, 'r') as f:
+ summary = json.load(f)
+
+ for test in summary.keys():
+
+ msg = summary[test]['status_msg']
+
+ for signal in [*regex_paths, *regex_strs]:
+ if signal in msg:
+ msg = _escape_regex(msg)
+ if not msg.startswith('REGEX:'):
+ continue
+
+ for signal in regex_paths:
+ if signal in msg:
+ msg = _escape_path(msg, path=signal)
+
+ for signal in regex_strs:
+ if signal in msg:
+ msg = _escape_float(msg, key=signal)
+
+ summary[test]['status_msg'] = msg
+
+ with open(file, 'w') as f:
+ json.dump(summary, f, indent=2)
diff --git a/tests/subtests/result_hashes/mpl33_ft261.json b/tests/subtests/result_hashes/mpl33_ft261.json
new file mode 100644
index 00000000..b5bf8b7a
--- /dev/null
+++ b/tests/subtests/result_hashes/mpl33_ft261.json
@@ -0,0 +1,16 @@
+{
+ "subtests.subtest.test_hmatch_imatch": "42c391b37022e2c4edb53f5fd988e94f421905b40cea1544e62ffb3c049292a8",
+ "subtests.subtest.test_hmatch_idiff": "c14ba098dbda0988e35be5724ffb15b8e666253a4b37dec6a21203607c17473d",
+ "subtests.subtest.test_hmatch_imissing": "6c07931bac1a926c88bea5d07c40c8c1ce30648712e3fc963028193863e3ae65",
+ "subtests.subtest.test_hdiff_imatch": "b48a383721a0c395c856302be7de8a8138a2693651425dc181ede262860aef7b",
+ "subtests.subtest.test_hdiff_idiff": "44edf55ace5ef7e45dcd9913b54e0d9970028cae59666e937ccb3586d0f76e9a",
+ "subtests.subtest.test_hdiff_imissing": "042235845c5887c034230e02aa4b60e053c779c693867e4803e1d72dde9240f7",
+ "subtests.subtest.test_hmissing_imatch": "d87e9aa72d2cd0ccd9959ec7af2dc00ab86e2fe8f06a6c54a017cde62f7e5c1f",
+ "subtests.subtest.test_hmissing_idiff": "00b074ef61b5061b8ec50dbe32b3214ffd21c9489ae212671558f338ad792e5f",
+ "subtests.subtest.test_hmissing_imissing": "3e5c4a4386e32133083c36ca0c95f5e38be904ed238e49ab1e06edd35603abfc",
+ "subtests.subtest.test_hdiff_imatch_tolerance": "04eb6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",
+ "subtests.subtest.test_hdiff_idiff_tolerance": "04eb6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",
+ "subtests.subtest.test_hdiff_imatch_savefig": "c47d5dc3f9e8acda06b0097ee893819be62ca9adbbcca7d2300602f079a93b92",
+ "subtests.subtest.test_hdiff_imatch_style": "8fa57692747ec72d3c8669cdb3d66468426b83ecf49a214cd918b8f5a0752a1f",
+ "subtests.subtest.test_hdiff_imatch_removetext": "f9fc0d60d794a7cdfec884463c4fe14612ab1fe7fda4bc7fa702c8f1615e1539"
+}
diff --git a/tests/subtests/result_hashes/mpl34_ft261.json b/tests/subtests/result_hashes/mpl34_ft261.json
new file mode 100644
index 00000000..b30b198a
--- /dev/null
+++ b/tests/subtests/result_hashes/mpl34_ft261.json
@@ -0,0 +1,16 @@
+{
+ "subtests.subtest.test_hmatch_imatch": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
+ "subtests.subtest.test_hmatch_idiff": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
+ "subtests.subtest.test_hmatch_imissing": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
+ "subtests.subtest.test_hdiff_imatch": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
+ "subtests.subtest.test_hdiff_idiff": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
+ "subtests.subtest.test_hdiff_imissing": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
+ "subtests.subtest.test_hmissing_imatch": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824",
+ "subtests.subtest.test_hmissing_idiff": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab",
+ "subtests.subtest.test_hmissing_imissing": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b",
+ "subtests.subtest.test_hdiff_imatch_tolerance": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "subtests.subtest.test_hdiff_idiff_tolerance": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "subtests.subtest.test_hdiff_imatch_savefig": "5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568",
+ "subtests.subtest.test_hdiff_imatch_style": "185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7",
+ "subtests.subtest.test_hdiff_imatch_removetext": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
+}
diff --git a/tests/subtests/result_hashes/mpl35_ft261.json b/tests/subtests/result_hashes/mpl35_ft261.json
new file mode 100644
index 00000000..edc8605a
--- /dev/null
+++ b/tests/subtests/result_hashes/mpl35_ft261.json
@@ -0,0 +1,16 @@
+{
+ "subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
+ "subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
+ "subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
+ "subtests.subtest.test_hdiff_imatch": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
+ "subtests.subtest.test_hdiff_idiff": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
+ "subtests.subtest.test_hdiff_imissing": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
+ "subtests.subtest.test_hmissing_imatch": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
+ "subtests.subtest.test_hmissing_idiff": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
+ "subtests.subtest.test_hmissing_imissing": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
+ "subtests.subtest.test_hdiff_imatch_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
+ "subtests.subtest.test_hdiff_idiff_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
+ "subtests.subtest.test_hdiff_imatch_savefig": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
+ "subtests.subtest.test_hdiff_imatch_style": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
+ "subtests.subtest.test_hdiff_imatch_removetext": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a"
+}
diff --git a/tests/subtests/subtest.py b/tests/subtests/subtest.py
new file mode 100644
index 00000000..ba927e57
--- /dev/null
+++ b/tests/subtests/subtest.py
@@ -0,0 +1,116 @@
+import matplotlib.pyplot as plt
+import pytest
+
+
+def plot(line, **kwargs):
+ fig = plt.figure()
+ ax = fig.add_subplot(1, 1, 1)
+ ax.plot(line, **kwargs)
+ return fig
+
+
+# ### Test all permutations of:
+# baseline hash: match, diff, or missing
+# baseline image: match, diff, or missing
+
+# hash match
+
+@pytest.mark.mpl_image_compare()
+def test_hmatch_imatch():
+ return plot([1, 2, 3, 4])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmatch_idiff():
+ return plot([1, 3, 2, 4])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmatch_imissing():
+ return plot([4, 3, 2, 1])
+
+
+# hash diff
+
+@pytest.mark.mpl_image_compare()
+def test_hdiff_imatch():
+ return plot([1, 4, 2, 3])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hdiff_idiff():
+ return plot([1, 2, 4, 3])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hdiff_imissing():
+ return plot([3, 2, 4, 1])
+
+
+# hash missing
+
+@pytest.mark.mpl_image_compare()
+def test_hmissing_imatch():
+ return plot([1, 3, 4, 2])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmissing_idiff():
+ return plot([1, 4, 3, 2])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmissing_imissing():
+ return plot([2, 4, 3, 1])
+
+
+# ### Specialized tests
+
+# Tolerance: high to force image match
+@pytest.mark.mpl_image_compare(tolerance=200)
+def test_hdiff_imatch_tolerance():
+ return plot([1, 2, 3, 4], linestyle='--')
+
+
+# Tolerance: non-default to verify option recorded in JSON
+@pytest.mark.mpl_image_compare(tolerance=3)
+def test_hdiff_idiff_tolerance():
+ return plot([1, 2, 3, 4], linestyle='--')
+
+
+# Savefig kwargs
+@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r'})
+def test_hdiff_imatch_savefig():
+ return plot([1, 2, 3, 4])
+
+
+# TODO: Implement these path altering tests later
+# # Different baseline directory
+# # TODO: Test with a remote `baseline_dir`
+# @pytest.mark.mpl_image_compare(baseline_dir='baseline/other')
+# def test_hdiff_imatch_baselinedir():
+# return plot([4, 2, 1, 4])
+#
+#
+# # Different filename
+# @pytest.mark.mpl_image_compare(filename='test_hdiff_imatch_filename_other.png')
+# def test_hdiff_imatch_filename():
+# return plot([4, 2, 1, 4])
+#
+#
+# # Different hash library
+# @pytest.mark.mpl_image_compare(hash_library='hashes/other/other.json')
+# def test_hdiff_imatch_hashlibrary():
+# return plot([4, 2, 1, 4])
+
+
+# Different style
+@pytest.mark.mpl_image_compare(style='fivethirtyeight')
+def test_hdiff_imatch_style():
+ return plot([4, 2, 1, 4])
+
+
+# Remove text
+@pytest.mark.mpl_image_compare(remove_text=True)
+def test_hdiff_imatch_removetext():
+ return plot([4, 2, 1, 4])
diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json
new file mode 100644
index 00000000..a4acadc6
--- /dev/null
+++ b/tests/subtests/summaries/test_default.json
@@ -0,0 +1,156 @@
+{
+ "subtests.subtest.test_hmatch_imatch": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmatch_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
+ "rms": 11.100353848213828,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmatch_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmatch_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmatch_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
+ "rms": 11.182677079602481,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmissing_imatch": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmissing_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
+ "rms": 12.12938597648977,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmissing_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_tolerance": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 200,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_idiff_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
+ "rms": 29.260332173249314,
+ "tolerance": 3,
+ "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_savefig": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_style": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_removetext": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": null,
+ "baseline_hash": null,
+ "result_hash": null
+ }
+}
diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json
new file mode 100644
index 00000000..bec04c5a
--- /dev/null
+++ b/tests/subtests/summaries/test_hash.json
@@ -0,0 +1,156 @@
+{
+ "subtests.subtest.test_hmatch_imatch": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": null,
+ "baseline_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
+ "result_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672"
+ },
+ "subtests.subtest.test_hmatch_idiff": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": null,
+ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
+ "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
+ },
+ "subtests.subtest.test_hmatch_imissing": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": null,
+ "baseline_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
+ "result_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f"
+ },
+ "subtests.subtest.test_hdiff_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch/result.png",
+ "baseline_hash": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
+ "result_hash": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1"
+ },
+ "subtests.subtest.test_hdiff_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
+ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
+ "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
+ },
+ "subtests.subtest.test_hdiff_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imissing/result.png",
+ "baseline_hash": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
+ "result_hash": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e"
+ },
+ "subtests.subtest.test_hmissing_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824"
+ },
+ "subtests.subtest.test_hmissing_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
+ },
+ "subtests.subtest.test_hmissing_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b"
+ },
+ "subtests.subtest.test_hdiff_imatch_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_idiff_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_imatch_savefig": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png",
+ "baseline_hash": "d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568",
+ "result_hash": "5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568"
+ },
+ "subtests.subtest.test_hdiff_imatch_style": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png",
+ "baseline_hash": "d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7",
+ "result_hash": "185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7"
+ },
+ "subtests.subtest.test_hdiff_imatch_removetext": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png",
+ "baseline_hash": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e",
+ "result_hash": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
+ }
+}
diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json
new file mode 100644
index 00000000..448ce9c2
--- /dev/null
+++ b/tests/subtests/summaries/test_html_hashes_only.json
@@ -0,0 +1,156 @@
+{
+ "subtests.subtest.test_hmatch_imatch": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmatch_imatch/result.png",
+ "baseline_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
+ "result_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672"
+ },
+ "subtests.subtest.test_hmatch_idiff": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmatch_idiff/result.png",
+ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
+ "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
+ },
+ "subtests.subtest.test_hmatch_imissing": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmatch_imissing/result.png",
+ "baseline_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
+ "result_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f"
+ },
+ "subtests.subtest.test_hdiff_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch/result.png",
+ "baseline_hash": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
+ "result_hash": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1"
+ },
+ "subtests.subtest.test_hdiff_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
+ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
+ "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
+ },
+ "subtests.subtest.test_hdiff_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imissing/result.png",
+ "baseline_hash": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
+ "result_hash": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e"
+ },
+ "subtests.subtest.test_hmissing_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824"
+ },
+ "subtests.subtest.test_hmissing_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
+ },
+ "subtests.subtest.test_hmissing_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b"
+ },
+ "subtests.subtest.test_hdiff_imatch_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_idiff_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_imatch_savefig": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png",
+ "baseline_hash": "d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568",
+ "result_hash": "5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568"
+ },
+ "subtests.subtest.test_hdiff_imatch_style": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png",
+ "baseline_hash": "d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7",
+ "result_hash": "185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7"
+ },
+ "subtests.subtest.test_hdiff_imatch_removetext": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png",
+ "baseline_hash": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e",
+ "result_hash": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
+ }
+}
diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json
new file mode 100644
index 00000000..c8a5058c
--- /dev/null
+++ b/tests/subtests/summaries/test_html_images_only.json
@@ -0,0 +1,156 @@
+{
+ "subtests.subtest.test_hmatch_imatch": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmatch_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmatch_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
+ "rms": 11.100353848213828,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmatch_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmatch_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmatch_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
+ "rms": 11.182677079602481,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmissing_imatch": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmissing_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
+ "rms": 12.12938597648977,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hmissing_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_tolerance": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 200,
+ "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_idiff_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
+ "rms": 29.260332173249314,
+ "tolerance": 3,
+ "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_savefig": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_style": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ },
+ "subtests.subtest.test_hdiff_imatch_removetext": {
+ "status": "passed",
+ "status_msg": "Image comparison passed.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png",
+ "baseline_hash": null,
+ "result_hash": null
+ }
+}
diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json
new file mode 100644
index 00000000..2f2930b0
--- /dev/null
+++ b/tests/subtests/summaries/test_hybrid.json
@@ -0,0 +1,156 @@
+{
+ "subtests.subtest.test_hmatch_imatch": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": null,
+ "baseline_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
+ "result_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672"
+ },
+ "subtests.subtest.test_hmatch_idiff": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": null,
+ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
+ "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
+ },
+ "subtests.subtest.test_hmatch_imissing": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": null,
+ "baseline_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
+ "result_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f"
+ },
+ "subtests.subtest.test_hdiff_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch/result.png",
+ "baseline_hash": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
+ "result_hash": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1"
+ },
+ "subtests.subtest.test_hdiff_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
+ "rms": 11.182677079602481,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
+ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
+ "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
+ },
+ "subtests.subtest.test_hdiff_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imissing/result.png",
+ "baseline_hash": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
+ "result_hash": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e"
+ },
+ "subtests.subtest.test_hmissing_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824"
+ },
+ "subtests.subtest.test_hmissing_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
+ "rms": 12.12938597648977,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
+ },
+ "subtests.subtest.test_hmissing_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b"
+ },
+ "subtests.subtest.test_hdiff_imatch_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 200,
+ "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_idiff_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
+ "rms": 29.260332173249314,
+ "tolerance": 3,
+ "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_imatch_savefig": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png",
+ "baseline_hash": "d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568",
+ "result_hash": "5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568"
+ },
+ "subtests.subtest.test_hdiff_imatch_style": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png",
+ "baseline_hash": "d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7",
+ "result_hash": "185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7"
+ },
+ "subtests.subtest.test_hdiff_imatch_removetext": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png",
+ "baseline_hash": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e",
+ "result_hash": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
+ }
+}
diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json
new file mode 100644
index 00000000..c42fee29
--- /dev/null
+++ b/tests/subtests/summaries/test_results_always.json
@@ -0,0 +1,156 @@
+{
+ "subtests.subtest.test_hmatch_imatch": {
+ "status": "passed",
+ "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.",
+ "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmatch_imatch/result.png",
+ "baseline_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
+ "result_hash": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672"
+ },
+ "subtests.subtest.test_hmatch_idiff": {
+ "status": "passed",
+ "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
+ "rms": 11.100353848213828,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmatch_idiff/result.png",
+ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
+ "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
+ },
+ "subtests.subtest.test_hmatch_imissing": {
+ "status": "passed",
+ "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmatch_imissing/result.png",
+ "baseline_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
+ "result_hash": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f"
+ },
+ "subtests.subtest.test_hdiff_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch/result.png",
+ "baseline_hash": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
+ "result_hash": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1"
+ },
+ "subtests.subtest.test_hdiff_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
+ "rms": 11.182677079602481,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
+ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
+ "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
+ },
+ "subtests.subtest.test_hdiff_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hdiff_imissing/result.png",
+ "baseline_hash": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
+ "result_hash": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e"
+ },
+ "subtests.subtest.test_hmissing_imatch": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_imatch/result.png",
+ "baseline_hash": null,
+ "result_hash": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824"
+ },
+ "subtests.subtest.test_hmissing_idiff": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
+ "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
+ "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
+ "rms": 12.12938597648977,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
+ "baseline_hash": null,
+ "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
+ },
+ "subtests.subtest.test_hmissing_imissing": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
+ "baseline_image": null,
+ "diff_image": null,
+ "rms": null,
+ "tolerance": null,
+ "result_image": "subtests.subtest.test_hmissing_imissing/result.png",
+ "baseline_hash": null,
+ "result_hash": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b"
+ },
+ "subtests.subtest.test_hdiff_imatch_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 200,
+ "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_idiff_tolerance": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3",
+ "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
+ "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
+ "rms": 29.260332173249314,
+ "tolerance": 3,
+ "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
+ "baseline_hash": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
+ "result_hash": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96"
+ },
+ "subtests.subtest.test_hdiff_imatch_savefig": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png",
+ "baseline_hash": "d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568",
+ "result_hash": "5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568"
+ },
+ "subtests.subtest.test_hdiff_imatch_style": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png",
+ "baseline_hash": "d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7",
+ "result_hash": "185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7"
+ },
+ "subtests.subtest.test_hdiff_imatch_removetext": {
+ "status": "failed",
+ "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.",
+ "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png",
+ "diff_image": null,
+ "rms": null,
+ "tolerance": 2,
+ "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png",
+ "baseline_hash": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e",
+ "result_hash": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
+ }
+}
diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py
new file mode 100644
index 00000000..48a2f461
--- /dev/null
+++ b/tests/subtests/test_subtest.py
@@ -0,0 +1,162 @@
+import sys
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+import matplotlib
+import matplotlib.ft2font
+import pytest
+from packaging.version import Version
+
+from .helpers import assert_existence, diff_summary, patch_summary
+
+# Handle Matplotlib and FreeType versions
+MPL_VERSION = Version(matplotlib.__version__)
+FTV = matplotlib.ft2font.__freetype_version__.replace('.', '')
+VERSION_ID = f"mpl{MPL_VERSION.major}{MPL_VERSION.minor}_ft{FTV}"
+HASH_LIBRARY = Path(__file__).parent / 'hashes' / (VERSION_ID + ".json")
+RESULT_LIBRARY = Path(__file__).parent / 'result_hashes' / (VERSION_ID + ".json")
+HASH_LIBRARY_FLAG = rf'--mpl-hash-library={HASH_LIBRARY}'
+FULL_BASELINE_PATH = Path(__file__).parent / 'baseline'
+
+BASELINE_IMAGES_FLAG_REL = ['--mpl-baseline-path=baseline', '--mpl-baseline-relative']
+BASELINE_IMAGES_FLAG_ABS = rf'--mpl-baseline-path={FULL_BASELINE_PATH}'
+
+TEST_FILE = Path(__file__).parent / 'subtest.py'
+
+# Global settings to update baselines when running pytest
+# Note: when updating baseline make sure you don't commit "fixes"
+# for tests that are expected to fail
+# (See also `run_subtest` argument `update_baseline` and `update_summary`.)
+UPDATE_BASELINE = False # baseline images and hashes
+UPDATE_SUMMARY = False # baseline summaries
+
+
+def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True,
+ update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY):
+ """ Run pytest (within pytest) and check JSON summary report.
+
+ Parameters
+ ----------
+ baseline_summary_name : str
+ String of the filename without extension for the baseline summary.
+ tmp_path : pathlib.Path
+ Path of a temporary directory to store results.
+ args : list
+ Extra arguments to pass to pytest.
+ summaries : tuple or list or set, optional, default=[]
+ Summaries to generate in addition to `json`.
+ xfail : bool, optional, default=True
+ Whether the overall pytest run should fail.
+ """
+ # Parse arguments
+ if summaries is None:
+ summaries = []
+ assert isinstance(summaries, (tuple, list, set))
+ summaries = ','.join({'json'} | set(summaries))
+
+ # Create the results path
+ results_path = tmp_path / 'results'
+ results_path.mkdir()
+
+ # Configure the arguments to run the test
+ pytest_args = [sys.executable, '-m', 'pytest', str(TEST_FILE)]
+ mpl_args = ['--mpl', rf'--mpl-results-path={results_path.as_posix()}',
+ f'--mpl-generate-summary={summaries}']
+ if update_baseline:
+ mpl_args += [rf'--mpl-generate-path={FULL_BASELINE_PATH}']
+ if HASH_LIBRARY.exists():
+ mpl_args += [rf'--mpl-generate-hash-library={HASH_LIBRARY}']
+
+ # Run the test and record exit status
+ status = subprocess.call(pytest_args + mpl_args + args)
+
+ # If updating baseline, don't check summaries
+ if update_baseline:
+ assert status == 0
+ if HASH_LIBRARY.exists():
+ # Keep in sync. Use `git add -p` to commit specific lines.
+ shutil.copy(HASH_LIBRARY, RESULT_LIBRARY)
+ return
+
+ # Ensure exit status is as expected
+ if xfail:
+ assert status != 0
+ else:
+ assert status == 0
+
+ # Load summaries
+ baseline_path = Path(__file__).parent / 'summaries'
+ baseline_file = baseline_path / (baseline_summary_name + '.json')
+ results_file = results_path / 'results.json'
+ if update_summary:
+ shutil.copy(results_file, baseline_file)
+ with open(baseline_file, 'r') as f:
+ baseline_summary = json.load(f)
+ with open(results_file, 'r') as f:
+ result_summary = json.load(f)
+
+ # Apply version specific patches
+ patch = baseline_path / (baseline_summary_name + f'_{VERSION_ID}.patch.json')
+ if patch.exists():
+ baseline_summary = patch_summary(baseline_summary, patch)
+ # Note: version specific hashes should be handled by diff_summary instead
+
+ # Compare summaries
+ diff_summary(baseline_summary, result_summary,
+ baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY)
+
+ # Ensure reported images exist
+ assert_existence(result_summary, path=results_path)
+
+
+def test_default(tmp_path):
+ run_subtest('test_default', tmp_path, [])
+
+
+@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
+def test_hash(tmp_path):
+ run_subtest('test_hash', tmp_path, [HASH_LIBRARY_FLAG])
+
+
+@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
+def test_hybrid(tmp_path):
+ run_subtest('test_hybrid', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS])
+
+
+@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
+def test_results_always(tmp_path):
+ run_subtest('test_results_always', tmp_path,
+ [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'])
+
+
+@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
+def test_html(tmp_path):
+ run_subtest('test_results_always', tmp_path,
+ [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'])
+ assert (tmp_path / 'results' / 'fig_comparison.html').exists()
+ assert (tmp_path / 'results' / 'extra.js').exists()
+ assert (tmp_path / 'results' / 'styles.css').exists()
+
+
+@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
+def test_html_hashes_only(tmp_path):
+ run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'])
+ assert (tmp_path / 'results' / 'fig_comparison.html').exists()
+ assert (tmp_path / 'results' / 'extra.js').exists()
+ assert (tmp_path / 'results' / 'styles.css').exists()
+
+
+def test_html_images_only(tmp_path):
+ run_subtest('test_html_images_only', tmp_path, [], summaries=['html'])
+ assert (tmp_path / 'results' / 'fig_comparison.html').exists()
+ assert (tmp_path / 'results' / 'extra.js').exists()
+ assert (tmp_path / 'results' / 'styles.css').exists()
+
+
+@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
+def test_basic_html(tmp_path):
+ run_subtest('test_results_always', tmp_path,
+ [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'])
+ assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists()
diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py
index 62fe0091..0843fe68 100644
--- a/tests/test_pytest_mpl.py
+++ b/tests/test_pytest_mpl.py
@@ -291,7 +291,7 @@ def test_hash_fails(tmpdir):
# If we use --mpl, it should detect that the figure is wrong
output = assert_pytest_fails_with(['--mpl', test_file], "doesn't match hash FAIL in library")
# We didn't specify a baseline dir so we shouldn't attempt to find one
- assert "Unable to find baseline image" not in output, output
+ assert "Image file not found for comparison test" not in output, output
# Check that the summary path is printed and that it exists.
output = assert_pytest_fails_with(['--mpl', test_file, '--mpl-generate-summary=html'],
@@ -337,13 +337,13 @@ def test_hash_fail_hybrid(tmpdir):
output = assert_pytest_fails_with(['--mpl', test_file,
'--mpl-baseline-path=/not/a/path'],
"doesn't match hash FAIL in library")
- assert "Unable to find baseline image" in output, output
+ assert "Image file not found for comparison test" in output, output
# Assert reports image comparison succeeds
output = assert_pytest_fails_with(['--mpl', test_file,
rf'--mpl-baseline-path={hash_baseline_dir_abs / "succeed"}'],
"doesn't match hash FAIL in library")
- assert "However, the comparison to the baseline image succeeded." in output, output
+ assert "The comparison to the baseline image succeeded." in output, output
# If we don't use --mpl option, the test should succeed
code = call_pytest([test_file])
diff --git a/tox.ini b/tox.ini
index 8c8e617c..774cc0f9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,13 +25,16 @@ deps =
mpl31: matplotlib==3.1.*
mpl32: matplotlib==3.2.*
mpl33: matplotlib==3.3.*
+ mpl34: matplotlib==3.4.*
+ mpl35: matplotlib==3.5.*
mpldev: git+https://github.com/matplotlib/matplotlib.git#egg=matplotlib
extras =
test
commands =
pip freeze
# Make sure the tests pass with and without --mpl
- pytest '{toxinidir}' {posargs}
+ # Use -m so pytest skips "subtests" which always apply --mpl
+ pytest '{toxinidir}' -m "mpl_image_compare" {posargs}
pytest '{toxinidir}' --mpl --cov pytest_mpl {posargs}
[testenv:codestyle]