From 4a2fa22d705ee247c86cd4fa5aca8935c8fba353 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 14 Nov 2016 14:59:21 +0000 Subject: [PATCH] Generalize plugin so that it is not FITS-specific --- README.md | 133 +++++----- {pytest_fits => pytest_arraydiff}/__init__.py | 0 pytest_arraydiff/plugin.py | 240 ++++++++++++++++++ pytest_fits/plugin.py | 208 --------------- setup.py | 10 +- tests/test_pytest_arraydiff.py | 100 ++++++++ tests/test_pytest_fits.py | 115 --------- 7 files changed, 417 insertions(+), 389 deletions(-) rename {pytest_fits => pytest_arraydiff}/__init__.py (100%) create mode 100755 pytest_arraydiff/plugin.py delete mode 100755 pytest_fits/plugin.py create mode 100644 tests/test_pytest_arraydiff.py delete mode 100644 tests/test_pytest_fits.py diff --git a/README.md b/README.md index 6457984..7647d74 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,22 @@ -[![Travis Build Status](https://travis-ci.org/astrofrog/pytest-fits.svg?branch=master)](https://travis-ci.org/astrofrog/pytest-fits) -[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/kwbvm9u79mrq6i0w?svg=true)](https://ci.appveyor.com/project/astrofrog/pytest-fits) +[![Travis Build Status](https://travis-ci.org/astrofrog/pytest-arraydiff.svg?branch=master)](https://travis-ci.org/astrofrog/pytest-arraydiff) +[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/kwbvm9u79mrq6i0w?svg=true)](https://ci.appveyor.com/project/astrofrog/pytest-arraydiff) About ----- -This is a [py.test](http://pytest.org) plugin to faciliate the generation and -comparison of FITS files produced during tests (this is a spin-off from -[pytest-fits](https://github.com/astrofrog/pytest-fits)). +This is a [py.test](http://pytest.org) plugin to facilitate the generation and +comparison of arrays produced during tests (this is a spin-off from +[pytest-arraydiff](https://github.com/astrofrog/pytest-arraydiff)). -The basic idea is that you can write a test that generates an Astropy HDU or -HDUList object. You can then either run the tests in a mode to **generate** -the reference FITS files from those HDUs or HDULists, or you can run the tests -in **comparison** mode, which will compare the results of the tests to the -reference ones within some tolerance. +The basic idea is that you can write a test that generates a Numpy array. You +can then either run the tests in a mode to **generate** reference files from the +arrays, or you can run the tests in **comparison** mode, which will compare the +results of the tests to the reference ones within some tolerance. + +At the moment, the supported file formats for the reference files are: + +* The FITS format (requires [astropy](http://www.astropy.org)) +* A plain text-based format (baed on Numpy ``loadtxt`` output) For more information on how to write tests to do this, see the **Using** section below. @@ -21,12 +25,11 @@ Installing ---------- This plugin is compatible with Python 2.7, and 3.3 and later, and requires -[pytest](http://pytest.org), [numpy](http://www.numpy.org), and -[astropy](http://www.astropy.org) to be installed. +[pytest](http://pytest.org) and [numpy](http://www.numpy.org) to be installed. To install, you can do: - pip install https://github.com/astrofrog/pytest-fits/archive/master.zip + pip install https://github.com/astrofrog/pytest-arraydiff/archive/master.zip You can check that the plugin is registered with pytest by doing: @@ -36,95 +39,103 @@ which will show a list of plugins: This is pytest version 2.7.1, imported from ... setuptools registered plugins: - pytest-fits-0.1 at ... + pytest-arraydiff-0.1 at ... Using ----- To use, you simply need to mark the function where you want to compare images -using ``@pytest.mark.fits_compare``, and make sure that the function -returns an Astropy HDU or HDUList object, or a plain Numpy array:: +using ``@pytest.mark.array_compare``, and make sure that the function +returns a plain Numpy array:: python import pytest import numpy as np - from astropy.io import fits - @pytest.mark.fits_compare + @pytest.mark.array_compare def test_succeeds(): - data = np.arange(3 * 5 * 4).reshape((3, 5, 4)) - header = fits.Header() - header['TEST'] = 'example' - header['VALUE'] = 1.344 - return fits.PrimaryHDU(data, header) + return np.arange(3 * 5 * 4).reshape((3, 5, 4)) To generate the reference FITS files, run the tests with the -``--fits-generate-path`` option with the name of the directory where the -generated images should be placed: +``--arraydiff-generate-path`` option with the name of the directory where the +generated files should be placed: - py.test --fits-generate-path=baseline + py.test --arraydiff-generate-path=reference If the directory does not exist, it will be created. The directory will be interpreted as being relative to where you are running ``py.test``. Make sure you manually check the reference images to ensure they are correct. Once you are happy with the generated FITS files, you should move them to a -sub-directory called ``baseline`` relative to the test files (this name is +sub-directory called ``reference`` relative to the test files (this name is configurable, see below). You can also generate the baseline images directly in the right directory. You can then run the tests simply with: - py.test --fits + py.test --arraydiff -and the tests will pass if the images are the same. If you omit the ``--fits`` -option, the tests will run but will only check that the code runs without -checking the output images. +and the tests will pass if the images are the same. If you omit the +``--arraydiff`` option, the tests will run but will only check that the code +runs without checking the output images. Options ------- -The ``@pytest.mark.fits_compare`` marker can take an argument which is the -relative tolerance for floating point values (which defaults to 1e-7): +The ``@pytest.mark.array_compare`` marker take an argument to specify the format +to use for the reference files: + +```python +@pytest.mark.array_compare(file_format='text') +def test_image(): + ... +``` + +The supported formats at this time are ``text`` and ``fits``, and contributions +for other formats are welcome. The default format is ``text``. + +Another argument is the relative tolerance for floating point values (which +defaults to 1e-7): ```python -@pytest.mark.fits_compare(rtol=20) +@pytest.mark.array_compare(rtol=20) def test_image(): ... ``` -You can also pass keyword arguments to the Astropy FITS ``writeto`` methods -by using ``writeto_kwargs``: +You can also pass keyword arguments to the writers using the ``write_kwargs``. +For the ``text`` format, these arguments are passed to ``savetxt`` while for +the ``fits`` format they are passed to Astropy's ``fits.writeto`` function. ```python -@pytest.mark.fits_compare(writeto_kwargs={'output_verify': 'silentfix'}) +@pytest.mark.array_compare(file_format='fits', write_kwargs={'output_verify': 'silentfix'}) def test_image(): ... ``` -Other options include the name of the baseline directory (which defaults to -``baseline`` ) and the filename of the plot (which defaults to the name of the -test with a ``.fits`` suffix): +Other options include the name of the reference directory (which defaults to +``reference`` ) and the filename for the reference file (which defaults to the +name of the test with a format-dependent extension). ```python -@pytest.mark.fits_compare(baseline_dir='baseline_images', +@pytest.mark.array_compare(reference_dir='baseline_images', filename='other_name.fits') def test_image(): ... ``` -The baseline directory in the decorator above will be interpreted as being +The reference directory in the decorator above will be interpreted as being relative to the test file. Note that the baseline directory can also be a URL (which should start with ``http://`` or ``https://`` and end in a slash). Finally, you can also set a custom baseline directory globally when running tests by running ``py.test`` with: - py.test --fits --fits-baseline-path=baseline_images + py.test --arraydiff --arraydiff-reference-path=baseline_images This directory will be interpreted as being relative to where the tests are -run. In addition, if both this option and the ``baseline_dir`` option in the -``fits_compare`` decorator are used, the one in the decorator takes +run. In addition, if both this option and the ``reference_dir`` option in the +``array_compare`` decorator are used, the one in the decorator takes precedence. Test failure example @@ -133,32 +144,32 @@ Test failure example If the images produced by the tests are correct, then the test will pass, but if they are not, the test will fail with a message similar to the following: ``` -E Exception: -E fitsdiff: 1.2.1 -E a: /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp7067vdm6/baseline-test_succeeds_func.fits -E b: /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp7067vdm6/test_succeeds_func.fits -E Maximum number of different data values to be reported: 10 -E Data comparison level: 1e-07 +E AssertionError: +E +E a: /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmpbvjkzt_q/test_to_mask_rect-mode_subpixels-subpixels_18.txt +E b: /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmpbvjkzt_q/reference-test_to_mask_rect-mode_subpixels-subpixels_18.txt E -E Primary HDU: +E Not equal to tolerance rtol=1e-07, atol=0 E -E Data contains differences: -E Data differs at [3, 3, 3]: -E a> 50 -E b> 100 -E 1 different pixels found (1.67% different). +E (mismatch 47.22222222222222%) +E x: array([[ 0. , 0. , 0. , 0. , 0.404012, 0.55 , +E 0.023765, 0. , 0. ], +E [ 0. , 0. , 0. , 0.112037, 1.028704, 1.1 ,... +E y: array([[ 0. , 0. , 0. , 0. , 0.367284, 0.5 , +E 0.021605, 0. , 0. ], +E [ 0. , 0. , 0. , 0.101852, 0.935185, 1. ,... ``` -The file paths included in the exception are then available for inspection: +The file paths included in the exception are then available for inspection. -Running the tests for pytest-fits --------------------------------- +Running the tests for pytest-arraydiff +-------------------------------------- If you are contributing some changes and want to run the tests, first install the latest version of the plugin then do: cd tests - py.test --fits + py.test --arraydiff The reason for having to install the plugin first is to ensure that the plugin is correctly loaded as part of the test suite. diff --git a/pytest_fits/__init__.py b/pytest_arraydiff/__init__.py similarity index 100% rename from pytest_fits/__init__.py rename to pytest_arraydiff/__init__.py diff --git a/pytest_arraydiff/plugin.py b/pytest_arraydiff/plugin.py new file mode 100755 index 0000000..300964a --- /dev/null +++ b/pytest_arraydiff/plugin.py @@ -0,0 +1,240 @@ +# Copyright (c) 2016, Thomas P. Robitaille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# This package was derived from pytest-mpl, which is released under a BSD +# license and can be found here: +# +# https://github.com/astrofrog/pytest-mpl + + +from functools import wraps + +import os +import sys +import shutil +import tempfile +import warnings + +import pytest +import numpy as np + +if sys.version_info[0] == 2: + from urllib import urlopen +else: + from urllib.request import urlopen + + +class FITSDiff(object): + + extension = 'fits' + + @staticmethod + def read(filename): + from astropy.io import fits + return fits.getdata(filename) + + @staticmethod + def write(filename, array, **kwargs): + from astropy.io import fits + return fits.writeto(filename, array, **kwargs) + + +class TextDiff(object): + + extension = 'txt' + + @staticmethod + def read(filename): + return np.loadtxt(filename) + + @staticmethod + def write(filename, array, **kwargs): + if 'fmt' not in kwargs: + kwargs['fmt'] = '%g' + return np.savetxt(filename, array, **kwargs) + + +FORMATS = {} +FORMATS['fits'] = FITSDiff +FORMATS['text'] = TextDiff + + +def _download_file(url): + u = urlopen(url) + result_dir = tempfile.mkdtemp() + filename = os.path.join(result_dir, 'downloaded') + with open(filename, 'wb') as tmpfile: + tmpfile.write(u.read()) + return filename + + +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--arraydiff', action='store_true', + help="Enable comparison of arrays to reference arrays stored in files") + group.addoption('--arraydiff-generate-path', + help="directory to generate reference files in, relative to location where py.test is run", action='store') + group.addoption('--arraydiff-reference-path', + help="directory containing reference files, relative to location where py.test is run", action='store') + + +def pytest_configure(config): + + if config.getoption("--arraydiff") or config.getoption("--arraydiff-generate-path") is not None: + + reference_dir = config.getoption("--arraydiff-reference-path") + generate_dir = config.getoption("--arraydiff-generate-path") + + if reference_dir is not None and generate_dir is not None: + warnings.warn("Ignoring --arraydiff-reference-path since --arraydiff-generate-path is set") + + if reference_dir is not None: + reference_dir = os.path.abspath(reference_dir) + if generate_dir is not None: + reference_dir = os.path.abspath(generate_dir) + + config.pluginmanager.register(ArrayComparison(config, + reference_dir=reference_dir, + generate_dir=generate_dir)) + + +class ArrayComparison(object): + + def __init__(self, config, reference_dir=None, generate_dir=None): + self.config = config + self.reference_dir = reference_dir + self.generate_dir = generate_dir + + def pytest_runtest_setup(self, item): + + compare = item.keywords.get('array_compare') + + if compare is None: + return + + file_format = compare.kwargs.get('file_format', 'text') + + if file_format not in FORMATS: + raise ValueError("Unknown format: {0}".format(file_format)) + + if 'extension' in compare.kwargs: + extension = compare.kwargs['extension'] + else: + extension = FORMATS[file_format].extension + + atol = compare.kwargs.get('atol', 0.) + rtol = compare.kwargs.get('rtol', 1e-7) + + single_reference = compare.kwargs.get('single_reference', False) + + write_kwargs = compare.kwargs.get('write_kwargs', {}) + + original = item.function + + @wraps(item.function) + def item_function_wrapper(*args, **kwargs): + + reference_dir = compare.kwargs.get('reference_dir', None) + if reference_dir is None: + if self.reference_dir is None: + reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), 'reference') + else: + reference_dir = self.reference_dir + else: + if not reference_dir.startswith(('http://', 'https://')): + reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), reference_dir) + + baseline_remote = reference_dir.startswith('http') + + # Run test and get figure object + import inspect + if inspect.ismethod(original): # method + array = original(*args[1:], **kwargs) + else: # function + array = original(*args, **kwargs) + + # Find test name to use as plot name + filename = compare.kwargs.get('filename', None) + if filename is None: + if single_reference: + filename = original.__name__ + '.' + extension + else: + filename = item.name + '.' + extension + filename = filename.replace('[', '_').replace(']', '_') + filename = filename.replace('_.' + extension, '.' + extension) + + # What we do now depends on whether we are generating the reference + # files or simply running the test. + if self.generate_dir is None: + + # Save the figure + result_dir = tempfile.mkdtemp() + test_image = os.path.abspath(os.path.join(result_dir, filename)) + + FORMATS[file_format].write(test_image, array, **write_kwargs) + + # Find path to baseline image + if baseline_remote: + baseline_file_ref = _download_file(reference_dir + filename) + else: + baseline_file_ref = os.path.abspath(os.path.join(os.path.dirname(item.fspath.strpath), reference_dir, filename)) + + if not os.path.exists(baseline_file_ref): + raise Exception("""File not found for comparison test + Generated file: + \t{test} + This is expected for new tests.""".format( + test=test_image)) + + # distutils may put the baseline images in non-accessible places, + # copy to our tmpdir to be sure to keep them in case of failure + baseline_file = os.path.abspath(os.path.join(result_dir, 'reference-' + filename)) + shutil.copyfile(baseline_file_ref, baseline_file) + + array_ref = FORMATS[file_format].read(baseline_file) + + try: + np.testing.assert_allclose(array_ref, array, atol=atol, rtol=rtol) + except AssertionError as exc: + message = "\n\na: {0}".format(test_image) + '\n' + message += "b: {0}".format(baseline_file) + '\n' + message += exc.args[0] + raise AssertionError(message) + + shutil.rmtree(result_dir) + + else: + + if not os.path.exists(self.generate_dir): + os.makedirs(self.generate_dir) + + FORMATS[file_format].write(os.path.abspath(os.path.join(self.generate_dir, filename)), array, **write_kwargs) + + pytest.skip("Skipping test, since generating data") + + if item.cls is not None: + setattr(item.cls, item.function.__name__, item_function_wrapper) + else: + item.obj = item_function_wrapper diff --git a/pytest_fits/plugin.py b/pytest_fits/plugin.py deleted file mode 100755 index 2e253d0..0000000 --- a/pytest_fits/plugin.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) 2016, Thomas P. Robitaille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# This package was derived from pytest-mpl, which is released under a BSD -# license and can be found here: -# -# https://github.com/astrofrog/pytest-mpl - - -from functools import wraps - -import os -import sys -import shutil -import tempfile -import warnings - -import pytest -import numpy as np -from astropy.io.fits import PrimaryHDU -from astropy.io.fits.diff import FITSDiff - -if sys.version_info[0] == 2: - from urllib import urlopen -else: - from urllib.request import urlopen - - -def compare_fits_files(filename1, filename2, atol=None, rtol=1e-7): - - if atol is not None: - raise NotImplementedError("atol argument not yet supported") - - diff = FITSDiff(filename1, filename2, tolerance=rtol) - - return diff.identical, diff.report() - - -def _download_file(url): - u = urlopen(url) - result_dir = tempfile.mkdtemp() - filename = os.path.join(result_dir, 'downloaded') - with open(filename, 'wb') as tmpfile: - tmpfile.write(u.read()) - return filename - - -def pytest_addoption(parser): - group = parser.getgroup("general") - group.addoption('--fits', action='store_true', - help="Enable comparison of FITS files to reference files") - group.addoption('--fits-generate-path', - help="directory to generate reference FITS files in, relative to location where py.test is run", action='store') - group.addoption('--fits-baseline-path', - help="directory containing baseline FITS files, relative to location where py.test is run", action='store') - - -def pytest_configure(config): - - if config.getoption("--fits") or config.getoption("--fits-generate-path") is not None: - - baseline_dir = config.getoption("--fits-baseline-path") - generate_dir = config.getoption("--fits-generate-path") - - if baseline_dir is not None and generate_dir is not None: - warnings.warn("Ignoring --fits-baseline-path since --fits-generate-path is set") - - if baseline_dir is not None: - baseline_dir = os.path.abspath(baseline_dir) - if generate_dir is not None: - baseline_dir = os.path.abspath(generate_dir) - - config.pluginmanager.register(FITSComparison(config, - baseline_dir=baseline_dir, - generate_dir=generate_dir)) - - -class FITSComparison(object): - - def __init__(self, config, baseline_dir=None, generate_dir=None): - self.config = config - self.baseline_dir = baseline_dir - self.generate_dir = generate_dir - - def pytest_runtest_setup(self, item): - - compare = item.keywords.get('fits_compare') - - if compare is None: - return - - atol = compare.kwargs.get('atol', None) - rtol = compare.kwargs.get('rtol', 1e-7) - - single_reference = compare.kwargs.get('single_reference', False) - - writeto_kwargs = compare.kwargs.get('writeto_kwargs', {}) - if not 'clobber' in writeto_kwargs: - writeto_kwargs['clobber'] = True - - original = item.function - - @wraps(item.function) - def item_function_wrapper(*args, **kwargs): - - baseline_dir = compare.kwargs.get('baseline_dir', None) - if baseline_dir is None: - if self.baseline_dir is None: - baseline_dir = os.path.join(os.path.dirname(item.fspath.strpath), 'baseline') - else: - baseline_dir = self.baseline_dir - else: - if not baseline_dir.startswith(('http://', 'https://')): - baseline_dir = os.path.join(os.path.dirname(item.fspath.strpath), baseline_dir) - - baseline_remote = baseline_dir.startswith('http') - - # Run test and get figure object - import inspect - if inspect.ismethod(original): # method - hdu = original(*args[1:], **kwargs) - else: # function - hdu = original(*args, **kwargs) - - # If a Numpy array was returned, wrap in FITS - if isinstance(hdu, np.ndarray): - hdu = PrimaryHDU(hdu) - - # Find test name to use as plot name - filename = compare.kwargs.get('filename', None) - if filename is None: - if single_reference: - filename = original.__name__ + '.fits' - else: - filename = item.name + '.fits' - filename = filename.replace('[', '_').replace(']', '_') - filename = filename.replace('_.fits', '.fits') - - # What we do now depends on whether we are generating the reference - # files or simply running the test. - if self.generate_dir is None: - - # Save the figure - result_dir = tempfile.mkdtemp() - test_image = os.path.abspath(os.path.join(result_dir, filename)) - - hdu.writeto(test_image, **writeto_kwargs) - - # Find path to baseline image - if baseline_remote: - baseline_file_ref = _download_file(baseline_dir + filename) - else: - baseline_file_ref = os.path.abspath(os.path.join(os.path.dirname(item.fspath.strpath), baseline_dir, filename)) - - if not os.path.exists(baseline_file_ref): - print("ICI") - raise Exception("""FITS file not found for comparison test - Generated FITS file: - \t{test} - This is expected for new tests.""".format( - test=test_image)) - - # distutils may put the baseline images in non-accessible places, - # copy to our tmpdir to be sure to keep them in case of failure - baseline_file = os.path.abspath(os.path.join(result_dir, 'baseline-' + filename)) - shutil.copyfile(baseline_file_ref, baseline_file) - - identical, msg = compare_fits_files(baseline_file, test_image, atol=atol, rtol=rtol) - - if identical: - shutil.rmtree(result_dir) - else: - raise Exception(msg) - - else: - - if not os.path.exists(self.generate_dir): - os.makedirs(self.generate_dir) - - hdu.writeto(os.path.abspath(os.path.join(self.generate_dir, filename)), **writeto_kwargs) - pytest.skip("Skipping test, since generating data") - - if item.cls is not None: - setattr(item.cls, item.function.__name__, item_function_wrapper) - else: - item.obj = item_function_wrapper diff --git a/setup.py b/setup.py index 78c2e61..c0c8aa2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -from pytest_fits import __version__ +from pytest_arraydiff import __version__ try: import pypandoc @@ -12,12 +12,12 @@ setup( version=__version__, url="https://github.com/astrofrog/pytest-fits", - name="pytest-fits", - description='pytest plugin to help with comparing FITS output from tests', + name="pytest-arraydiff", + description='pytest plugin to help with comparing array output from tests', long_description=long_description, - packages = ['pytest_fits'], + packages = ['pytest_arraydiff'], license='BSD', author='Thomas Robitaille', author_email='thomas.robitaille@gmail.com', - entry_points = {'pytest11': ['pytest_fits = pytest_fits.plugin',]}, + entry_points = {'pytest11': ['pytest_arraydiff = pytest_arraydiff.plugin',]}, ) diff --git a/tests/test_pytest_arraydiff.py b/tests/test_pytest_arraydiff.py new file mode 100644 index 0000000..692ba33 --- /dev/null +++ b/tests/test_pytest_arraydiff.py @@ -0,0 +1,100 @@ +import os +import subprocess +import tempfile + +import pytest +import numpy as np + +reference_dir = 'baseline' + + +@pytest.mark.array_compare(reference_dir=reference_dir) +def test_succeeds_func_default(): + return np.arange(3 * 5 * 4).reshape((3, 5, 4)) + + +@pytest.mark.array_compare(file_format='text', reference_dir=reference_dir) +def test_succeeds_func_text(): + return np.arange(3 * 5 * 4).reshape((3, 5, 4)) + + +@pytest.mark.array_compare(file_format='fits', reference_dir=reference_dir) +def test_succeeds_func_fits(): + return np.arange(3 * 5 * 4).reshape((3, 5, 4)) + + +class TestClass(object): + + @pytest.mark.array_compare(reference_dir=reference_dir) + def test_succeeds_class(self): + return np.arange(2 * 4 * 3).reshape((2, 4, 3)) + + +TEST_FAILING = """ +import pytest +import numpy as np +from astropy.io import fits +@pytest.mark.array_compare +def test_fail(): + return np.ones((3, 4)) +""" + + +def test_fails(): + + tmpdir = tempfile.mkdtemp() + + test_file = os.path.join(tmpdir, 'test.py') + with open(test_file, 'w') as f: + f.write(TEST_FAILING) + + # If we use --arraydiff, it should detect that the file is missing + code = subprocess.call('py.test --arraydiff {0}'.format(test_file), shell=True) + assert code != 0 + + # If we don't use --arraydiff option, the test should succeed + code = subprocess.call('py.test {0}'.format(test_file), shell=True) + assert code == 0 + + +TEST_GENERATE = """ +import pytest +import numpy as np +from astropy.io import fits +@pytest.mark.array_compare(file_format='{file_format}') +def test_gen(): + return np.arange(6 * 5).reshape((6, 5)) +""" + + +@pytest.mark.parametrize('file_format', ('fits', 'text')) +def test_generate(file_format): + + tmpdir = tempfile.mkdtemp() + + test_file = os.path.join(tmpdir, 'test.py') + with open(test_file, 'w') as f: + f.write(TEST_GENERATE.format(file_format=file_format)) + + gen_dir = os.path.join(tmpdir, 'spam', 'egg') + + # If we don't generate, the test will fail + p = subprocess.Popen('py.test --arraydiff {0}'.format(test_file), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.wait() + output = p.stdout.read() + assert b'File not found for comparison test' in output + + # If we do generate, the test should succeed and a new file will appear + code = subprocess.call('py.test --arraydiff-generate-path={0} {1}'.format(gen_dir, test_file), shell=True) + assert code == 0 + assert os.path.exists(os.path.join(gen_dir, 'test_gen.' + ('fits' if file_format == 'fits' else 'txt'))) + + +@pytest.mark.array_compare(reference_dir=reference_dir, rtol=0.5) +def test_tolerance(): + return np.ones((3,4)) * 1.6 + + +def test_nofile(): + pass diff --git a/tests/test_pytest_fits.py b/tests/test_pytest_fits.py deleted file mode 100644 index e425f6f..0000000 --- a/tests/test_pytest_fits.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import subprocess -import tempfile - -import pytest -import numpy as np -from astropy.io import fits - -baseline_dir = 'baseline' - - -@pytest.mark.fits_compare(baseline_dir=baseline_dir) -def test_succeeds_func(): - data = np.arange(3 * 5 * 4).reshape((3, 5, 4)) - header = fits.Header() - header['TEST'] = 'function' - header['VALUE'] = 1.344 - return fits.PrimaryHDU(data, header) - - -@pytest.mark.fits_compare(baseline_dir=baseline_dir) -def test_succeeds_func_array(): - data = np.arange(3 * 5 * 4).reshape((3, 5, 4)) - return data - - -class TestClass(object): - - @pytest.mark.fits_compare(baseline_dir=baseline_dir) - def test_succeeds_class(self): - data = np.arange(2 * 4 * 3).reshape((2, 4, 3)) - header = fits.Header() - header['TEST'] = 'class' - header['VALUE'] = 1.344 - return fits.PrimaryHDU(data, header) - - -TEST_FAILING = """ -import pytest -import numpy as np -from astropy.io import fits -@pytest.mark.fits_compare -def test_fail(): - data = np.ones((3,4)) - header = fits.Header() - header['TEST'] = 'tolerance' - header['VALUE'] = 2 - return fits.PrimaryHDU(data, header) -""" - - -def test_fails(): - - tmpdir = tempfile.mkdtemp() - - test_file = os.path.join(tmpdir, 'test.py') - with open(test_file, 'w') as f: - f.write(TEST_FAILING) - - # If we use --fits, it should detect that the file is missing - code = subprocess.call('py.test --fits {0}'.format(test_file), shell=True) - assert code != 0 - - # If we don't use --fits option, the test should succeed - code = subprocess.call('py.test {0}'.format(test_file), shell=True) - assert code == 0 - - -TEST_GENERATE = """ -import pytest -import numpy as np -from astropy.io import fits -@pytest.mark.fits_compare -def test_gen(): - data = np.arange(6 * 5 * 3).reshape((6, 5, 3)) - header = fits.Header() - header['TEST'] = 'generate' - header['VALUE'] = 3.14 - return fits.PrimaryHDU(data, header) -""" - - -def test_generate(): - - tmpdir = tempfile.mkdtemp() - - test_file = os.path.join(tmpdir, 'test.py') - with open(test_file, 'w') as f: - f.write(TEST_GENERATE) - - gen_dir = os.path.join(tmpdir, 'spam', 'egg') - - # If we don't generate, the test will fail - p = subprocess.Popen('py.test --fits {0}'.format(test_file), shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.wait() - assert b'FITS file not found for comparison test' in p.stdout.read() - - # If we do generate, the test should succeed and a new file will appear - code = subprocess.call('py.test --fits-generate-path={0} {1}'.format(gen_dir, test_file), shell=True) - assert code == 0 - assert os.path.exists(os.path.join(gen_dir, 'test_gen.fits')) - - -@pytest.mark.fits_compare(baseline_dir=baseline_dir, rtol=0.5) -def test_tolerance(): - data = np.ones((3,4)) * 1.6 - header = fits.Header() - header['TEST'] = 'tolerance' - header['VALUE'] = 2 - return fits.PrimaryHDU(data, header) - - -def test_nofile(): - pass