Merge pull request #328 from bcipolli/deprecate_binopener

MRG: More graceful handling of compressed file Opening

This PR deprecates BinOpener, instead exposing a decorator for creating arbitrary file extension / opener methods (similar to that proposed in #317 and created in #319).

Author: matthew-brett

bin/nib-nifti-dx

@@ -25,8 +25,8 @@ def main():
     (opts, files) = parser.parse_args()
 
     for fname in files:
-        with nib.volumeutils.BinOpener(fname) as fobj:
-            hdr =  fobj.read(nib.nifti1.header_dtype.itemsize)
+        with nib.openers.ImageOpener(fname) as fobj:
+            hdr = fobj.read(nib.nifti1.header_dtype.itemsize)
         result = nib.Nifti1Header.diagnose_binaryblock(hdr)
         if len(result):
             print('Picky header check output for "%s"\n' % fname)

nibabel/arrayproxy.py

@@ -27,9 +27,10 @@
 """
 import warnings
 
-from .volumeutils import BinOpener, array_from_file, apply_read_scaling
+from .volumeutils import array_from_file, apply_read_scaling
 from .fileslice import fileslice
 from .keywordonly import kw_only_meth
+from .openers import ImageOpener
 
 
 class ArrayProxy(object):
@@ -130,7 +131,7 @@ def get_unscaled(self):
 
         This is an optional part of the proxy API
         '''
-        with BinOpener(self.file_like) as fileobj:
+        with ImageOpener(self.file_like) as fileobj:
             raw_data = array_from_file(self._shape,
                                        self._dtype,
                                        fileobj,
@@ -145,7 +146,7 @@ def __array__(self):
         return apply_read_scaling(raw_data, self._slope, self._inter)
 
     def __getitem__(self, slicer):
-        with BinOpener(self.file_like) as fileobj:
+        with ImageOpener(self.file_like) as fileobj:
             raw_data = fileslice(fileobj,
                                  slicer,
                                  self._shape,

nibabel/benchmarks/bench_fileslice.py

@@ -19,7 +19,7 @@
 import numpy as np
 
 from io import BytesIO
-from ..openers import Opener
+from ..openers import ImageOpener
 from ..fileslice import fileslice
 from ..rstutils import rst_table
 from ..tmpdirs import InTemporaryDirectory
@@ -47,10 +47,10 @@ def run_slices(file_like, repeat=3, offset=0, order='F'):
     n_dim = len(SHAPE)
     n_slicers = len(_slices_for_len(1))
     times_arr = np.zeros((n_dim, n_slicers))
-    with Opener(file_like, 'wb') as fobj:
+    with ImageOpener(file_like, 'wb') as fobj:
         fobj.write(b'\0' * offset)
         fobj.write(arr.tostring(order=order))
-    with Opener(file_like, 'rb') as fobj:
+    with ImageOpener(file_like, 'rb') as fobj:
         for i, L in enumerate(SHAPE):
             for j, slicer in enumerate(_slices_for_len(L)):
                 sliceobj = [slice(None)] * n_dim

nibabel/fileholders.py

@@ -10,7 +10,7 @@
 
 from copy import copy
 
-from .volumeutils import BinOpener
+from .openers import ImageOpener
 
 
 class FileHolderError(Exception):
@@ -63,10 +63,10 @@ def get_prepare_fileobj(self, *args, **kwargs):
            ``self.pos``
         '''
         if self.fileobj is not None:
-            obj = BinOpener(self.fileobj)  # for context manager
+            obj = ImageOpener(self.fileobj)  # for context manager
             obj.seek(self.pos)
         elif self.filename is not None:
-            obj = BinOpener(self.filename, *args, **kwargs)
+            obj = ImageOpener(self.filename, *args, **kwargs)
             if self.pos != 0:
                 obj.seek(self.pos)
         else:

nibabel/freesurfer/mghformat.py

@@ -18,6 +18,7 @@
 from ..fileholders import FileHolder,  copy_file_map
 from ..arrayproxy import ArrayProxy
 from ..keywordonly import kw_only_meth
+from ..openers import ImageOpener
 
 # mgh header
 # See https://surfer.nmr.mgh.harvard.edu/fswiki/FsTutorial/MghFormat
@@ -453,6 +454,7 @@ def writeftr_to(self, fileobj):
         fileobj.write(ftr_nd.tostring())
 
 
+@ImageOpener.register_ext_from_image('.mgz', ImageOpener.gz_def)
 class MGHImage(SpatialImage):
     """ Class for MGH format image
     """

nibabel/loadsave.py

@@ -12,7 +12,7 @@
 import numpy as np
 
 from .filename_parser import types_filenames, splitext_addext
-from .volumeutils import BinOpener, Opener
+from .openers import ImageOpener
 from .analyze import AnalyzeImage
 from .spm2analyze import Spm2AnalyzeImage
 from .nifti1 import Nifti1Image, Nifti1Pair, header_dtype as ni1_hdr_dtype
@@ -68,18 +68,18 @@ def guessed_image_type(filename):
     elif lext == '.mnc':
         # Look for HDF5 signature for MINC2
         # https://www.hdfgroup.org/HDF5/doc/H5.format.html
-        with Opener(filename) as fobj:
+        with ImageOpener(filename) as fobj:
             signature = fobj.read(4)
             klass = Minc2Image if signature == b'\211HDF' else Minc1Image
     elif lext == '.nii':
-        with BinOpener(filename) as fobj:
+        with ImageOpener(filename) as fobj:
             binaryblock = fobj.read(348)
         ft = which_analyze_type(binaryblock)
         klass = Nifti2Image if ft == 'nifti2' else Nifti1Image
     else:  # might be nifti 1 or 2 pair or analyze of some sort
         files_types = (('image', '.img'), ('header', '.hdr'))
         filenames = types_filenames(filename, files_types)
-        with BinOpener(filenames['header']) as fobj:
+        with ImageOpener(filenames['header']) as fobj:
             binaryblock = fobj.read(348)
         ft = which_analyze_type(binaryblock)
         if ft == 'nifti2':
@@ -208,7 +208,7 @@ def read_img_data(img, prefer='scaled'):
             hdr.set_data_offset(dao.offset)
         if default_scaling and (dao.slope, dao.inter) != (1, 0):
             hdr.set_slope_inter(dao.slope, dao.inter)
-    with BinOpener(img_file_like) as fileobj:
+    with ImageOpener(img_file_like) as fileobj:
         if prefer == 'scaled':
             return hdr.data_from_fileobj(fileobj)
         return hdr.raw_data_from_fileobj(fileobj)

nibabel/nicom/dicomwrappers.py

@@ -18,7 +18,7 @@
 
 from . import csareader as csar
 from .dwiparams import B2q, nearest_pos_semi_def, q2bg
-from ..volumeutils import BinOpener
+from ..openers import ImageOpener
 from ..onetime import setattr_on_read as one_time
 
 
@@ -51,7 +51,7 @@ def wrapper_from_file(file_like, *args, **kwargs):
     """
     import dicom
 
-    with BinOpener(file_like) as fobj:
+    with ImageOpener(file_like) as fobj:
         dcm_data = dicom.read_file(fobj, *args, **kwargs)
     return wrapper_from_data(dcm_data)
 

nibabel/openers.py

@@ -146,3 +146,42 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close_if_mine()
+
+
+class ImageOpener(Opener):
+    """ Opener-type class passed to image classes to collect compressed extensions
+
+    This class allows itself to have image extensions added to its class
+    attributes, via the `register_ex_from_images`.  The class can therefore
+    change state when image classes are defined.
+    """
+
+    @classmethod
+    def register_ext_from_image(opener_klass, ext, func_def):
+        """Decorator for adding extension / opener_function associations.
+
+        Should be used to decorate classes.
+
+        Parameters
+        ----------
+        opener_klass : decorated class
+        ext : file extension to associate `func_def` with.
+          should start with '.'
+        func_def : opener function/parameter tuple
+          Should be a `(function, (args,))` tuple, where `function` accepts
+          a filename as the first parameter, and `args` defines the
+          other arguments that `function` accepts. These arguments must
+          be any (unordered) subset of `mode`, `compresslevel`,
+          and `buffering`.
+
+        Returns
+        -------
+        opener_klass, with a side-effect of updating the ImageOpener class
+        with the desired extension / opener association.
+        """
+        def decorate(klass):
+            assert ext not in opener_klass.compress_ext_map, \
+                "Cannot redefine extension-function mappings."
+            opener_klass.compress_ext_map[ext] = func_def
+            return klass
+        return decorate

nibabel/parrec.py

@@ -99,10 +99,11 @@
 from .keywordonly import kw_only_meth
 from .spatialimages import SpatialImage, Header
 from .eulerangles import euler2mat
-from .volumeutils import Recoder, array_from_file, BinOpener
+from .volumeutils import Recoder, array_from_file
 from .affines import from_matvec, dot_reduce, apply_affine
 from .nifti1 import unit_codes
 from .fileslice import fileslice, strided_scalar
+from .openers import ImageOpener
 
 # PSL to RAS affine
 PSL_TO_RAS = np.array([[0, 0, -1, 0],  # L -> R
@@ -581,13 +582,13 @@ def is_proxy(self):
         return True
 
     def get_unscaled(self):
-        with BinOpener(self.file_like) as fileobj:
+        with ImageOpener(self.file_like) as fileobj:
             return _data_from_rec(fileobj, self._rec_shape, self._dtype,
                                   self._slice_indices, self._shape,
                                   mmap=self._mmap)
 
     def __array__(self):
-        with BinOpener(self.file_like) as fileobj:
+        with ImageOpener(self.file_like) as fileobj:
             return _data_from_rec(fileobj,
                                   self._rec_shape,
                                   self._dtype,
@@ -603,7 +604,7 @@ def __getitem__(self, slicer):
             return np.asanyarray(self)[slicer]
         # Slices all sequential from zero, can use fileslice
         # This gives more efficient volume by volume loading, for example
-        with BinOpener(self.file_like) as fileobj:
+        with ImageOpener(self.file_like) as fileobj:
             raw_data = fileslice(fileobj, slicer, self._shape, self._dtype, 0,
                                  'F')
         # Broadcast scaling to shape of original data

nibabel/tests/test_helpers.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 
-from ..openers import Opener
+from ..openers import ImageOpener
 from ..tmpdirs import InTemporaryDirectory
 from ..optpkg import optional_package
 _, have_scipy, _ = optional_package('scipy.io')
@@ -49,7 +49,7 @@ def bz2_mio_error():
     import scipy.io
 
     with InTemporaryDirectory():
-        with Opener('test.mat.bz2', 'wb') as fobj:
+        with ImageOpener('test.mat.bz2', 'wb') as fobj:
             try:
                 scipy.io.savemat(fobj, {'a': 1}, format='4')
             except ValueError:

nibabel/tests/test_openers.py

@@ -10,14 +10,13 @@
 import os
 from gzip import GzipFile
 from bz2 import BZ2File
-
 from io import BytesIO, UnsupportedOperation
-from ..externals.six import PY3
-from ..py3k import asstr, asbytes
 
+from ..checkwarns import ErrorWarnings
+from ..py3k import asstr, asbytes
+from ..openers import Opener, ImageOpener
 from ..tmpdirs import InTemporaryDirectory
-
-from ..openers import Opener
+from ..volumeutils import BinOpener
 
 from nose.tools import (assert_true, assert_false, assert_equal,
                         assert_not_equal, assert_raises)
@@ -57,7 +56,6 @@ def test_Opener():
         # mode is gently ignored
         fobj = Opener(obj, mode='r')
 
-
 def test_Opener_various():
     # Check we can do all sorts of files here
     message = b"Oh what a giveaway"
@@ -85,6 +83,43 @@ def test_Opener_various():
                     # Just check there is a fileno
                     assert_not_equal(fobj.fileno(), 0)
 
+def test_BinOpener():
+    with ErrorWarnings():
+        assert_raises(DeprecationWarning,
+                      BinOpener, 'test.txt', 'r')
+
+class TestImageOpener:
+    def setUp(self):
+        self.compress_ext_map = ImageOpener.compress_ext_map.copy()
+
+    def teardown(self):
+        ImageOpener.compress_ext_map = self.compress_ext_map
+
+    def test_vanilla(self):
+        # Test that ImageOpener does add '.mgz' as gzipped file type
+        with InTemporaryDirectory():
+            with ImageOpener('test.gz', 'w') as fobj:
+                assert_true(hasattr(fobj.fobj, 'compress'))
+            with ImageOpener('test.mgz', 'w') as fobj:
+                assert_true(hasattr(fobj.fobj, 'compress'))
+
+    def test_new_association(self):
+        def file_opener(fileish, mode):
+            return open(fileish, mode)
+
+        # Add the association
+        n_associations = len(ImageOpener.compress_ext_map)
+        dec = ImageOpener.register_ext_from_image('.foo',
+                                                  (file_opener, ('mode',)))
+        dec(self.__class__)
+        assert_equal(n_associations + 1, len(ImageOpener.compress_ext_map))
+        assert_true('.foo' in ImageOpener.compress_ext_map)
+
+        with InTemporaryDirectory():
+            with ImageOpener('test.foo', 'w'):
+                pass
+            assert_true(os.path.exists('test.foo'))
+
 
 def test_file_like_wrapper():
     # Test wrapper using BytesIO (full API)

nibabel/tests/test_parrec.py

@@ -13,7 +13,7 @@
 from .. import parrec
 from ..parrec import (parse_PAR_header, PARRECHeader, PARRECError, vol_numbers,
                       vol_is_full, PARRECImage, PARRECArrayProxy, exts2pars)
-from ..openers import Opener
+from ..openers import ImageOpener
 from ..fileholders import FileHolder
 from ..volumeutils import array_from_file
 
@@ -32,7 +32,7 @@
 DATA_PATH = pjoin(dirname(__file__), 'data')
 EG_PAR = pjoin(DATA_PATH, 'phantom_EPI_asc_CLEAR_2_1.PAR')
 EG_REC = pjoin(DATA_PATH, 'phantom_EPI_asc_CLEAR_2_1.REC')
-with Opener(EG_PAR, 'rt') as _fobj:
+with ImageOpener(EG_PAR, 'rt') as _fobj:
     HDR_INFO, HDR_DEFS = parse_PAR_header(_fobj)
 # Fake truncated
 TRUNC_PAR = pjoin(DATA_PATH, 'phantom_truncated.PAR')