diff --git a/.travis.yml b/.travis.yml index 31be980ea8..64b7fea960 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ cache: -- apt + apt: true + language: python python: - 2.7 @@ -10,11 +11,7 @@ env: - INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" - INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: -- function bef_inst { - wget http://repo.continuum.io/miniconda/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh - -O /home/travis/.cache/miniconda.sh && - bash /home/travis/.cache/miniconda.sh -b -p /home/travis/miniconda && - export PATH=/home/travis/miniconda/bin:$PATH && +- function apt_inst { if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && if $INSTALL_DEB_DEPENDECIES; then sudo ln -s /run/shm /dev/shm; fi && bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) && @@ -26,18 +23,24 @@ before_install: source /etc/fsl/fsl.sh; source /etc/afni/afni.sh; export FSLOUTPUTTYPE=NIFTI_GZ; fi } -- travis_retry bef_inst -install: -# Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && -- function inst { +- function conda_inst { + export CONDA_HOME=$HOME/conda && + wget https://repo.continuum.io/miniconda/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh + -O /home/travis/.cache/conda.sh && + bash /home/travis/.cache/conda.sh -b -p ${CONDA_HOME} && + export PATH=${CONDA_HOME}/bin:$PATH && + hash -r && + conda config --set always_yes yes --set changeps1 no && + conda update -q conda && + conda install python=${TRAVIS_PYTHON_VERSION} && conda config --add channels conda-forge && - conda update --yes conda && - conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y nipype icu && - rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && - pip install -r requirements.txt && - pip install -e .[$NIPYPE_EXTRAS]; } -- travis_retry inst + rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } +# Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi +- travis_retry apt_inst +- travis_retry conda_inst +install: +- travis_retry pip install -e .[$NIPYPE_EXTRAS] script: - py.test --doctest-modules nipype deploy: diff --git a/examples/dmri_camino_dti.py b/examples/dmri_camino_dti.py index 20392d9bf5..3a9528bf10 100755 --- a/examples/dmri_camino_dti.py +++ b/examples/dmri_camino_dti.py @@ -18,6 +18,7 @@ Import necessary modules from nipype. """ +import os # system functions import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine @@ -25,7 +26,6 @@ import nipype.interfaces.fsl as fsl import nipype.interfaces.camino2trackvis as cam2trk import nipype.algorithms.misc as misc -import os # system functions """ We use the following functions to scrape the voxel and data dimensions of the input images. This allows the @@ -36,9 +36,10 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] @@ -46,9 +47,10 @@ def get_vox_dims(volume): def get_data_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header datadims = hdr.get_data_shape() return [int(datadims[0]), int(datadims[1]), int(datadims[2])] @@ -56,7 +58,8 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb - nii = nb.load(volume) + from nipype.utils import NUMPY_MMAP + nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine subject_list = ['subj1'] diff --git a/examples/dmri_connectivity.py b/examples/dmri_connectivity.py index 4b7debff8f..5ebbbb77d5 100755 --- a/examples/dmri_connectivity.py +++ b/examples/dmri_connectivity.py @@ -47,6 +47,10 @@ First, we import the necessary modules from nipype. """ +import inspect + +import os.path as op # system functions +import cmp # connectome mapper import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine @@ -56,10 +60,6 @@ import nipype.interfaces.freesurfer as fs # freesurfer import nipype.interfaces.cmtk as cmtk import nipype.algorithms.misc as misc -import inspect - -import os.path as op # system functions -import cmp # connectome mapper """ We define the following functions to scrape the voxel and data dimensions of the input images. This allows the @@ -74,9 +74,10 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] @@ -84,9 +85,10 @@ def get_vox_dims(volume): def get_data_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header datadims = hdr.get_data_shape() return [int(datadims[0]), int(datadims[1]), int(datadims[2])] @@ -94,7 +96,8 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb - nii = nb.load(volume) + from nipype.utils import NUMPY_MMAP + nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine diff --git a/examples/fmri_ants_openfmri.py b/examples/fmri_ants_openfmri.py index 05cbb4efa4..ba5ce3ce0c 100755 --- a/examples/fmri_ants_openfmri.py +++ b/examples/fmri_ants_openfmri.py @@ -36,6 +36,8 @@ from nipype.workflows.fmri.fsl import (create_featreg_preproc, create_modelfit_workflow, create_fixed_effects_flow) +from nipype.utils import NUMPY_MMAP + config.enable_provenance() version = 0 @@ -68,7 +70,7 @@ def median(in_files): """ average = None for idx, filename in enumerate(filename_to_list(in_files)): - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) data = np.median(img.get_data(), axis=3) if average is None: average = data diff --git a/examples/fmri_fsl.py b/examples/fmri_fsl.py index 7fddfc76c4..9772290200 100755 --- a/examples/fmri_fsl.py +++ b/examples/fmri_fsl.py @@ -106,10 +106,11 @@ def pickfirst(files): def getmiddlevolume(func): from nibabel import load + from nipype.utils import NUMPY_MMAP funcfile = func if isinstance(func, list): funcfile = func[0] - _, _, _, timepoints = load(funcfile).shape + _, _, _, timepoints = load(funcfile, mmap=NUMPY_MMAP).shape return int(timepoints / 2) - 1 preproc.connect(inputnode, ('func', getmiddlevolume), extract_ref, 't_min') diff --git a/examples/fmri_spm_auditory.py b/examples/fmri_spm_auditory.py index 94b6f1a565..33c03f8bf0 100755 --- a/examples/fmri_spm_auditory.py +++ b/examples/fmri_spm_auditory.py @@ -29,6 +29,7 @@ import nipype.algorithms.modelgen as model # model specification import os # system functions + """ Preliminaries @@ -120,9 +121,10 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] diff --git a/examples/fmri_spm_face.py b/examples/fmri_spm_face.py index 942bace0ff..c716e4793f 100755 --- a/examples/fmri_spm_face.py +++ b/examples/fmri_spm_face.py @@ -20,13 +20,13 @@ from __future__ import division from builtins import range +import os # system functions import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.spm as spm # spm import nipype.interfaces.matlab as mlab # how to run matlab import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification -import os # system functions """ @@ -114,9 +114,10 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] diff --git a/examples/rsfmri_vol_surface_preprocessing.py b/examples/rsfmri_vol_surface_preprocessing.py index 4b35ba37a4..77c7598f84 100644 --- a/examples/rsfmri_vol_surface_preprocessing.py +++ b/examples/rsfmri_vol_surface_preprocessing.py @@ -76,6 +76,7 @@ import scipy as sp import nibabel as nb + imports = ['import os', 'import nibabel as nb', 'import numpy as np', @@ -117,9 +118,10 @@ def median(in_files): """ import numpy as np import nibabel as nb + from nipype.utils import NUMPY_MMAP average = None for idx, filename in enumerate(filename_to_list(in_files)): - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) data = np.median(img.get_data(), axis=3) if average is None: average = data @@ -145,11 +147,12 @@ def bandpass_filter(files, lowpass_freq, highpass_freq, fs): from nipype.utils.filemanip import split_filename, list_to_filename import numpy as np import nibabel as nb + from nipype.utils import NUMPY_MMAP out_files = [] for filename in filename_to_list(files): path, name, ext = split_filename(filename) out_file = os.path.join(os.getcwd(), name + '_bp' + ext) - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) timepoints = img.shape[-1] F = np.zeros((timepoints)) lowidx = int(timepoints / 2) + 1 @@ -260,11 +263,12 @@ def extract_noise_components(realigned_file, mask_file, num_components=5, from scipy.linalg.decomp_svd import svd import numpy as np import nibabel as nb + from nipype.utils import NUMPY_MMAP import os - imgseries = nb.load(realigned_file) + imgseries = nb.load(realigned_file, mmap=NUMPY_MMAP) components = None for filename in filename_to_list(mask_file): - mask = nb.load(filename).get_data() + mask = nb.load(filename, mmap=NUMPY_MMAP).get_data() if len(np.nonzero(mask > 0)[0]) == 0: continue voxel_timecourses = imgseries.get_data()[mask > 0] @@ -329,10 +333,11 @@ def extract_subrois(timeseries_file, label_file, indices): """ from nipype.utils.filemanip import split_filename import nibabel as nb + from nipype.utils import NUMPY_MMAP import os - img = nb.load(timeseries_file) + img = nb.load(timeseries_file, mmap=NUMPY_MMAP) data = img.get_data() - roiimg = nb.load(label_file) + roiimg = nb.load(label_file, mmap=NUMPY_MMAP) rois = roiimg.get_data() prefix = split_filename(timeseries_file)[1] out_ts_file = os.path.join(os.getcwd(), '%s_subcortical_ts.txt' % prefix) @@ -352,8 +357,9 @@ def combine_hemi(left, right): """ import os import numpy as np - lh_data = nb.load(left).get_data() - rh_data = nb.load(right).get_data() + from nipype.utils import NUMPY_MMAP + lh_data = nb.load(left, mmap=NUMPY_MMAP).get_data() + rh_data = nb.load(right, mmap=NUMPY_MMAP).get_data() indices = np.vstack((1000000 + np.arange(0, lh_data.shape[0])[:, None], 2000000 + np.arange(0, rh_data.shape[0])[:, None])) diff --git a/examples/rsfmri_vol_surface_preprocessing_nipy.py b/examples/rsfmri_vol_surface_preprocessing_nipy.py index b745704023..ab1d4f14d8 100644 --- a/examples/rsfmri_vol_surface_preprocessing_nipy.py +++ b/examples/rsfmri_vol_surface_preprocessing_nipy.py @@ -75,6 +75,7 @@ import numpy as np import scipy as sp import nibabel as nb +from nipype.utils import NUMPY_MMAP imports = ['import os', 'import nibabel as nb', @@ -116,7 +117,7 @@ def median(in_files): """ average = None for idx, filename in enumerate(filename_to_list(in_files)): - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) data = np.median(img.get_data(), axis=3) if average is None: average = data @@ -143,7 +144,7 @@ def bandpass_filter(files, lowpass_freq, highpass_freq, fs): for filename in filename_to_list(files): path, name, ext = split_filename(filename) out_file = os.path.join(os.getcwd(), name + '_bp' + ext) - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) timepoints = img.shape[-1] F = np.zeros((timepoints)) lowidx = int(timepoints / 2) + 1 @@ -268,9 +269,9 @@ def extract_subrois(timeseries_file, label_file, indices): The first four columns are: freesurfer index, i, j, k positions in the label file """ - img = nb.load(timeseries_file) + img = nb.load(timeseries_file, mmap=NUMPY_MMAP) data = img.get_data() - roiimg = nb.load(label_file) + roiimg = nb.load(label_file, mmap=NUMPY_MMAP) rois = roiimg.get_data() prefix = split_filename(timeseries_file)[1] out_ts_file = os.path.join(os.getcwd(), '%s_subcortical_ts.txt' % prefix) @@ -288,8 +289,8 @@ def extract_subrois(timeseries_file, label_file, indices): def combine_hemi(left, right): """Combine left and right hemisphere time series into a single text file """ - lh_data = nb.load(left).get_data() - rh_data = nb.load(right).get_data() + lh_data = nb.load(left, mmap=NUMPY_MMAP).get_data() + rh_data = nb.load(right, mmap=NUMPY_MMAP).get_data() indices = np.vstack((1000000 + np.arange(0, lh_data.shape[0])[:, None], 2000000 + np.arange(0, rh_data.shape[0])[:, None])) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 087b273190..c6503c703a 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -20,15 +20,18 @@ import nibabel as nb import numpy as np from numpy.polynomial import Legendre -from scipy import linalg, signal +from scipy import linalg from .. import logging from ..external.due import BibTeX from ..interfaces.base import (traits, TraitedSpec, BaseInterface, BaseInterfaceInputSpec, File, isdefined, InputMultiPath) +from nipype.utils import NUMPY_MMAP + IFLOG = logging.getLogger('interface') + class ComputeDVARSInputSpec(BaseInterfaceInputSpec): in_file = File(exists=True, mandatory=True, desc='functional data, after HMC') in_mask = File(exists=True, mandatory=True, desc='a brain mask') @@ -127,9 +130,9 @@ def _run_interface(self, runtime): dvars = compute_dvars(self.inputs.in_file, self.inputs.in_mask, remove_zerovariance=self.inputs.remove_zerovariance) - self._results['avg_std'] = float(dvars[0].mean()) - self._results['avg_nstd'] = float(dvars[1].mean()) - self._results['avg_vxstd'] = float(dvars[2].mean()) + (self._results['avg_std'], + self._results['avg_nstd'], + self._results['avg_vxstd']) = np.mean(dvars, axis=1).astype(float) tr = None if isdefined(self.inputs.series_tr): @@ -329,8 +332,8 @@ class CompCor(BaseInterface): }] def _run_interface(self, runtime): - imgseries = nb.load(self.inputs.realigned_file).get_data() - mask = nb.load(self.inputs.mask_file).get_data() + imgseries = nb.load(self.inputs.realigned_file, mmap=NUMPY_MMAP).get_data() + mask = nb.load(self.inputs.mask_file, mmap=NUMPY_MMAP).get_data() if imgseries.shape[:3] != mask.shape: raise ValueError('Inputs for CompCor, func {} and mask {}, do not have matching ' @@ -435,7 +438,7 @@ class TCompCor(CompCor): output_spec = TCompCorOutputSpec def _run_interface(self, runtime): - imgseries = nb.load(self.inputs.realigned_file).get_data() + imgseries = nb.load(self.inputs.realigned_file, mmap=NUMPY_MMAP).get_data() if imgseries.ndim != 4: raise ValueError('tCompCor expected a 4-D nifti file. Input {} has {} dimensions ' @@ -443,7 +446,7 @@ def _run_interface(self, runtime): .format(self.inputs.realigned_file, imgseries.ndim, imgseries.shape)) if isdefined(self.inputs.mask_file): - in_mask_data = nb.load(self.inputs.mask_file).get_data() + in_mask_data = nb.load(self.inputs.mask_file, mmap=NUMPY_MMAP).get_data() imgseries = imgseries[in_mask_data != 0, :] # From the paper: @@ -521,9 +524,9 @@ class TSNR(BaseInterface): output_spec = TSNROutputSpec def _run_interface(self, runtime): - img = nb.load(self.inputs.in_file[0]) + img = nb.load(self.inputs.in_file[0], mmap=NUMPY_MMAP) header = img.header.copy() - vollist = [nb.load(filename) for filename in self.inputs.in_file] + vollist = [nb.load(filename, mmap=NUMPY_MMAP) for filename in self.inputs.in_file] data = np.concatenate([vol.get_data().reshape( vol.get_shape()[:3] + (-1,)) for vol in vollist], axis=3) data = np.nan_to_num(data) @@ -626,8 +629,8 @@ def compute_dvars(in_file, in_mask, remove_zerovariance=False): from nitime.algorithms import AR_est_YW import warnings - func = nb.load(in_file).get_data().astype(np.float32) - mask = nb.load(in_mask).get_data().astype(np.uint8) + func = nb.load(in_file, mmap=NUMPY_MMAP).get_data().astype(np.float32) + mask = nb.load(in_mask, mmap=NUMPY_MMAP).get_data().astype(np.uint8) if len(func.shape) != 4: raise RuntimeError( diff --git a/nipype/algorithms/icc.py b/nipype/algorithms/icc.py index 3a9e8237e7..36d2cb4221 100644 --- a/nipype/algorithms/icc.py +++ b/nipype/algorithms/icc.py @@ -8,6 +8,7 @@ from scipy.linalg import pinv from ..interfaces.base import BaseInterfaceInputSpec, TraitedSpec, \ BaseInterface, traits, File +from nipype.utils import NUMPY_MMAP class ICCInputSpec(BaseInterfaceInputSpec): @@ -37,7 +38,7 @@ def _run_interface(self, runtime): maskdata = nb.load(self.inputs.mask).get_data() maskdata = np.logical_not(np.logical_or(maskdata == 0, np.isnan(maskdata))) - session_datas = [[nb.load(fname).get_data()[maskdata].reshape(-1, 1) for fname in sessions] for sessions in self.inputs.subjects_sessions] + session_datas = [[nb.load(fname, mmap=NUMPY_MMAP).get_data()[maskdata].reshape(-1, 1) for fname in sessions] for sessions in self.inputs.subjects_sessions] list_of_sessions = [np.dstack(session_data) for session_data in session_datas] all_data = np.hstack(list_of_sessions) icc = np.zeros(session_datas[0][0].shape) diff --git a/nipype/algorithms/metrics.py b/nipype/algorithms/metrics.py index f2ea7594c6..a184dbad20 100644 --- a/nipype/algorithms/metrics.py +++ b/nipype/algorithms/metrics.py @@ -30,6 +30,8 @@ from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File, InputMultiPath, BaseInterfaceInputSpec, isdefined) +from nipype.utils import NUMPY_MMAP + iflogger = logging.getLogger('interface') @@ -410,8 +412,8 @@ def _run_interface(self, runtime): assert(ncomp == len(self.inputs.in_tst)) weights = np.ones(shape=ncomp) - img_ref = np.array([nb.load(fname).get_data() for fname in self.inputs.in_ref]) - img_tst = np.array([nb.load(fname).get_data() for fname in self.inputs.in_tst]) + img_ref = np.array([nb.load(fname, mmap=NUMPY_MMAP).get_data() for fname in self.inputs.in_ref]) + img_tst = np.array([nb.load(fname, mmap=NUMPY_MMAP).get_data() for fname in self.inputs.in_tst]) msk = np.sum(img_ref, axis=0) msk[msk > 0] = 1.0 diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 58a2276af6..b69bc7baaf 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -34,6 +34,8 @@ BaseInterfaceInputSpec, isdefined, DynamicTraitedSpec, Undefined) from ..utils.filemanip import fname_presuffix, split_filename +from nipype.utils import NUMPY_MMAP + from . import confounds iflogger = logging.getLogger('interface') @@ -140,7 +142,7 @@ class SimpleThreshold(BaseInterface): def _run_interface(self, runtime): for fname in self.inputs.volumes: - img = nb.load(fname) + img = nb.load(fname, mmap=NUMPY_MMAP) data = np.array(img.get_data()) active_map = data > self.inputs.threshold @@ -196,7 +198,7 @@ def _gen_output_filename(self, name): def _run_interface(self, runtime): for fname in self.inputs.volumes: - img = nb.load(fname) + img = nb.load(fname, mmap=NUMPY_MMAP) affine = img.affine affine = np.dot(self.inputs.transformation_matrix, affine) @@ -1158,7 +1160,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): Returns the input tissue probability maps (tpms, aka volume fractions) normalized to sum up 1.0 at each voxel within the mask. """ - import nibabel as nib + import nibabel as nb import numpy as np import os.path as op @@ -1174,7 +1176,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): out_file = op.abspath('%s_norm_%02d%s' % (fname, i, fext)) out_files += [out_file] - imgs = [nib.load(fim) for fim in in_files] + imgs = [nb.load(fim, mmap=NUMPY_MMAP) for fim in in_files] if len(in_files) == 1: img_data = imgs[0].get_data() @@ -1182,7 +1184,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): hdr = imgs[0].header.copy() hdr['data_type'] = 16 hdr.set_data_dtype(np.float32) - nib.save(nib.Nifti1Image(img_data.astype(np.float32), imgs[0].affine, + nb.save(nb.Nifti1Image(img_data.astype(np.float32), imgs[0].affine, hdr), out_files[0]) return out_files[0] @@ -1195,7 +1197,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): msk[weights <= 0] = 0 if in_mask is not None: - msk = nib.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk[msk <= 0] = 0 msk[msk > 0] = 1 @@ -1207,7 +1209,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): hdr = imgs[i].header.copy() hdr['data_type'] = 16 hdr.set_data_dtype('float32') - nib.save(nib.Nifti1Image(probmap.astype(np.float32), imgs[i].affine, + nb.save(nb.Nifti1Image(probmap.astype(np.float32), imgs[i].affine, hdr), out_file) return out_files @@ -1225,7 +1227,7 @@ def split_rois(in_file, mask=None, roishape=None): if roishape is None: roishape = (10, 10, 1) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) imshape = im.shape dshape = imshape[:3] nvols = imshape[-1] @@ -1233,7 +1235,7 @@ def split_rois(in_file, mask=None, roishape=None): droishape = (roishape[0], roishape[1], roishape[2], nvols) if mask is not None: - mask = nb.load(mask).get_data() + mask = nb.load(mask, mmap=NUMPY_MMAP).get_data() mask[mask > 0] = 1 mask[mask < 1] = 0 else: @@ -1314,7 +1316,7 @@ def merge_rois(in_files, in_idxs, in_ref, except: pass - ref = nb.load(in_ref) + ref = nb.load(in_ref, mmap=NUMPY_MMAP) aff = ref.affine hdr = ref.header.copy() rsh = ref.shape @@ -1335,7 +1337,7 @@ def merge_rois(in_files, in_idxs, in_ref, for cname, iname in zip(in_files, in_idxs): f = np.load(iname) idxs = np.squeeze(f['arr_0']) - cdata = nb.load(cname).get_data().reshape(-1, ndirs) + cdata = nb.load(cname, mmap=NUMPY_MMAP).get_data().reshape(-1, ndirs) nels = len(idxs) idata = (idxs, ) try: @@ -1363,15 +1365,15 @@ def merge_rois(in_files, in_idxs, in_ref, idxs = np.squeeze(f['arr_0']) for d, fname in enumerate(nii): - data = nb.load(fname).get_data().reshape(-1) - cdata = nb.load(cname).get_data().reshape(-1, ndirs)[:, d] + data = nb.load(fname, mmap=NUMPY_MMAP).get_data().reshape(-1) + cdata = nb.load(cname, mmap=NUMPY_MMAP).get_data().reshape(-1, ndirs)[:, d] nels = len(idxs) idata = (idxs, ) data[idata] = cdata[0:nels] nb.Nifti1Image(data.reshape(rsh[:3]), aff, hdr).to_filename(fname) - imgs = [nb.load(im) for im in nii] + imgs = [nb.load(im, mmap=NUMPY_MMAP) for im in nii] allim = nb.concat_images(imgs) allim.to_filename(out_file) diff --git a/nipype/algorithms/modelgen.py b/nipype/algorithms/modelgen.py index 3cafa8c0ac..3a0b2ef8b0 100644 --- a/nipype/algorithms/modelgen.py +++ b/nipype/algorithms/modelgen.py @@ -28,6 +28,7 @@ import numpy as np from scipy.special import gammaln +from ..utils import NUMPY_MMAP from ..interfaces.base import (BaseInterface, TraitedSpec, InputMultiPath, traits, File, Bunch, BaseInterfaceInputSpec, isdefined) @@ -354,7 +355,7 @@ def _generate_standard_design(self, infolist, for i, out in enumerate(outliers): numscans = 0 for f in filename_to_list(sessinfo[i]['scans']): - shape = load(f).shape + shape = load(f, mmap=NUMPY_MMAP).shape if len(shape) == 3 or shape[3] == 1: iflogger.warning(("You are using 3D instead of 4D " "files. Are you sure this was " @@ -457,7 +458,7 @@ def _concatenate_info(self, infolist): if isinstance(f, list): numscans = len(f) elif isinstance(f, (str, bytes)): - img = load(f) + img = load(f, mmap=NUMPY_MMAP) numscans = img.shape[3] else: raise Exception('Functional input not specified correctly') @@ -781,7 +782,7 @@ def _generate_clustered_design(self, infolist): infoout[i].onsets = None infoout[i].durations = None if info.conditions: - img = load(self.inputs.functional_runs[i]) + img = load(self.inputs.functional_runs[i], mmap=NUMPY_MMAP) nscans = img.shape[3] reg, regnames = self._cond_to_regress(info, nscans) if hasattr(infoout[i], 'regressors') and infoout[i].regressors: diff --git a/nipype/algorithms/rapidart.py b/nipype/algorithms/rapidart.py index 084005b25b..3b3295eab0 100644 --- a/nipype/algorithms/rapidart.py +++ b/nipype/algorithms/rapidart.py @@ -29,6 +29,7 @@ from scipy import signal import scipy.io as sio +from ..utils import NUMPY_MMAP from ..interfaces.base import (BaseInterface, traits, InputMultiPath, OutputMultiPath, TraitedSpec, File, BaseInterfaceInputSpec, isdefined) @@ -352,12 +353,12 @@ def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None): # read in functional image if isinstance(imgfile, (str, bytes)): - nim = load(imgfile) + nim = load(imgfile, mmap=NUMPY_MMAP) elif isinstance(imgfile, list): if len(imgfile) == 1: - nim = load(imgfile[0]) + nim = load(imgfile[0], mmap=NUMPY_MMAP) else: - images = [load(f) for f in imgfile] + images = [load(f, mmap=NUMPY_MMAP) for f in imgfile] nim = funcs.concat_images(images) # compute global intensity signal @@ -394,7 +395,7 @@ def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None): mask[:, :, :, t0] = mask_tmp g[t0] = np.nansum(vol * mask_tmp) / np.nansum(mask_tmp) elif masktype == 'file': # uses a mask image to determine intensity - maskimg = load(self.inputs.mask_file) + maskimg = load(self.inputs.mask_file, mmap=NUMPY_MMAP) mask = maskimg.get_data() affine = maskimg.affine mask = mask > 0.5 diff --git a/nipype/algorithms/tests/test_errormap.py b/nipype/algorithms/tests/test_errormap.py index a86c932ca4..a700725e41 100644 --- a/nipype/algorithms/tests/test_errormap.py +++ b/nipype/algorithms/tests/test_errormap.py @@ -4,7 +4,7 @@ import pytest from nipype.testing import example_data from nipype.algorithms.metrics import ErrorMap -import nibabel as nib +import nibabel as nb import numpy as np import os @@ -18,13 +18,13 @@ def test_errormap(tmpdir): volume2 = np.array([[[0.0, 7.0], [2.0, 3.0]], [[1.0, 9.0], [1.0, 2.0]]]) # Alan Turing's birthday mask = np.array([[[1, 0], [0, 1]], [[1, 0], [0, 1]]]) - img1 = nib.Nifti1Image(volume1, np.eye(4)) - img2 = nib.Nifti1Image(volume2, np.eye(4)) - maskimg = nib.Nifti1Image(mask, np.eye(4)) + img1 = nb.Nifti1Image(volume1, np.eye(4)) + img2 = nb.Nifti1Image(volume2, np.eye(4)) + maskimg = nb.Nifti1Image(mask, np.eye(4)) - nib.save(img1, os.path.join(tempdir, 'von.nii.gz')) - nib.save(img2, os.path.join(tempdir, 'alan.nii.gz')) - nib.save(maskimg, os.path.join(tempdir, 'mask.nii.gz')) + nb.save(img1, os.path.join(tempdir, 'von.nii.gz')) + nb.save(img2, os.path.join(tempdir, 'alan.nii.gz')) + nb.save(maskimg, os.path.join(tempdir, 'mask.nii.gz')) # Default metric errmap = ErrorMap() @@ -55,15 +55,15 @@ def test_errormap(tmpdir): msvolume1 = np.zeros(shape=(2, 2, 2, 2)) msvolume1[:, :, :, 0] = volume1 msvolume1[:, :, :, 1] = volume3 - msimg1 = nib.Nifti1Image(msvolume1, np.eye(4)) + msimg1 = nb.Nifti1Image(msvolume1, np.eye(4)) msvolume2 = np.zeros(shape=(2, 2, 2, 2)) msvolume2[:, :, :, 0] = volume3 msvolume2[:, :, :, 1] = volume1 - msimg2 = nib.Nifti1Image(msvolume2, np.eye(4)) + msimg2 = nb.Nifti1Image(msvolume2, np.eye(4)) - nib.save(msimg1, os.path.join(tempdir, 'von-ray.nii.gz')) - nib.save(msimg2, os.path.join(tempdir, 'alan-ray.nii.gz')) + nb.save(msimg1, os.path.join(tempdir, 'von-ray.nii.gz')) + nb.save(msimg2, os.path.join(tempdir, 'alan-ray.nii.gz')) errmap.inputs.in_tst = os.path.join(tempdir, 'von-ray.nii.gz') errmap.inputs.in_ref = os.path.join(tempdir, 'alan-ray.nii.gz') diff --git a/nipype/algorithms/tests/test_normalize_tpms.py b/nipype/algorithms/tests/test_normalize_tpms.py index 907c1d9619..19a183bee0 100644 --- a/nipype/algorithms/tests/test_normalize_tpms.py +++ b/nipype/algorithms/tests/test_normalize_tpms.py @@ -14,13 +14,14 @@ import nipype.testing as nit from nipype.algorithms.misc import normalize_tpms +from nipype.utils import NUMPY_MMAP def test_normalize_tpms(tmpdir): tempdir = str(tmpdir) in_mask = example_data('tpms_msk.nii.gz') - mskdata = nb.load(in_mask).get_data() + mskdata = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() mskdata[mskdata > 0.0] = 1.0 mapdata = [] @@ -32,7 +33,7 @@ def test_normalize_tpms(tmpdir): filename = os.path.join(tempdir, 'modtpm_%02d.nii.gz' % i) out_files.append(os.path.join(tempdir, 'normtpm_%02d.nii.gz' % i)) - im = nb.load(mapname) + im = nb.load(mapname, mmap=NUMPY_MMAP) data = im.get_data() mapdata.append(data.copy()) @@ -45,7 +46,7 @@ def test_normalize_tpms(tmpdir): sumdata = np.zeros_like(mskdata) for i, tstfname in enumerate(out_files): - normdata = nb.load(tstfname).get_data() + normdata = nb.load(tstfname, mmap=NUMPY_MMAP).get_data() sumdata += normdata assert np.all(normdata[mskdata == 0] == 0) assert np.allclose(normdata, mapdata[i]) diff --git a/nipype/algorithms/tests/test_splitmerge.py b/nipype/algorithms/tests/test_splitmerge.py index 46cd05f995..e122fef077 100644 --- a/nipype/algorithms/tests/test_splitmerge.py +++ b/nipype/algorithms/tests/test_splitmerge.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from nipype.testing import example_data +from nipype.utils import NUMPY_MMAP def test_split_and_merge(tmpdir): @@ -14,8 +15,8 @@ def test_split_and_merge(tmpdir): in_mask = example_data('tpms_msk.nii.gz') dwfile = op.join(str(tmpdir), 'dwi.nii.gz') - mskdata = nb.load(in_mask).get_data() - aff = nb.load(in_mask).affine + mskdata = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() + aff = nb.load(in_mask, mmap=NUMPY_MMAP).affine dwshape = (mskdata.shape[0], mskdata.shape[1], mskdata.shape[2], 6) dwdata = np.random.normal(size=dwshape) @@ -25,7 +26,7 @@ def test_split_and_merge(tmpdir): resdw, resmsk, resid = split_rois(dwfile, in_mask, roishape=(20, 20, 2)) merged = merge_rois(resdw, resid, in_mask) - dwmerged = nb.load(merged).get_data() + dwmerged = nb.load(merged, mmap=NUMPY_MMAP).get_data() dwmasked = dwdata * mskdata[:, :, :, np.newaxis] diff --git a/nipype/interfaces/cmtk/cmtk.py b/nipype/interfaces/cmtk/cmtk.py index 8903321ba5..7d65af99a7 100644 --- a/nipype/interfaces/cmtk/cmtk.py +++ b/nipype/interfaces/cmtk/cmtk.py @@ -22,6 +22,8 @@ from ... import logging from ...utils.filemanip import split_filename +from ...utils import NUMPY_MMAP + from ..base import (BaseInterface, BaseInterfaceInputSpec, traits, File, TraitedSpec, Directory, OutputMultiPath, isdefined) iflogger = logging.getLogger('interface') @@ -183,7 +185,7 @@ def cmat(track_file, roi_file, resolution_network_file, matrix_name, matrix_mat_ fib, hdr = nb.trackvis.read(track_file, False) stats['orig_n_fib'] = len(fib) - roi = nb.load(roi_file) + roi = nb.load(roi_file, mmap=NUMPY_MMAP) roiData = roi.get_data() roiVoxelSize = roi.header.get_zooms() (endpoints, endpointsmm) = create_endpoints_array(fib, roiVoxelSize) @@ -619,7 +621,7 @@ def _run_interface(self, runtime): aparc_aseg_file = self.inputs.aparc_aseg_file aparcpath, aparcname, aparcext = split_filename(aparc_aseg_file) iflogger.info('Using Aparc+Aseg file: {name}'.format(name=aparcname + aparcext)) - niiAPARCimg = nb.load(aparc_aseg_file) + niiAPARCimg = nb.load(aparc_aseg_file, mmap=NUMPY_MMAP) niiAPARCdata = niiAPARCimg.get_data() niiDataLabels = np.unique(niiAPARCdata) numDataLabels = np.size(niiDataLabels) @@ -742,7 +744,7 @@ def _gen_outfilename(self, ext): def create_nodes(roi_file, resolution_network_file, out_filename): G = nx.Graph() gp = nx.read_graphml(resolution_network_file) - roi_image = nb.load(roi_file) + roi_image = nb.load(roi_file, mmap=NUMPY_MMAP) roiData = roi_image.get_data() nROIs = len(gp.nodes()) for u, d in gp.nodes_iter(data=True): diff --git a/nipype/interfaces/dcmstack.py b/nipype/interfaces/dcmstack.py index ca1e8d7d1b..e9dab240f6 100644 --- a/nipype/interfaces/dcmstack.py +++ b/nipype/interfaces/dcmstack.py @@ -23,6 +23,7 @@ InputMultiPath, File, Directory, traits, BaseInterface) from .traits_extension import isdefined, Undefined +from ..utils import NUMPY_MMAP have_dcmstack = True @@ -357,7 +358,7 @@ class MergeNifti(NiftiGeneratorBase): output_spec = MergeNiftiOutputSpec def _run_interface(self, runtime): - niis = [nb.load(fn) + niis = [nb.load(fn, mmap=NUMPY_MMAP) for fn in self.inputs.in_files ] nws = [NiftiWrapper(nii, make_empty=True) diff --git a/nipype/interfaces/dipy/preprocess.py b/nipype/interfaces/dipy/preprocess.py index 163118e0cb..8bc28dbdbd 100644 --- a/nipype/interfaces/dipy/preprocess.py +++ b/nipype/interfaces/dipy/preprocess.py @@ -16,6 +16,8 @@ from ... import logging from ..base import (traits, TraitedSpec, File, isdefined) from .base import DipyBaseInterface +from ...utils import NUMPY_MMAP + IFLOGGER = logging.getLogger('interface') @@ -186,7 +188,7 @@ def resample_proxy(in_file, order=3, new_zooms=None, out_file=None): fext = fext2 + fext out_file = op.abspath('./%s_reslice%s' % (fname, fext)) - img = nb.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) hdr = img.header.copy() data = img.get_data().astype(np.float32) affine = img.affine @@ -229,7 +231,7 @@ def nlmeans_proxy(in_file, settings, fext = fext2 + fext out_file = op.abspath('./%s_denoise%s' % (fname, fext)) - img = nb.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) hdr = img.header data = img.get_data() aff = img.affine diff --git a/nipype/interfaces/dipy/simulate.py b/nipype/interfaces/dipy/simulate.py index 0baa50244b..0331171811 100644 --- a/nipype/interfaces/dipy/simulate.py +++ b/nipype/interfaces/dipy/simulate.py @@ -13,6 +13,7 @@ import nibabel as nb from ... import logging +from ...utils import NUMPY_MMAP from ..base import (traits, TraitedSpec, BaseInterfaceInputSpec, File, InputMultiPath, isdefined) from .base import DipyBaseInterface @@ -118,7 +119,7 @@ def _run_interface(self, runtime): # Volume fractions of isotropic compartments nballs = len(self.inputs.in_vfms) vfs = np.squeeze(nb.concat_images( - [nb.load(f) for f in self.inputs.in_vfms]).get_data()) + [nb.load(f, mmap=NUMPY_MMAP) for f in self.inputs.in_vfms]).get_data()) if nballs == 1: vfs = vfs[..., np.newaxis] total_vf = np.sum(vfs, axis=3) @@ -136,7 +137,7 @@ def _run_interface(self, runtime): nvox = len(msk[msk > 0]) # Fiber fractions - ffsim = nb.concat_images([nb.load(f) for f in self.inputs.in_frac]) + ffsim = nb.concat_images([nb.load(f, mmap=NUMPY_MMAP) for f in self.inputs.in_frac]) ffs = np.nan_to_num(np.squeeze(ffsim.get_data())) # fiber fractions ffs = np.clip(ffs, 0., 1.) if nsticks == 1: @@ -179,7 +180,7 @@ def _run_interface(self, runtime): dirs = None for i in range(nsticks): f = self.inputs.in_dirs[i] - fd = np.nan_to_num(nb.load(f).get_data()) + fd = np.nan_to_num(nb.load(f, mmap=NUMPY_MMAP).get_data()) w = np.linalg.norm(fd, axis=3)[..., np.newaxis] w[w < np.finfo(float).eps] = 1.0 fd /= w diff --git a/nipype/interfaces/freesurfer/tests/test_model.py b/nipype/interfaces/freesurfer/tests/test_model.py index d9da543154..28e49401e0 100644 --- a/nipype/interfaces/freesurfer/tests/test_model.py +++ b/nipype/interfaces/freesurfer/tests/test_model.py @@ -4,9 +4,11 @@ import os import numpy as np -import nibabel as nib +import nibabel as nb import pytest + +from nipype.utils import NUMPY_MMAP from nipype.interfaces.freesurfer import model, no_freesurfer import nipype.pipeline.engine as pe @@ -24,18 +26,18 @@ def test_concatenate(tmpdir): out_data = np.concatenate((data1, data2), axis=3) mean_data = np.mean(out_data, axis=3) - nib.Nifti1Image(data1, affine=np.eye(4)).to_filename(in1) - nib.Nifti1Image(data2, affine=np.eye(4)).to_filename(in2) + nb.Nifti1Image(data1, affine=np.eye(4)).to_filename(in1) + nb.Nifti1Image(data2, affine=np.eye(4)).to_filename(in2) # Test default behavior res = model.Concatenate(in_files=[in1, in2]).run() assert res.outputs.concatenated_file == os.path.join(tempdir, 'concat_output.nii.gz') - assert np.allclose(nib.load('concat_output.nii.gz').get_data(), out_data) + assert np.allclose(nb.load('concat_output.nii.gz').get_data(), out_data) # Test specified concatenated_file res = model.Concatenate(in_files=[in1, in2], concatenated_file=out).run() assert res.outputs.concatenated_file == os.path.join(tempdir, out) - assert np.allclose(nib.load(out).get_data(), out_data) + assert np.allclose(nb.load(out, mmap=NUMPY_MMAP).get_data(), out_data) # Test in workflow wf = pe.Workflow('test_concatenate', base_dir=tempdir) @@ -44,7 +46,7 @@ def test_concatenate(tmpdir): name='concat') wf.add_nodes([concat]) wf.run() - assert np.allclose(nib.load(os.path.join(tempdir, + assert np.allclose(nb.load(os.path.join(tempdir, 'test_concatenate', 'concat', out)).get_data(), out_data) @@ -52,4 +54,4 @@ def test_concatenate(tmpdir): # Test a simple statistic res = model.Concatenate(in_files=[in1, in2], concatenated_file=out, stats='mean').run() - assert np.allclose(nib.load(out).get_data(), mean_data) + assert np.allclose(nb.load(out, mmap=NUMPY_MMAP).get_data(), mean_data) diff --git a/nipype/interfaces/fsl/epi.py b/nipype/interfaces/fsl/epi.py index c854c0400c..2a54ec14b4 100644 --- a/nipype/interfaces/fsl/epi.py +++ b/nipype/interfaces/fsl/epi.py @@ -17,10 +17,12 @@ import os import numpy as np -import nibabel as nib +import nibabel as nb import warnings from ...utils.filemanip import split_filename +from ...utils import NUMPY_MMAP + from ..base import (traits, TraitedSpec, InputMultiPath, File, isdefined) from .base import FSLCommand, FSLCommandInputSpec @@ -102,11 +104,11 @@ def _run_interface(self, runtime): if runtime.returncode == 0: out_file = self.inputs.out_fieldmap - im = nib.load(out_file) - dumb_img = nib.Nifti1Image(np.zeros(im.shape), im.affine, + im = nb.load(out_file, mmap=NUMPY_MMAP) + dumb_img = nb.Nifti1Image(np.zeros(im.shape), im.affine, im.header) - out_nii = nib.funcs.concat_images((im, dumb_img)) - nib.save(out_nii, out_file) + out_nii = nb.funcs.concat_images((im, dumb_img)) + nb.save(out_nii, out_file) return runtime diff --git a/nipype/interfaces/nipy/model.py b/nipype/interfaces/nipy/model.py index 584152d01e..58c07b5b95 100644 --- a/nipype/interfaces/nipy/model.py +++ b/nipype/interfaces/nipy/model.py @@ -8,6 +8,8 @@ import numpy as np from ...utils.misc import package_check +from ...utils import NUMPY_MMAP + from ..base import (BaseInterface, TraitedSpec, traits, File, OutputMultiPath, BaseInterfaceInputSpec, isdefined) @@ -99,7 +101,7 @@ def _run_interface(self, runtime): del data for functional_run in functional_runs[1:]: - nii = nb.load(functional_run) + nii = nb.load(functional_run, mmap=NUMPY_MMAP) data = nii.get_data() npdata = data.copy() del data diff --git a/nipype/interfaces/nipy/preprocess.py b/nipype/interfaces/nipy/preprocess.py index 579f9f7988..c4b3ece2f1 100644 --- a/nipype/interfaces/nipy/preprocess.py +++ b/nipype/interfaces/nipy/preprocess.py @@ -16,6 +16,8 @@ import numpy as np from ...utils.misc import package_check +from ...utils import NUMPY_MMAP + from ...utils.filemanip import split_filename, fname_presuffix from ..base import (TraitedSpec, BaseInterface, traits, BaseInterfaceInputSpec, isdefined, File, @@ -61,7 +63,7 @@ def _run_interface(self, runtime): value = getattr(self.inputs, key) if isdefined(value): if key in ['mean_volume', 'reference_volume']: - nii = nb.load(value) + nii = nb.load(value, mmap=NUMPY_MMAP) value = nii.get_data() args[key] = value diff --git a/nipype/interfaces/spm/base.py b/nipype/interfaces/spm/base.py index 09c9595a6b..25ab67d412 100644 --- a/nipype/interfaces/spm/base.py +++ b/nipype/interfaces/spm/base.py @@ -28,7 +28,7 @@ # Local imports from ... import logging -from ...utils import spm_docs as sd +from ...utils import spm_docs as sd, NUMPY_MMAP from ..base import (BaseInterface, traits, isdefined, InputMultiPath, BaseInterfaceInputSpec, Directory, Undefined) from ..matlab import MatlabCommand @@ -45,7 +45,7 @@ def func_is_3d(in_file): if isinstance(in_file, list): return func_is_3d(in_file[0]) else: - img = load(in_file) + img = load(in_file, mmap=NUMPY_MMAP) shape = img.shape if len(shape) == 3 or (len(shape) == 4 and shape[3] == 1): return True @@ -73,7 +73,7 @@ def scans_for_fname(fname): for sno, f in enumerate(fname): scans[sno] = '%s,1' % f return scans - img = load(fname) + img = load(fname, mmap=NUMPY_MMAP) if len(img.shape) == 3: return np.array(('%s,1' % fname,), dtype=object) else: diff --git a/nipype/utils/__init__.py b/nipype/utils/__init__.py index 691947f82f..decca90323 100644 --- a/nipype/utils/__init__.py +++ b/nipype/utils/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +from nipype.utils.config import NUMPY_MMAP from nipype.utils.onetime import OneTimeProperty, setattr_on_read from nipype.utils.tmpdirs import TemporaryDirectory, InTemporaryDirectory diff --git a/nipype/utils/config.py b/nipype/utils/config.py index 3bbeb22323..7130c100e3 100644 --- a/nipype/utils/config.py +++ b/nipype/utils/config.py @@ -10,22 +10,25 @@ @author: Chris Filo Gorgolewski ''' from __future__ import print_function, division, unicode_literals, absolute_import -from builtins import str, object, open - -from future import standard_library -standard_library.install_aliases() - -import configparser import os import shutil import errno from warnings import warn from io import StringIO +from distutils.version import LooseVersion from simplejson import load, dump +import numpy as np +from builtins import str, object, open +from future import standard_library +standard_library.install_aliases() + +import configparser from ..external import portalocker +NUMPY_MMAP = LooseVersion(np.__version__) >= LooseVersion('1.12.0') + # Get home directory in platform-agnostic way homedir = os.path.expanduser('~') default_cfg = """ diff --git a/nipype/workflows/dmri/dipy/denoise.py b/nipype/workflows/dmri/dipy/denoise.py index c060bb97c1..14e85c938c 100644 --- a/nipype/workflows/dmri/dipy/denoise.py +++ b/nipype/workflows/dmri/dipy/denoise.py @@ -4,6 +4,7 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: from builtins import range +from nipype.utils import NUMPY_MMAP from ....pipeline import engine as pe from ....interfaces import utility as niu from ....interfaces import dipy @@ -64,12 +65,12 @@ def csf_mask(in_file, in_mask, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_csfmask%s" % (fname, ext)) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) hdr = im.header.copy() hdr.set_data_dtype(np.uint8) hdr.set_xyzt_units('mm') imdata = im.get_data() - msk = nb.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk = binary_erosion(msk, structure=np.ones((15, 15, 10))).astype(np.uint8) thres = np.percentile(imdata[msk > 0].reshape(-1), 90.0) @@ -107,12 +108,12 @@ def bg_mask(in_file, in_mask, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_bgmask%s" % (fname, ext)) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) hdr = im.header.copy() hdr.set_data_dtype(np.uint8) hdr.set_xyzt_units('mm') imdata = im.get_data() - msk = nb.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk = 1 - binary_dilation(msk, structure=np.ones((20, 20, 20))) nb.Nifti1Image(msk.astype(np.uint8), im.affine, hdr).to_filename(out_file) return out_file diff --git a/nipype/workflows/dmri/fsl/artifacts.py b/nipype/workflows/dmri/fsl/artifacts.py index 906541829f..237d7b2e01 100644 --- a/nipype/workflows/dmri/fsl/artifacts.py +++ b/nipype/workflows/dmri/fsl/artifacts.py @@ -4,6 +4,8 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: from __future__ import division +from nipype.utils import NUMPY_MMAP + from ....interfaces.io import JSONFileGrabber from ....interfaces import utility as niu from ....interfaces import ants @@ -235,7 +237,7 @@ def _gen_index(in_file): import nibabel as nb import os out_file = os.path.abspath('index.txt') - vols = nb.load(in_file).get_data().shape[-1] + vols = nb.load(in_file, mmap=NUMPY_MMAP).get_data().shape[-1] np.savetxt(out_file, np.ones((vols,)).T) return out_file @@ -900,7 +902,7 @@ def _xfm_jacobian(in_xfm): def _get_zoom(in_file, enc_dir): import nibabel as nb - zooms = nb.load(in_file).header.get_zooms() + zooms = nb.load(in_file, mmap=NUMPY_MMAP).header.get_zooms() if 'y' in enc_dir: return zooms[1] diff --git a/nipype/workflows/dmri/fsl/epi.py b/nipype/workflows/dmri/fsl/epi.py index 16bcab3ab6..de023215ab 100644 --- a/nipype/workflows/dmri/fsl/epi.py +++ b/nipype/workflows/dmri/fsl/epi.py @@ -5,6 +5,7 @@ import warnings +from nipype.utils import NUMPY_MMAP from ....pipeline import engine as pe from ....interfaces import utility as niu from ....interfaces import fsl as fsl @@ -718,10 +719,10 @@ def _effective_echospacing(dwell_time, pi_factor=1.0): def _prepare_phasediff(in_file): - import nibabel as nib + import nibabel as nb import os import numpy as np - img = nib.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) max_diff = np.max(img.get_data().reshape(-1)) min_diff = np.min(img.get_data().reshape(-1)) A = (2.0 * np.pi) / (max_diff - min_diff) @@ -732,47 +733,47 @@ def _prepare_phasediff(in_file): if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_2pi.nii.gz' % name) - nib.save(nib.Nifti1Image(diff_norm, img.affine, img.header), out_file) + nb.save(nb.Nifti1Image(diff_norm, img.affine, img.header), out_file) return out_file def _dilate_mask(in_file, iterations=4): - import nibabel as nib + import nibabel as nb import scipy.ndimage as ndimage import os - img = nib.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) img._data = ndimage.binary_dilation(img.get_data(), iterations=iterations) name, fext = os.path.splitext(os.path.basename(in_file)) if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_dil.nii.gz' % name) - nib.save(img, out_file) + nb.save(img, out_file) return out_file def _fill_phase(in_file): - import nibabel as nib + import nibabel as nb import os import numpy as np - img = nib.load(in_file) - dumb_img = nib.Nifti1Image(np.zeros(img.shape), img.affine, img.header) - out_nii = nib.funcs.concat_images((img, dumb_img)) + img = nb.load(in_file, mmap=NUMPY_MMAP) + dumb_img = nb.Nifti1Image(np.zeros(img.shape), img.affine, img.header) + out_nii = nb.funcs.concat_images((img, dumb_img)) name, fext = os.path.splitext(os.path.basename(in_file)) if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_fill.nii.gz' % name) - nib.save(out_nii, out_file) + nb.save(out_nii, out_file) return out_file def _vsm_remove_mean(in_file, mask_file, in_unwarped): - import nibabel as nib + import nibabel as nb import os import numpy as np import numpy.ma as ma - img = nib.load(in_file) - msk = nib.load(mask_file).get_data() + img = nb.load(in_file, mmap=NUMPY_MMAP) + msk = nb.load(mask_file, mmap=NUMPY_MMAP).get_data() img_data = img.get_data() img_data[msk == 0] = 0 vsmmag_masked = ma.masked_values(img_data.reshape(-1), 0.0) @@ -782,7 +783,7 @@ def _vsm_remove_mean(in_file, mask_file, in_unwarped): if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_demeaned.nii.gz' % name) - nib.save(img, out_file) + nb.save(img, out_file) return out_file @@ -791,15 +792,15 @@ def _ms2sec(val): def _split_dwi(in_file): - import nibabel as nib + import nibabel as nb import os out_files = [] - frames = nib.funcs.four_to_three(nib.load(in_file)) + frames = nb.funcs.four_to_three(nb.load(in_file, mmap=NUMPY_MMAP)) name, fext = os.path.splitext(os.path.basename(in_file)) if fext == '.gz': name, _ = os.path.splitext(name) for i, frame in enumerate(frames): out_file = os.path.abspath('./%s_%03d.nii.gz' % (name, i)) - nib.save(frame, out_file) + nb.save(frame, out_file) out_files.append(out_file) return out_files diff --git a/nipype/workflows/dmri/fsl/tbss.py b/nipype/workflows/dmri/fsl/tbss.py index feede0d223..4567b65f90 100644 --- a/nipype/workflows/dmri/fsl/tbss.py +++ b/nipype/workflows/dmri/fsl/tbss.py @@ -5,16 +5,17 @@ import os from warnings import warn +from nipype.utils import NUMPY_MMAP from ....pipeline import engine as pe from ....interfaces import utility as util from ....interfaces import fsl as fsl def tbss1_op_string(in_files): - import nibabel as nib + import nibabel as nb op_strings = [] for infile in in_files: - img = nib.load(infile) + img = nb.load(infile, mmap=NUMPY_MMAP) dimtup = tuple(d - 2 for d in img.shape) dimtup = dimtup[0:3] op_str = '-min 1 -ero -roi 1 %d 1 %d 1 %d 0 1' % dimtup diff --git a/nipype/workflows/dmri/fsl/utils.py b/nipype/workflows/dmri/fsl/utils.py index ba1658d468..8078241ab3 100644 --- a/nipype/workflows/dmri/fsl/utils.py +++ b/nipype/workflows/dmri/fsl/utils.py @@ -6,6 +6,8 @@ from __future__ import print_function, division, unicode_literals, absolute_import from builtins import zip, next, range, str +from nipype.utils import NUMPY_MMAP + from ....pipeline import engine as pe from ....interfaces import utility as niu from ....interfaces import fsl @@ -215,7 +217,7 @@ def extract_bval(in_dwi, in_bval, b=0, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_tsoi%s" % (fname, ext)) - im = nb.load(in_dwi) + im = nb.load(in_dwi, mmap=NUMPY_MMAP) dwidata = im.get_data() bvals = np.loadtxt(in_bval) @@ -243,7 +245,7 @@ def hmc_split(in_file, in_bval, ref_num=0, lowbval=5.0): import os.path as op from nipype.interfaces.base import isdefined - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data() hdr = im.header.copy() bval = np.loadtxt(in_bval) @@ -294,7 +296,7 @@ def remove_comp(in_file, in_bval, volid=0, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_extract%s" % (fname, ext)) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data() hdr = im.header.copy() bval = np.loadtxt(in_bval) @@ -343,7 +345,7 @@ def recompose_dwi(in_dwi, in_bval, in_corrected, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_eccorrect%s" % (fname, ext)) - im = nb.load(in_dwi) + im = nb.load(in_dwi, mmap=NUMPY_MMAP) dwidata = im.get_data() bvals = np.loadtxt(in_bval) dwis = np.where(bvals != 0)[0].tolist() @@ -353,7 +355,7 @@ def recompose_dwi(in_dwi, in_bval, in_corrected, out_file=None): 'correction should match')) for bindex, dwi in zip(dwis, in_corrected): - dwidata[..., bindex] = nb.load(dwi).get_data() + dwidata[..., bindex] = nb.load(dwi, mmap=NUMPY_MMAP).get_data() nb.Nifti1Image(dwidata, im.affine, im.header).to_filename(out_file) return out_file @@ -405,7 +407,7 @@ def time_avg(in_file, index=[0], out_file=None): index = np.atleast_1d(index).tolist() - imgs = np.array(nb.four_to_three(nb.load(in_file)))[index] + imgs = np.array(nb.four_to_three(nb.load(in_file, mmap=NUMPY_MMAP)))[index] if len(index) == 1: data = imgs[0].get_data().astype(np.float32) else: @@ -450,7 +452,7 @@ def b0_average(in_dwi, in_bval, max_b=10.0, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_avg_b0%s" % (fname, ext)) - imgs = np.array(nb.four_to_three(nb.load(in_dwi))) + imgs = np.array(nb.four_to_three(nb.load(in_dwi, mmap=NUMPY_MMAP))) bval = np.loadtxt(in_bval) index = np.argwhere(bval <= max_b).flatten().tolist() @@ -628,7 +630,7 @@ def rads2radsec(in_file, delta_te, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_radsec.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data().astype(np.float32) * (1.0 / delta_te) nb.Nifti1Image(data, im.affine, im.header).to_filename(out_file) return out_file @@ -649,12 +651,12 @@ def demean_image(in_file, in_mask=None, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_demean.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data().astype(np.float32) msk = np.ones_like(data) if in_mask is not None: - msk = nb.load(in_mask).get_data().astype(np.float32) + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data().astype(np.float32) msk[msk > 0] = 1.0 msk[msk < 1] = 0.0 @@ -679,7 +681,7 @@ def add_empty_vol(in_file, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_4D.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) zim = nb.Nifti1Image(np.zeros_like(im.get_data()), im.affine, im.header) nb.funcs.concat_images([im, zim]).to_filename(out_file) @@ -702,8 +704,8 @@ def reorient_bvecs(in_dwi, old_dwi, in_bvec): bvecs = np.loadtxt(in_bvec).T new_bvecs = [] - N = nb.load(in_dwi).affine - O = nb.load(old_dwi).affine + N = nb.load(in_dwi, mmap=NUMPY_MMAP).affine + O = nb.load(old_dwi, mmap=NUMPY_MMAP).affine RS = N.dot(np.linalg.inv(O))[:3, :3] sc_idx = np.where((np.abs(RS) != 1) & (RS != 0)) S = np.ones_like(RS) @@ -726,10 +728,10 @@ def copy_hdr(in_file, in_file_hdr, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_fixhdr.nii.gz' % fname) - imref = nb.load(in_file_hdr) + imref = nb.load(in_file_hdr, mmap=NUMPY_MMAP) hdr = imref.header.copy() hdr.set_data_dtype(np.float32) - vsm = nb.load(in_file).get_data().astype(np.float32) + vsm = nb.load(in_file, mmap=NUMPY_MMAP).get_data().astype(np.float32) hdr.set_data_shape(vsm.shape) hdr.set_xyzt_units('mm') nii = nb.Nifti1Image(vsm, imref.affine, hdr) @@ -749,12 +751,12 @@ def enhance(in_file, clip_limit=0.010, in_mask=None, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_enh.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) imdata = im.get_data() imshape = im.shape if in_mask is not None: - msk = nb.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk[msk > 0] = 1 msk[msk < 1] = 0 imdata = imdata * msk diff --git a/nipype/workflows/fmri/fsl/preprocess.py b/nipype/workflows/fmri/fsl/preprocess.py index d3f2d25ebe..9660f19c53 100644 --- a/nipype/workflows/fmri/fsl/preprocess.py +++ b/nipype/workflows/fmri/fsl/preprocess.py @@ -28,19 +28,21 @@ def pickfirst(files): def pickmiddle(files): from nibabel import load import numpy as np + from nipype.utils import NUMPY_MMAP middlevol = [] for f in files: - middlevol.append(int(np.ceil(load(f).shape[3] / 2))) + middlevol.append(int(np.ceil(load(f, mmap=NUMPY_MMAP).shape[3] / 2))) return middlevol def pickvol(filenames, fileidx, which): from nibabel import load import numpy as np + from nipype.utils import NUMPY_MMAP if which.lower() == 'first': idx = 0 elif which.lower() == 'middle': - idx = int(np.ceil(load(filenames[fileidx]).shape[3] / 2)) + idx = int(np.ceil(load(filenames[fileidx], mmap=NUMPY_MMAP).shape[3] / 2)) elif which.lower() == 'last': idx = load(filenames[fileidx]).shape[3] - 1 else: diff --git a/nipype/workflows/misc/utils.py b/nipype/workflows/misc/utils.py index 1d47be1e31..8d3524e2e0 100644 --- a/nipype/workflows/misc/utils.py +++ b/nipype/workflows/misc/utils.py @@ -4,13 +4,14 @@ from __future__ import print_function, division, unicode_literals, absolute_import from builtins import map, range +from nipype.utils import NUMPY_MMAP def get_vox_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] @@ -20,7 +21,7 @@ def get_data_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header datadims = hdr.get_data_shape() return [int(datadims[0]), int(datadims[1]), int(datadims[2])] @@ -28,7 +29,7 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine @@ -49,7 +50,7 @@ def select_aparc_annot(list_of_files): def region_list_from_volume(in_file): import nibabel as nb import numpy as np - segmentation = nb.load(in_file) + segmentation = nb.load(in_file, mmap=NUMPY_MMAP) segmentationdata = segmentation.get_data() rois = np.unique(segmentationdata) region_list = list(rois) diff --git a/nipype/workflows/rsfmri/fsl/resting.py b/nipype/workflows/rsfmri/fsl/resting.py index a51693fda5..c15ca27061 100644 --- a/nipype/workflows/rsfmri/fsl/resting.py +++ b/nipype/workflows/rsfmri/fsl/resting.py @@ -14,10 +14,12 @@ def select_volume(filename, which): """ from nibabel import load import numpy as np + from nipype.utils import NUMPY_MMAP + if which.lower() == 'first': idx = 0 elif which.lower() == 'middle': - idx = int(np.ceil(load(filename).shape[3] / 2)) + idx = int(np.ceil(load(filename, mmap=NUMPY_MMAP).shape[3] / 2)) else: raise Exception('unknown value for volume selection : %s' % which) return idx diff --git a/nipype/workflows/smri/freesurfer/autorecon1.py b/nipype/workflows/smri/freesurfer/autorecon1.py index bd93b5b722..1ec49740b2 100644 --- a/nipype/workflows/smri/freesurfer/autorecon1.py +++ b/nipype/workflows/smri/freesurfer/autorecon1.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- + +from nipype.utils import NUMPY_MMAP from nipype.interfaces.utility import Function,IdentityInterface import nipype.pipeline.engine as pe # pypeline engine from nipype.interfaces.freesurfer import * @@ -8,7 +10,7 @@ def checkT1s(T1_files, cw256=False): """Verifying size of inputs and setting workflow parameters""" import sys - import nibabel as nib + import nibabel as nb from nipype.utils.filemanip import filename_to_list T1_files = filename_to_list(T1_files) @@ -16,9 +18,9 @@ def checkT1s(T1_files, cw256=False): print("ERROR: No T1's Given") sys.exit(-1) - shape = nib.load(T1_files[0]).shape + shape = nb.load(T1_files[0]).shape for t1 in T1_files[1:]: - if nib.load(t1).shape != shape: + if nb.load(t1, mmap=NUMPY_MMAP).shape != shape: print("ERROR: T1s not the same size. Cannot process {0} and {1} " "together".format(T1_files[0], t1)) sys.exit(-1)