From 7cbe9dfbf304b87d82f5b302b529226901c59373 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 2 Feb 2017 11:42:16 -0800 Subject: [PATCH 01/28] Do not open nifti files with mmap if numpy < 1.12.0 Fixes #1795 --- nipype/algorithms/confounds.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 087b273190..3a19398e49 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -16,6 +16,7 @@ import os import os.path as op +from distutils.version import LooseVersion import nibabel as nb import numpy as np @@ -29,6 +30,8 @@ InputMultiPath) IFLOG = logging.getLogger('interface') +NUMPY_MMAP = LooseVersion(np.__version__) >= LooseVersion('1.12.0') + class ComputeDVARSInputSpec(BaseInterfaceInputSpec): in_file = File(exists=True, mandatory=True, desc='functional data, after HMC') in_mask = File(exists=True, mandatory=True, desc='a brain mask') @@ -127,9 +130,9 @@ def _run_interface(self, runtime): dvars = compute_dvars(self.inputs.in_file, self.inputs.in_mask, remove_zerovariance=self.inputs.remove_zerovariance) - self._results['avg_std'] = float(dvars[0].mean()) - self._results['avg_nstd'] = float(dvars[1].mean()) - self._results['avg_vxstd'] = float(dvars[2].mean()) + (self._results['avg_std'], + self._results['avg_nstd'], + self._results['avg_vxstd']) = np.mean(dvars, axis=1).astype(float) tr = None if isdefined(self.inputs.series_tr): @@ -329,8 +332,8 @@ class CompCor(BaseInterface): }] def _run_interface(self, runtime): - imgseries = nb.load(self.inputs.realigned_file).get_data() - mask = nb.load(self.inputs.mask_file).get_data() + imgseries = nb.load(self.inputs.realigned_file, mmap=NUMPY_MMAP).get_data() + mask = nb.load(self.inputs.mask_file, mmap=NUMPY_MMAP).get_data() if imgseries.shape[:3] != mask.shape: raise ValueError('Inputs for CompCor, func {} and mask {}, do not have matching ' @@ -435,7 +438,7 @@ class TCompCor(CompCor): output_spec = TCompCorOutputSpec def _run_interface(self, runtime): - imgseries = nb.load(self.inputs.realigned_file).get_data() + imgseries = nb.load(self.inputs.realigned_file, mmap=NUMPY_MMAP).get_data() if imgseries.ndim != 4: raise ValueError('tCompCor expected a 4-D nifti file. Input {} has {} dimensions ' @@ -443,7 +446,7 @@ def _run_interface(self, runtime): .format(self.inputs.realigned_file, imgseries.ndim, imgseries.shape)) if isdefined(self.inputs.mask_file): - in_mask_data = nb.load(self.inputs.mask_file).get_data() + in_mask_data = nb.load(self.inputs.mask_file, mmap=NUMPY_MMAP).get_data() imgseries = imgseries[in_mask_data != 0, :] # From the paper: @@ -521,9 +524,9 @@ class TSNR(BaseInterface): output_spec = TSNROutputSpec def _run_interface(self, runtime): - img = nb.load(self.inputs.in_file[0]) + img = nb.load(self.inputs.in_file[0], mmap=NUMPY_MMAP) header = img.header.copy() - vollist = [nb.load(filename) for filename in self.inputs.in_file] + vollist = [nb.load(filename, mmap=NUMPY_MMAP) for filename in self.inputs.in_file] data = np.concatenate([vol.get_data().reshape( vol.get_shape()[:3] + (-1,)) for vol in vollist], axis=3) data = np.nan_to_num(data) @@ -626,8 +629,9 @@ def compute_dvars(in_file, in_mask, remove_zerovariance=False): from nitime.algorithms import AR_est_YW import warnings - func = nb.load(in_file).get_data().astype(np.float32) - mask = nb.load(in_mask).get_data().astype(np.uint8) + mmap + func = nb.load(in_file, mmap=NUMPY_MMAP).get_data().astype(np.float32) + mask = nb.load(in_mask, mmap=NUMPY_MMAP).get_data().astype(np.uint8) if len(func.shape) != 4: raise RuntimeError( From 0db15b5ac8ae3542df58c56acdc78e65fe93a224 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 2 Feb 2017 22:03:45 -0800 Subject: [PATCH 02/28] fix leftover typo --- nipype/algorithms/confounds.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 3a19398e49..3163e98e74 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -629,7 +629,6 @@ def compute_dvars(in_file, in_mask, remove_zerovariance=False): from nitime.algorithms import AR_est_YW import warnings - mmap func = nb.load(in_file, mmap=NUMPY_MMAP).get_data().astype(np.float32) mask = nb.load(in_mask, mmap=NUMPY_MMAP).get_data().astype(np.uint8) From b67868f3b4d6636cafe2e9db86d98d9547d2f574 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 13:42:27 -0800 Subject: [PATCH 03/28] use anaconda in travis cc #1788 --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ca29323e5f..70e76ca203 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,10 @@ env: - INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: - function bef_inst { - wget http://repo.continuum.io/miniconda/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh - -O /home/travis/.cache/miniconda.sh && - bash /home/travis/.cache/miniconda.sh -b -p /home/travis/miniconda && - export PATH=/home/travis/miniconda/bin:$PATH && + wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh + -O /home/travis/.cache/anaconda.sh && + bash /home/travis/.cache/anaconda.sh -b -p /home/travis/anaconda && + export PATH=/home/travis/anaconda/bin:$PATH && if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && if $INSTALL_DEB_DEPENDECIES; then sudo ln -s /run/shm /dev/shm; fi && bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) && @@ -34,7 +34,7 @@ install: conda update --yes conda && conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y nipype && - rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && + rm -r /home/travis/anaconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && pip install -r requirements.txt && pip install -e .[$NIPYPE_EXTRAS]; } - travis_retry inst From b5899cda95baa5c4e987ee1cb0aa30a9e0df7433 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 14:07:31 -0800 Subject: [PATCH 04/28] simplify conda installation --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70e76ca203..fcb7f7b0b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,9 +33,7 @@ install: conda config --add channels conda-forge && conda update --yes conda && conda update --all -y python=$TRAVIS_PYTHON_VERSION && - conda install -y nipype && - rm -r /home/travis/anaconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && - pip install -r requirements.txt && + conda install -y $(sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' requirements.txt) && pip install -e .[$NIPYPE_EXTRAS]; } - travis_retry inst script: From 332d46d096daeb679f6fb5fa73d1d58e803e0948 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 14:26:36 -0800 Subject: [PATCH 05/28] remove adding conda-forge channel --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fcb7f7b0b7..5f2e03cbef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,6 @@ before_install: install: # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && - function inst { - conda config --add channels conda-forge && conda update --yes conda && conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y $(sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' requirements.txt) && From 85ca6e5035d19c0064947ae4e70a5852f88a8380 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 14:37:08 -0800 Subject: [PATCH 06/28] readd conda update --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5f2e03cbef..fcb7f7b0b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,7 @@ before_install: install: # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && - function inst { + conda config --add channels conda-forge && conda update --yes conda && conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y $(sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' requirements.txt) && From c411e23afb2a1336cc9ddb9cd673fc49933d71e6 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 14:37:56 -0800 Subject: [PATCH 07/28] remove requirement version in nibabel --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c06bfbfee5..b610fcd635 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ scipy>=0.11 networkx>=1.7 traits>=4.3 python-dateutil>=1.5 -nibabel>=2.0.1 +nibabel future>=0.15.2 simplejson>=3.8.0 prov>=1.4.0 From 0d5d6dd2ab405aab36e12ff6f311448ba84c1465 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 15:46:07 -0800 Subject: [PATCH 08/28] revise conda installation --- .travis.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcb7f7b0b7..1f40de860a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,6 @@ env: - INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: - function bef_inst { - wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh - -O /home/travis/.cache/anaconda.sh && - bash /home/travis/.cache/anaconda.sh -b -p /home/travis/anaconda && - export PATH=/home/travis/anaconda/bin:$PATH && if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && if $INSTALL_DEB_DEPENDECIES; then sudo ln -s /run/shm /dev/shm; fi && bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) && @@ -25,16 +21,20 @@ before_install: if $INSTALL_DEB_DEPENDECIES; then source /etc/fsl/fsl.sh; source /etc/afni/afni.sh; - export FSLOUTPUTTYPE=NIFTI_GZ; fi } + export FSLOUTPUTTYPE=NIFTI_GZ; fi && + wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh + -O /home/travis/.cache/anaconda.sh && + bash /home/travis/.cache/anaconda.sh -b -p /home/travis/anaconda && + export PATH=/home/travis/anaconda/bin:$PATH && + conda config --add channels conda-forge && + echo 'always_yes: True' >> /home/travis/.condarc && + conda update --all -y python=$TRAVIS_PYTHON_VERSION && + conda install -y numpy scipy + } - travis_retry bef_inst install: # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && -- function inst { - conda config --add channels conda-forge && - conda update --yes conda && - conda update --all -y python=$TRAVIS_PYTHON_VERSION && - conda install -y $(sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' requirements.txt) && - pip install -e .[$NIPYPE_EXTRAS]; } +- function inst {pip install -e .[$NIPYPE_EXTRAS];} - travis_retry inst script: - py.test --doctest-modules nipype From 70a7cb2d65ba59c37ebf697e2f2289962b0d67ed Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 16:03:15 -0800 Subject: [PATCH 09/28] fix travis file --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f40de860a..68c0016a39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,15 +27,13 @@ before_install: bash /home/travis/.cache/anaconda.sh -b -p /home/travis/anaconda && export PATH=/home/travis/anaconda/bin:$PATH && conda config --add channels conda-forge && - echo 'always_yes: True' >> /home/travis/.condarc && conda update --all -y python=$TRAVIS_PYTHON_VERSION && - conda install -y numpy scipy + conda install -y numpy scipy; } - travis_retry bef_inst install: # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && -- function inst {pip install -e .[$NIPYPE_EXTRAS];} -- travis_retry inst +- travis_retry pip install -e .[$NIPYPE_EXTRAS] script: - py.test --doctest-modules nipype deploy: From 452a284b22a03e02b359ff8a6b7bb0be8d5c3718 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 16:34:51 -0800 Subject: [PATCH 10/28] add icu to conda (#1798), append conda-forge instead of prepend --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68c0016a39..83940aa449 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,9 @@ before_install: -O /home/travis/.cache/anaconda.sh && bash /home/travis/.cache/anaconda.sh -b -p /home/travis/anaconda && export PATH=/home/travis/anaconda/bin:$PATH && - conda config --add channels conda-forge && + conda config --append channels conda-forge && conda update --all -y python=$TRAVIS_PYTHON_VERSION && - conda install -y numpy scipy; - } + conda install -y icu numpy scipy traits;} - travis_retry bef_inst install: # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && From 4291ffcc844aa16c0da1ee6b0f2e3601c61a26e8 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 3 Feb 2017 16:49:23 -0800 Subject: [PATCH 11/28] install boto3 in python 3 --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 83940aa449..069eb8a3c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,10 @@ before_install: export PATH=/home/travis/anaconda/bin:$PATH && conda config --append channels conda-forge && conda update --all -y python=$TRAVIS_PYTHON_VERSION && - conda install -y icu numpy scipy traits;} + conda install -y icu numpy scipy traits && + if [ "${TRAVIS_PYTHON_VERSION:0:1}" -gt "2" ]; then + conda install -y boto3; + fi } - travis_retry bef_inst install: # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && From a34c0fd03f911f4d94a9f6c7dffaf0bb5317b891 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 9 Feb 2017 16:48:26 -0800 Subject: [PATCH 12/28] fix travis.xml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f801ddffa9..318930ce48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,8 @@ before_install: conda config --append channels conda-forge && conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y nipype icu==56.1 && -# Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*} +# Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && - travis_retry bef_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] From 11fd6193320c2bab97372b50f4fd20fcf262cc3c Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 18:24:11 -0800 Subject: [PATCH 13/28] fix travis (second round) --- .travis.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 318930ce48..cb550b79d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ env: - INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" - INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: -- function bef_inst { +- function apt_inst { if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && if $INSTALL_DEB_DEPENDECIES; then sudo ln -s /run/shm /dev/shm; fi && bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) && @@ -21,17 +21,20 @@ before_install: if $INSTALL_DEB_DEPENDECIES; then source /etc/fsl/fsl.sh; source /etc/afni/afni.sh; - export FSLOUTPUTTYPE=NIFTI_GZ; fi && + export FSLOUTPUTTYPE=NIFTI_GZ; fi } +- function conda_inst { + export CONDA_HOME=/home/travis/anaconda && wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh -O /home/travis/.cache/anaconda.sh && - bash /home/travis/.cache/anaconda.sh -b -p /home/travis/anaconda && - export PATH=/home/travis/anaconda/bin:$PATH && + bash /home/travis/.cache/anaconda.sh -b -p ${CONDA_HOME} && + export PATH=${CONDA_HOME}/bin:$PATH && conda config --append channels conda-forge && conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y nipype icu==56.1 && - rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*} -# Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi && -- travis_retry bef_inst + rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } +# Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi +- travis_retry apt_inst +- travis_retry conda_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] script: From 2ec16f8cd7997616841389d90c3ae934adf96e0f Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 18:38:49 -0800 Subject: [PATCH 14/28] fix conda-forge channel, split anaconda download --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb550b79d5..7f10987968 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,16 +24,17 @@ before_install: export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { export CONDA_HOME=/home/travis/anaconda && - wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh - -O /home/travis/.cache/anaconda.sh && bash /home/travis/.cache/anaconda.sh -b -p ${CONDA_HOME} && export PATH=${CONDA_HOME}/bin:$PATH && - conda config --append channels conda-forge && + conda config --add channels conda-forge && conda update --all -y python=$TRAVIS_PYTHON_VERSION && conda install -y nipype icu==56.1 && rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi - travis_retry apt_inst +- travis_retry if [ ! -f /home/travis/.cache/anaconda.sh]; then + wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh -O /home/travis/.cache/anaconda.sh; + fi - travis_retry conda_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] From 5b4677222f81d4c10ed511fda7be27db81752c09 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 18:42:15 -0800 Subject: [PATCH 15/28] remove breaklines --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f10987968..749ad63a40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,9 +32,7 @@ before_install: rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi - travis_retry apt_inst -- travis_retry if [ ! -f /home/travis/.cache/anaconda.sh]; then - wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh -O /home/travis/.cache/anaconda.sh; - fi +- travis_retry if [ ! -f /home/travis/.cache/anaconda.sh]; then wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh -O /home/travis/.cache/anaconda.sh; fi - travis_retry conda_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] From 13089211f92b31619c686bd20a068fcd4cbb1bb1 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 18:55:01 -0800 Subject: [PATCH 16/28] do not try to install conda after a travis_retry --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 749ad63a40..878d06b7a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,10 @@ before_install: source /etc/afni/afni.sh; export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { - export CONDA_HOME=/home/travis/anaconda && - bash /home/travis/.cache/anaconda.sh -b -p ${CONDA_HOME} && + if [ ! -d "${CONDA_HOME}" ]; then + wget -N https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh + -O /home/travis/.cache/anaconda.sh && + bash /home/travis/.cache/anaconda.sh -b -p ${CONDA_HOME}; fi && export PATH=${CONDA_HOME}/bin:$PATH && conda config --add channels conda-forge && conda update --all -y python=$TRAVIS_PYTHON_VERSION && @@ -32,7 +34,7 @@ before_install: rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi - travis_retry apt_inst -- travis_retry if [ ! -f /home/travis/.cache/anaconda.sh]; then wget https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh -O /home/travis/.cache/anaconda.sh; fi +- export CONDA_HOME=/home/travis/anaconda - travis_retry conda_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] From 2abe8f5df1eb9709f5c1ed1e4830f63bb816bfbb Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 19:15:34 -0800 Subject: [PATCH 17/28] do not use travis python --- .travis.yml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 878d06b7a3..125cc237f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,19 @@ cache: -- apt -language: python -python: -- 2.7 -- 3.4 -- 3.5 + apt: true + directories: + - $HOME/anaconda + +language: c env: -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" +- PYTHON_VER=2.7 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- PYTHON_VER=2.7 INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- PYTHON_VER=2.7 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" +- PYTHON_VER=3.4 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- PYTHON_VER=3.4 INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- PYTHON_VER=3.4 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" +- PYTHON_VER=3.5 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- PYTHON_VER=3.5 INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- PYTHON_VER=3.5 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: - function apt_inst { if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && @@ -24,17 +29,17 @@ before_install: export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { if [ ! -d "${CONDA_HOME}" ]; then - wget -N https://repo.continuum.io/archive/Anaconda${TRAVIS_PYTHON_VERSION:0:1}-4.3.0-Linux-x86_64.sh + wget -N https://repo.continuum.io/archive/Anaconda${PYTHON_VER:0:1}-4.3.0-Linux-x86_64.sh -O /home/travis/.cache/anaconda.sh && bash /home/travis/.cache/anaconda.sh -b -p ${CONDA_HOME}; fi && export PATH=${CONDA_HOME}/bin:$PATH && conda config --add channels conda-forge && - conda update --all -y python=$TRAVIS_PYTHON_VERSION && + conda update --all -y python=${PYTHON_VER} && conda install -y nipype icu==56.1 && - rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } + rm -r ${CONDA_HOME}/lib/python${PYTHON_VER}/site-packages/nipype*; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi - travis_retry apt_inst -- export CONDA_HOME=/home/travis/anaconda +- export CONDA_HOME=$HOME/anaconda - travis_retry conda_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] From 25e65b91d04fb1ff7690c46409b8e3a4ff1ecb04 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 21:51:12 -0800 Subject: [PATCH 18/28] roll back to miniconda, use some hints from https://conda.io/docs/travis.html --- .travis.yml | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 125cc237f5..06c71bc051 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,17 @@ cache: apt: true directories: - - $HOME/anaconda + - $HOME/conda -language: c +language: python +python: +- 2.7 +- 3.4 +- 3.5 env: -- PYTHON_VER=2.7 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- PYTHON_VER=2.7 INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- PYTHON_VER=2.7 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" -- PYTHON_VER=3.4 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- PYTHON_VER=3.4 INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- PYTHON_VER=3.4 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" -- PYTHON_VER=3.5 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- PYTHON_VER=3.5 INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- PYTHON_VER=3.5 INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: - function apt_inst { if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && @@ -28,18 +26,21 @@ before_install: source /etc/afni/afni.sh; export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { + export CONDA_HOME=$HOME/conda if [ ! -d "${CONDA_HOME}" ]; then - wget -N https://repo.continuum.io/archive/Anaconda${PYTHON_VER:0:1}-4.3.0-Linux-x86_64.sh - -O /home/travis/.cache/anaconda.sh && - bash /home/travis/.cache/anaconda.sh -b -p ${CONDA_HOME}; fi && + wget https://repo.continuum.io/archive/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh + -O /home/travis/.cache/conda.sh && + bash /home/travis/.cache/conda.sh -b -p ${CONDA_HOME}; fi && export PATH=${CONDA_HOME}/bin:$PATH && + hash -r && + conda config --set always_yes yes --set changeps1 no && + conda update -q conda && conda config --add channels conda-forge && - conda update --all -y python=${PYTHON_VER} && + conda update --all -y python=${TRAVIS_PYTHON_VERSION} && conda install -y nipype icu==56.1 && - rm -r ${CONDA_HOME}/lib/python${PYTHON_VER}/site-packages/nipype*; } + rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi - travis_retry apt_inst -- export CONDA_HOME=$HOME/anaconda - travis_retry conda_inst install: - travis_retry pip install -e .[$NIPYPE_EXTRAS] From 5d60cef36f90c2a22ba0f55a2d4cb9044d51f683 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 21:53:49 -0800 Subject: [PATCH 19/28] fix error in travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 06c71bc051..7740f70d2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: source /etc/afni/afni.sh; export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { - export CONDA_HOME=$HOME/conda + export CONDA_HOME=$HOME/conda && if [ ! -d "${CONDA_HOME}" ]; then wget https://repo.continuum.io/archive/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh -O /home/travis/.cache/conda.sh && From 1c956b0ae771319a627f92a734f49c14a360c887 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 22:02:39 -0800 Subject: [PATCH 20/28] remove if switch --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7740f70d2f..5121812656 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ cache: apt: true - directories: - - $HOME/conda language: python python: @@ -27,10 +25,9 @@ before_install: export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { export CONDA_HOME=$HOME/conda && - if [ ! -d "${CONDA_HOME}" ]; then - wget https://repo.continuum.io/archive/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh + wget https://repo.continuum.io/archive/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh -O /home/travis/.cache/conda.sh && - bash /home/travis/.cache/conda.sh -b -p ${CONDA_HOME}; fi && + bash /home/travis/.cache/conda.sh -b -p ${CONDA_HOME} && export PATH=${CONDA_HOME}/bin:$PATH && hash -r && conda config --set always_yes yes --set changeps1 no && From 13b3cc689c53c8e9db643a6d2de8d6f1728cb6f6 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 9 Feb 2017 22:12:01 -0800 Subject: [PATCH 21/28] fix miniconda link --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5121812656..436cf66d96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ before_install: export FSLOUTPUTTYPE=NIFTI_GZ; fi } - function conda_inst { export CONDA_HOME=$HOME/conda && - wget https://repo.continuum.io/archive/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh + wget https://repo.continuum.io/miniconda/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh -O /home/travis/.cache/conda.sh && - bash /home/travis/.cache/conda.sh -b -p ${CONDA_HOME} && + bash /home/travis/.cache/conda.sh -b -p ${CONDA_HOME} && export PATH=${CONDA_HOME}/bin:$PATH && hash -r && conda config --set always_yes yes --set changeps1 no && From 3d1e644d79388f3e622b40ed0d91b807532178d1 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 10 Feb 2017 18:40:13 -0800 Subject: [PATCH 22/28] fix command to update conda to a certain version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3b6d5691a5..64b7fea960 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,8 @@ before_install: hash -r && conda config --set always_yes yes --set changeps1 no && conda update -q conda && + conda install python=${TRAVIS_PYTHON_VERSION} && conda config --add channels conda-forge && - conda update --all -y python=${TRAVIS_PYTHON_VERSION} && conda install -y nipype icu && rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi From f3ebcc5a108934129e8ec5374a4c31b2d1b6e0f1 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sat, 11 Feb 2017 11:12:47 -0800 Subject: [PATCH 23/28] reenable version pinning for nibabel in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b610fcd635..c06bfbfee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ scipy>=0.11 networkx>=1.7 traits>=4.3 python-dateutil>=1.5 -nibabel +nibabel>=2.0.1 future>=0.15.2 simplejson>=3.8.0 prov>=1.4.0 From 3820f9fb184c04cb103ee3db9eeace158c18cc8f Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 14:39:15 -0800 Subject: [PATCH 24/28] replace all nibabel imports and the nibabel.load to have the NUMPY_MMAP --- examples/dmri_camino_dti.py | 6 ++-- examples/dmri_connectivity.py | 6 ++-- examples/fmri_ants_openfmri.py | 2 +- examples/fmri_spm_auditory.py | 2 +- examples/fmri_spm_face.py | 2 +- examples/rsfmri_vol_surface_preprocessing.py | 16 ++++----- .../rsfmri_vol_surface_preprocessing_nipy.py | 12 +++---- nipype/algorithms/icc.py | 2 +- nipype/algorithms/metrics.py | 4 +-- nipype/algorithms/misc.py | 28 +++++++-------- nipype/algorithms/tests/test_errormap.py | 22 ++++++------ .../algorithms/tests/test_normalize_tpms.py | 6 ++-- nipype/algorithms/tests/test_splitmerge.py | 6 ++-- nipype/interfaces/cmtk/cmtk.py | 6 ++-- nipype/interfaces/dcmstack.py | 2 +- nipype/interfaces/dipy/preprocess.py | 4 +-- nipype/interfaces/dipy/simulate.py | 6 ++-- .../interfaces/freesurfer/tests/test_model.py | 14 ++++---- nipype/interfaces/fsl/epi.py | 10 +++--- nipype/interfaces/nipy/model.py | 2 +- nipype/interfaces/nipy/preprocess.py | 2 +- nipype/workflows/dmri/dipy/denoise.py | 8 ++--- nipype/workflows/dmri/fsl/artifacts.py | 4 +-- nipype/workflows/dmri/fsl/epi.py | 36 +++++++++---------- nipype/workflows/dmri/fsl/tbss.py | 4 +-- nipype/workflows/dmri/fsl/utils.py | 34 +++++++++--------- nipype/workflows/misc/utils.py | 8 ++--- .../workflows/smri/freesurfer/autorecon1.py | 6 ++-- 28 files changed, 130 insertions(+), 130 deletions(-) diff --git a/examples/dmri_camino_dti.py b/examples/dmri_camino_dti.py index 20392d9bf5..8395d77082 100755 --- a/examples/dmri_camino_dti.py +++ b/examples/dmri_camino_dti.py @@ -38,7 +38,7 @@ def get_vox_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] @@ -48,7 +48,7 @@ def get_data_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header datadims = hdr.get_data_shape() return [int(datadims[0]), int(datadims[1]), int(datadims[2])] @@ -56,7 +56,7 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine subject_list = ['subj1'] diff --git a/examples/dmri_connectivity.py b/examples/dmri_connectivity.py index 4b7debff8f..c3d904c237 100755 --- a/examples/dmri_connectivity.py +++ b/examples/dmri_connectivity.py @@ -76,7 +76,7 @@ def get_vox_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] @@ -86,7 +86,7 @@ def get_data_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header datadims = hdr.get_data_shape() return [int(datadims[0]), int(datadims[1]), int(datadims[2])] @@ -94,7 +94,7 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine diff --git a/examples/fmri_ants_openfmri.py b/examples/fmri_ants_openfmri.py index 05cbb4efa4..d421d6bfb7 100755 --- a/examples/fmri_ants_openfmri.py +++ b/examples/fmri_ants_openfmri.py @@ -68,7 +68,7 @@ def median(in_files): """ average = None for idx, filename in enumerate(filename_to_list(in_files)): - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) data = np.median(img.get_data(), axis=3) if average is None: average = data diff --git a/examples/fmri_spm_auditory.py b/examples/fmri_spm_auditory.py index 94b6f1a565..8c229ffd4d 100755 --- a/examples/fmri_spm_auditory.py +++ b/examples/fmri_spm_auditory.py @@ -122,7 +122,7 @@ def get_vox_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] diff --git a/examples/fmri_spm_face.py b/examples/fmri_spm_face.py index 942bace0ff..8e81bea664 100755 --- a/examples/fmri_spm_face.py +++ b/examples/fmri_spm_face.py @@ -116,7 +116,7 @@ def get_vox_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] diff --git a/examples/rsfmri_vol_surface_preprocessing.py b/examples/rsfmri_vol_surface_preprocessing.py index 4b35ba37a4..0fc4e56d13 100644 --- a/examples/rsfmri_vol_surface_preprocessing.py +++ b/examples/rsfmri_vol_surface_preprocessing.py @@ -119,7 +119,7 @@ def median(in_files): import nibabel as nb average = None for idx, filename in enumerate(filename_to_list(in_files)): - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) data = np.median(img.get_data(), axis=3) if average is None: average = data @@ -149,7 +149,7 @@ def bandpass_filter(files, lowpass_freq, highpass_freq, fs): for filename in filename_to_list(files): path, name, ext = split_filename(filename) out_file = os.path.join(os.getcwd(), name + '_bp' + ext) - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) timepoints = img.shape[-1] F = np.zeros((timepoints)) lowidx = int(timepoints / 2) + 1 @@ -261,10 +261,10 @@ def extract_noise_components(realigned_file, mask_file, num_components=5, import numpy as np import nibabel as nb import os - imgseries = nb.load(realigned_file) + imgseries = nb.load(realigned_file, mmap=NUMPY_MMAP) components = None for filename in filename_to_list(mask_file): - mask = nb.load(filename).get_data() + mask = nb.load(filename, mmap=NUMPY_MMAP).get_data() if len(np.nonzero(mask > 0)[0]) == 0: continue voxel_timecourses = imgseries.get_data()[mask > 0] @@ -330,9 +330,9 @@ def extract_subrois(timeseries_file, label_file, indices): from nipype.utils.filemanip import split_filename import nibabel as nb import os - img = nb.load(timeseries_file) + img = nb.load(timeseries_file, mmap=NUMPY_MMAP) data = img.get_data() - roiimg = nb.load(label_file) + roiimg = nb.load(label_file, mmap=NUMPY_MMAP) rois = roiimg.get_data() prefix = split_filename(timeseries_file)[1] out_ts_file = os.path.join(os.getcwd(), '%s_subcortical_ts.txt' % prefix) @@ -352,8 +352,8 @@ def combine_hemi(left, right): """ import os import numpy as np - lh_data = nb.load(left).get_data() - rh_data = nb.load(right).get_data() + lh_data = nb.load(left, mmap=NUMPY_MMAP).get_data() + rh_data = nb.load(right, mmap=NUMPY_MMAP).get_data() indices = np.vstack((1000000 + np.arange(0, lh_data.shape[0])[:, None], 2000000 + np.arange(0, rh_data.shape[0])[:, None])) diff --git a/examples/rsfmri_vol_surface_preprocessing_nipy.py b/examples/rsfmri_vol_surface_preprocessing_nipy.py index b745704023..b4837f2725 100644 --- a/examples/rsfmri_vol_surface_preprocessing_nipy.py +++ b/examples/rsfmri_vol_surface_preprocessing_nipy.py @@ -116,7 +116,7 @@ def median(in_files): """ average = None for idx, filename in enumerate(filename_to_list(in_files)): - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) data = np.median(img.get_data(), axis=3) if average is None: average = data @@ -143,7 +143,7 @@ def bandpass_filter(files, lowpass_freq, highpass_freq, fs): for filename in filename_to_list(files): path, name, ext = split_filename(filename) out_file = os.path.join(os.getcwd(), name + '_bp' + ext) - img = nb.load(filename) + img = nb.load(filename, mmap=NUMPY_MMAP) timepoints = img.shape[-1] F = np.zeros((timepoints)) lowidx = int(timepoints / 2) + 1 @@ -268,9 +268,9 @@ def extract_subrois(timeseries_file, label_file, indices): The first four columns are: freesurfer index, i, j, k positions in the label file """ - img = nb.load(timeseries_file) + img = nb.load(timeseries_file, mmap=NUMPY_MMAP) data = img.get_data() - roiimg = nb.load(label_file) + roiimg = nb.load(label_file, mmap=NUMPY_MMAP) rois = roiimg.get_data() prefix = split_filename(timeseries_file)[1] out_ts_file = os.path.join(os.getcwd(), '%s_subcortical_ts.txt' % prefix) @@ -288,8 +288,8 @@ def extract_subrois(timeseries_file, label_file, indices): def combine_hemi(left, right): """Combine left and right hemisphere time series into a single text file """ - lh_data = nb.load(left).get_data() - rh_data = nb.load(right).get_data() + lh_data = nb.load(left, mmap=NUMPY_MMAP).get_data() + rh_data = nb.load(right, mmap=NUMPY_MMAP).get_data() indices = np.vstack((1000000 + np.arange(0, lh_data.shape[0])[:, None], 2000000 + np.arange(0, rh_data.shape[0])[:, None])) diff --git a/nipype/algorithms/icc.py b/nipype/algorithms/icc.py index 3a9e8237e7..b497f9b7a6 100644 --- a/nipype/algorithms/icc.py +++ b/nipype/algorithms/icc.py @@ -37,7 +37,7 @@ def _run_interface(self, runtime): maskdata = nb.load(self.inputs.mask).get_data() maskdata = np.logical_not(np.logical_or(maskdata == 0, np.isnan(maskdata))) - session_datas = [[nb.load(fname).get_data()[maskdata].reshape(-1, 1) for fname in sessions] for sessions in self.inputs.subjects_sessions] + session_datas = [[nb.load(fname, mmap=NUMPY_MMAP).get_data()[maskdata].reshape(-1, 1) for fname in sessions] for sessions in self.inputs.subjects_sessions] list_of_sessions = [np.dstack(session_data) for session_data in session_datas] all_data = np.hstack(list_of_sessions) icc = np.zeros(session_datas[0][0].shape) diff --git a/nipype/algorithms/metrics.py b/nipype/algorithms/metrics.py index f2ea7594c6..7bc3a16e6c 100644 --- a/nipype/algorithms/metrics.py +++ b/nipype/algorithms/metrics.py @@ -410,8 +410,8 @@ def _run_interface(self, runtime): assert(ncomp == len(self.inputs.in_tst)) weights = np.ones(shape=ncomp) - img_ref = np.array([nb.load(fname).get_data() for fname in self.inputs.in_ref]) - img_tst = np.array([nb.load(fname).get_data() for fname in self.inputs.in_tst]) + img_ref = np.array([nb.load(fname, mmap=NUMPY_MMAP).get_data() for fname in self.inputs.in_ref]) + img_tst = np.array([nb.load(fname, mmap=NUMPY_MMAP).get_data() for fname in self.inputs.in_tst]) msk = np.sum(img_ref, axis=0) msk[msk > 0] = 1.0 diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 58a2276af6..fda84eef2e 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -140,7 +140,7 @@ class SimpleThreshold(BaseInterface): def _run_interface(self, runtime): for fname in self.inputs.volumes: - img = nb.load(fname) + img = nb.load(fname, mmap=NUMPY_MMAP) data = np.array(img.get_data()) active_map = data > self.inputs.threshold @@ -196,7 +196,7 @@ def _gen_output_filename(self, name): def _run_interface(self, runtime): for fname in self.inputs.volumes: - img = nb.load(fname) + img = nb.load(fname, mmap=NUMPY_MMAP) affine = img.affine affine = np.dot(self.inputs.transformation_matrix, affine) @@ -1158,7 +1158,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): Returns the input tissue probability maps (tpms, aka volume fractions) normalized to sum up 1.0 at each voxel within the mask. """ - import nibabel as nib + import nibabel as nb import numpy as np import os.path as op @@ -1174,7 +1174,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): out_file = op.abspath('%s_norm_%02d%s' % (fname, i, fext)) out_files += [out_file] - imgs = [nib.load(fim) for fim in in_files] + imgs = [nb.load(fim, mmap=NUMPY_MMAP) for fim in in_files] if len(in_files) == 1: img_data = imgs[0].get_data() @@ -1182,7 +1182,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): hdr = imgs[0].header.copy() hdr['data_type'] = 16 hdr.set_data_dtype(np.float32) - nib.save(nib.Nifti1Image(img_data.astype(np.float32), imgs[0].affine, + nb.save(nb.Nifti1Image(img_data.astype(np.float32), imgs[0].affine, hdr), out_files[0]) return out_files[0] @@ -1195,7 +1195,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): msk[weights <= 0] = 0 if in_mask is not None: - msk = nib.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk[msk <= 0] = 0 msk[msk > 0] = 1 @@ -1207,7 +1207,7 @@ def normalize_tpms(in_files, in_mask=None, out_files=[]): hdr = imgs[i].header.copy() hdr['data_type'] = 16 hdr.set_data_dtype('float32') - nib.save(nib.Nifti1Image(probmap.astype(np.float32), imgs[i].affine, + nb.save(nb.Nifti1Image(probmap.astype(np.float32), imgs[i].affine, hdr), out_file) return out_files @@ -1225,7 +1225,7 @@ def split_rois(in_file, mask=None, roishape=None): if roishape is None: roishape = (10, 10, 1) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) imshape = im.shape dshape = imshape[:3] nvols = imshape[-1] @@ -1233,7 +1233,7 @@ def split_rois(in_file, mask=None, roishape=None): droishape = (roishape[0], roishape[1], roishape[2], nvols) if mask is not None: - mask = nb.load(mask).get_data() + mask = nb.load(mask, mmap=NUMPY_MMAP).get_data() mask[mask > 0] = 1 mask[mask < 1] = 0 else: @@ -1314,7 +1314,7 @@ def merge_rois(in_files, in_idxs, in_ref, except: pass - ref = nb.load(in_ref) + ref = nb.load(in_ref, mmap=NUMPY_MMAP) aff = ref.affine hdr = ref.header.copy() rsh = ref.shape @@ -1335,7 +1335,7 @@ def merge_rois(in_files, in_idxs, in_ref, for cname, iname in zip(in_files, in_idxs): f = np.load(iname) idxs = np.squeeze(f['arr_0']) - cdata = nb.load(cname).get_data().reshape(-1, ndirs) + cdata = nb.load(cname, mmap=NUMPY_MMAP).get_data().reshape(-1, ndirs) nels = len(idxs) idata = (idxs, ) try: @@ -1363,15 +1363,15 @@ def merge_rois(in_files, in_idxs, in_ref, idxs = np.squeeze(f['arr_0']) for d, fname in enumerate(nii): - data = nb.load(fname).get_data().reshape(-1) - cdata = nb.load(cname).get_data().reshape(-1, ndirs)[:, d] + data = nb.load(fname, mmap=NUMPY_MMAP).get_data().reshape(-1) + cdata = nb.load(cname, mmap=NUMPY_MMAP).get_data().reshape(-1, ndirs)[:, d] nels = len(idxs) idata = (idxs, ) data[idata] = cdata[0:nels] nb.Nifti1Image(data.reshape(rsh[:3]), aff, hdr).to_filename(fname) - imgs = [nb.load(im) for im in nii] + imgs = [nb.load(im, mmap=NUMPY_MMAP) for im in nii] allim = nb.concat_images(imgs) allim.to_filename(out_file) diff --git a/nipype/algorithms/tests/test_errormap.py b/nipype/algorithms/tests/test_errormap.py index a86c932ca4..a700725e41 100644 --- a/nipype/algorithms/tests/test_errormap.py +++ b/nipype/algorithms/tests/test_errormap.py @@ -4,7 +4,7 @@ import pytest from nipype.testing import example_data from nipype.algorithms.metrics import ErrorMap -import nibabel as nib +import nibabel as nb import numpy as np import os @@ -18,13 +18,13 @@ def test_errormap(tmpdir): volume2 = np.array([[[0.0, 7.0], [2.0, 3.0]], [[1.0, 9.0], [1.0, 2.0]]]) # Alan Turing's birthday mask = np.array([[[1, 0], [0, 1]], [[1, 0], [0, 1]]]) - img1 = nib.Nifti1Image(volume1, np.eye(4)) - img2 = nib.Nifti1Image(volume2, np.eye(4)) - maskimg = nib.Nifti1Image(mask, np.eye(4)) + img1 = nb.Nifti1Image(volume1, np.eye(4)) + img2 = nb.Nifti1Image(volume2, np.eye(4)) + maskimg = nb.Nifti1Image(mask, np.eye(4)) - nib.save(img1, os.path.join(tempdir, 'von.nii.gz')) - nib.save(img2, os.path.join(tempdir, 'alan.nii.gz')) - nib.save(maskimg, os.path.join(tempdir, 'mask.nii.gz')) + nb.save(img1, os.path.join(tempdir, 'von.nii.gz')) + nb.save(img2, os.path.join(tempdir, 'alan.nii.gz')) + nb.save(maskimg, os.path.join(tempdir, 'mask.nii.gz')) # Default metric errmap = ErrorMap() @@ -55,15 +55,15 @@ def test_errormap(tmpdir): msvolume1 = np.zeros(shape=(2, 2, 2, 2)) msvolume1[:, :, :, 0] = volume1 msvolume1[:, :, :, 1] = volume3 - msimg1 = nib.Nifti1Image(msvolume1, np.eye(4)) + msimg1 = nb.Nifti1Image(msvolume1, np.eye(4)) msvolume2 = np.zeros(shape=(2, 2, 2, 2)) msvolume2[:, :, :, 0] = volume3 msvolume2[:, :, :, 1] = volume1 - msimg2 = nib.Nifti1Image(msvolume2, np.eye(4)) + msimg2 = nb.Nifti1Image(msvolume2, np.eye(4)) - nib.save(msimg1, os.path.join(tempdir, 'von-ray.nii.gz')) - nib.save(msimg2, os.path.join(tempdir, 'alan-ray.nii.gz')) + nb.save(msimg1, os.path.join(tempdir, 'von-ray.nii.gz')) + nb.save(msimg2, os.path.join(tempdir, 'alan-ray.nii.gz')) errmap.inputs.in_tst = os.path.join(tempdir, 'von-ray.nii.gz') errmap.inputs.in_ref = os.path.join(tempdir, 'alan-ray.nii.gz') diff --git a/nipype/algorithms/tests/test_normalize_tpms.py b/nipype/algorithms/tests/test_normalize_tpms.py index 907c1d9619..2550ddbdab 100644 --- a/nipype/algorithms/tests/test_normalize_tpms.py +++ b/nipype/algorithms/tests/test_normalize_tpms.py @@ -20,7 +20,7 @@ def test_normalize_tpms(tmpdir): tempdir = str(tmpdir) in_mask = example_data('tpms_msk.nii.gz') - mskdata = nb.load(in_mask).get_data() + mskdata = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() mskdata[mskdata > 0.0] = 1.0 mapdata = [] @@ -32,7 +32,7 @@ def test_normalize_tpms(tmpdir): filename = os.path.join(tempdir, 'modtpm_%02d.nii.gz' % i) out_files.append(os.path.join(tempdir, 'normtpm_%02d.nii.gz' % i)) - im = nb.load(mapname) + im = nb.load(mapname, mmap=NUMPY_MMAP) data = im.get_data() mapdata.append(data.copy()) @@ -45,7 +45,7 @@ def test_normalize_tpms(tmpdir): sumdata = np.zeros_like(mskdata) for i, tstfname in enumerate(out_files): - normdata = nb.load(tstfname).get_data() + normdata = nb.load(tstfname, mmap=NUMPY_MMAP).get_data() sumdata += normdata assert np.all(normdata[mskdata == 0] == 0) assert np.allclose(normdata, mapdata[i]) diff --git a/nipype/algorithms/tests/test_splitmerge.py b/nipype/algorithms/tests/test_splitmerge.py index 46cd05f995..1ac376e5ac 100644 --- a/nipype/algorithms/tests/test_splitmerge.py +++ b/nipype/algorithms/tests/test_splitmerge.py @@ -14,8 +14,8 @@ def test_split_and_merge(tmpdir): in_mask = example_data('tpms_msk.nii.gz') dwfile = op.join(str(tmpdir), 'dwi.nii.gz') - mskdata = nb.load(in_mask).get_data() - aff = nb.load(in_mask).affine + mskdata = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() + aff = nb.load(in_mask, mmap=NUMPY_MMAP).affine dwshape = (mskdata.shape[0], mskdata.shape[1], mskdata.shape[2], 6) dwdata = np.random.normal(size=dwshape) @@ -25,7 +25,7 @@ def test_split_and_merge(tmpdir): resdw, resmsk, resid = split_rois(dwfile, in_mask, roishape=(20, 20, 2)) merged = merge_rois(resdw, resid, in_mask) - dwmerged = nb.load(merged).get_data() + dwmerged = nb.load(merged, mmap=NUMPY_MMAP).get_data() dwmasked = dwdata * mskdata[:, :, :, np.newaxis] diff --git a/nipype/interfaces/cmtk/cmtk.py b/nipype/interfaces/cmtk/cmtk.py index 8903321ba5..d60587078c 100644 --- a/nipype/interfaces/cmtk/cmtk.py +++ b/nipype/interfaces/cmtk/cmtk.py @@ -183,7 +183,7 @@ def cmat(track_file, roi_file, resolution_network_file, matrix_name, matrix_mat_ fib, hdr = nb.trackvis.read(track_file, False) stats['orig_n_fib'] = len(fib) - roi = nb.load(roi_file) + roi = nb.load(roi_file, mmap=NUMPY_MMAP) roiData = roi.get_data() roiVoxelSize = roi.header.get_zooms() (endpoints, endpointsmm) = create_endpoints_array(fib, roiVoxelSize) @@ -619,7 +619,7 @@ def _run_interface(self, runtime): aparc_aseg_file = self.inputs.aparc_aseg_file aparcpath, aparcname, aparcext = split_filename(aparc_aseg_file) iflogger.info('Using Aparc+Aseg file: {name}'.format(name=aparcname + aparcext)) - niiAPARCimg = nb.load(aparc_aseg_file) + niiAPARCimg = nb.load(aparc_aseg_file, mmap=NUMPY_MMAP) niiAPARCdata = niiAPARCimg.get_data() niiDataLabels = np.unique(niiAPARCdata) numDataLabels = np.size(niiDataLabels) @@ -742,7 +742,7 @@ def _gen_outfilename(self, ext): def create_nodes(roi_file, resolution_network_file, out_filename): G = nx.Graph() gp = nx.read_graphml(resolution_network_file) - roi_image = nb.load(roi_file) + roi_image = nb.load(roi_file, mmap=NUMPY_MMAP) roiData = roi_image.get_data() nROIs = len(gp.nodes()) for u, d in gp.nodes_iter(data=True): diff --git a/nipype/interfaces/dcmstack.py b/nipype/interfaces/dcmstack.py index ca1e8d7d1b..584f8d3143 100644 --- a/nipype/interfaces/dcmstack.py +++ b/nipype/interfaces/dcmstack.py @@ -357,7 +357,7 @@ class MergeNifti(NiftiGeneratorBase): output_spec = MergeNiftiOutputSpec def _run_interface(self, runtime): - niis = [nb.load(fn) + niis = [nb.load(fn, mmap=NUMPY_MMAP) for fn in self.inputs.in_files ] nws = [NiftiWrapper(nii, make_empty=True) diff --git a/nipype/interfaces/dipy/preprocess.py b/nipype/interfaces/dipy/preprocess.py index 163118e0cb..1680e87ffc 100644 --- a/nipype/interfaces/dipy/preprocess.py +++ b/nipype/interfaces/dipy/preprocess.py @@ -186,7 +186,7 @@ def resample_proxy(in_file, order=3, new_zooms=None, out_file=None): fext = fext2 + fext out_file = op.abspath('./%s_reslice%s' % (fname, fext)) - img = nb.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) hdr = img.header.copy() data = img.get_data().astype(np.float32) affine = img.affine @@ -229,7 +229,7 @@ def nlmeans_proxy(in_file, settings, fext = fext2 + fext out_file = op.abspath('./%s_denoise%s' % (fname, fext)) - img = nb.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) hdr = img.header data = img.get_data() aff = img.affine diff --git a/nipype/interfaces/dipy/simulate.py b/nipype/interfaces/dipy/simulate.py index 0baa50244b..ac58a83f59 100644 --- a/nipype/interfaces/dipy/simulate.py +++ b/nipype/interfaces/dipy/simulate.py @@ -118,7 +118,7 @@ def _run_interface(self, runtime): # Volume fractions of isotropic compartments nballs = len(self.inputs.in_vfms) vfs = np.squeeze(nb.concat_images( - [nb.load(f) for f in self.inputs.in_vfms]).get_data()) + [nb.load(f, mmap=NUMPY_MMAP) for f in self.inputs.in_vfms]).get_data()) if nballs == 1: vfs = vfs[..., np.newaxis] total_vf = np.sum(vfs, axis=3) @@ -136,7 +136,7 @@ def _run_interface(self, runtime): nvox = len(msk[msk > 0]) # Fiber fractions - ffsim = nb.concat_images([nb.load(f) for f in self.inputs.in_frac]) + ffsim = nb.concat_images([nb.load(f, mmap=NUMPY_MMAP) for f in self.inputs.in_frac]) ffs = np.nan_to_num(np.squeeze(ffsim.get_data())) # fiber fractions ffs = np.clip(ffs, 0., 1.) if nsticks == 1: @@ -179,7 +179,7 @@ def _run_interface(self, runtime): dirs = None for i in range(nsticks): f = self.inputs.in_dirs[i] - fd = np.nan_to_num(nb.load(f).get_data()) + fd = np.nan_to_num(nb.load(f, mmap=NUMPY_MMAP).get_data()) w = np.linalg.norm(fd, axis=3)[..., np.newaxis] w[w < np.finfo(float).eps] = 1.0 fd /= w diff --git a/nipype/interfaces/freesurfer/tests/test_model.py b/nipype/interfaces/freesurfer/tests/test_model.py index d9da543154..bf8c167666 100644 --- a/nipype/interfaces/freesurfer/tests/test_model.py +++ b/nipype/interfaces/freesurfer/tests/test_model.py @@ -4,7 +4,7 @@ import os import numpy as np -import nibabel as nib +import nibabel as nb import pytest from nipype.interfaces.freesurfer import model, no_freesurfer @@ -24,18 +24,18 @@ def test_concatenate(tmpdir): out_data = np.concatenate((data1, data2), axis=3) mean_data = np.mean(out_data, axis=3) - nib.Nifti1Image(data1, affine=np.eye(4)).to_filename(in1) - nib.Nifti1Image(data2, affine=np.eye(4)).to_filename(in2) + nb.Nifti1Image(data1, affine=np.eye(4)).to_filename(in1) + nb.Nifti1Image(data2, affine=np.eye(4)).to_filename(in2) # Test default behavior res = model.Concatenate(in_files=[in1, in2]).run() assert res.outputs.concatenated_file == os.path.join(tempdir, 'concat_output.nii.gz') - assert np.allclose(nib.load('concat_output.nii.gz').get_data(), out_data) + assert np.allclose(nb.load('concat_output.nii.gz').get_data(), out_data) # Test specified concatenated_file res = model.Concatenate(in_files=[in1, in2], concatenated_file=out).run() assert res.outputs.concatenated_file == os.path.join(tempdir, out) - assert np.allclose(nib.load(out).get_data(), out_data) + assert np.allclose(nb.load(out, mmap=NUMPY_MMAP).get_data(), out_data) # Test in workflow wf = pe.Workflow('test_concatenate', base_dir=tempdir) @@ -44,7 +44,7 @@ def test_concatenate(tmpdir): name='concat') wf.add_nodes([concat]) wf.run() - assert np.allclose(nib.load(os.path.join(tempdir, + assert np.allclose(nb.load(os.path.join(tempdir, 'test_concatenate', 'concat', out)).get_data(), out_data) @@ -52,4 +52,4 @@ def test_concatenate(tmpdir): # Test a simple statistic res = model.Concatenate(in_files=[in1, in2], concatenated_file=out, stats='mean').run() - assert np.allclose(nib.load(out).get_data(), mean_data) + assert np.allclose(nb.load(out, mmap=NUMPY_MMAP).get_data(), mean_data) diff --git a/nipype/interfaces/fsl/epi.py b/nipype/interfaces/fsl/epi.py index c854c0400c..6c6a96e522 100644 --- a/nipype/interfaces/fsl/epi.py +++ b/nipype/interfaces/fsl/epi.py @@ -17,7 +17,7 @@ import os import numpy as np -import nibabel as nib +import nibabel as nb import warnings from ...utils.filemanip import split_filename @@ -102,11 +102,11 @@ def _run_interface(self, runtime): if runtime.returncode == 0: out_file = self.inputs.out_fieldmap - im = nib.load(out_file) - dumb_img = nib.Nifti1Image(np.zeros(im.shape), im.affine, + im = nb.load(out_file, mmap=NUMPY_MMAP) + dumb_img = nb.Nifti1Image(np.zeros(im.shape), im.affine, im.header) - out_nii = nib.funcs.concat_images((im, dumb_img)) - nib.save(out_nii, out_file) + out_nii = nb.funcs.concat_images((im, dumb_img)) + nb.save(out_nii, out_file) return runtime diff --git a/nipype/interfaces/nipy/model.py b/nipype/interfaces/nipy/model.py index 584152d01e..28e8cffc12 100644 --- a/nipype/interfaces/nipy/model.py +++ b/nipype/interfaces/nipy/model.py @@ -99,7 +99,7 @@ def _run_interface(self, runtime): del data for functional_run in functional_runs[1:]: - nii = nb.load(functional_run) + nii = nb.load(functional_run, mmap=NUMPY_MMAP) data = nii.get_data() npdata = data.copy() del data diff --git a/nipype/interfaces/nipy/preprocess.py b/nipype/interfaces/nipy/preprocess.py index 579f9f7988..b0fd107080 100644 --- a/nipype/interfaces/nipy/preprocess.py +++ b/nipype/interfaces/nipy/preprocess.py @@ -61,7 +61,7 @@ def _run_interface(self, runtime): value = getattr(self.inputs, key) if isdefined(value): if key in ['mean_volume', 'reference_volume']: - nii = nb.load(value) + nii = nb.load(value, mmap=NUMPY_MMAP) value = nii.get_data() args[key] = value diff --git a/nipype/workflows/dmri/dipy/denoise.py b/nipype/workflows/dmri/dipy/denoise.py index c060bb97c1..aefb2329d4 100644 --- a/nipype/workflows/dmri/dipy/denoise.py +++ b/nipype/workflows/dmri/dipy/denoise.py @@ -64,12 +64,12 @@ def csf_mask(in_file, in_mask, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_csfmask%s" % (fname, ext)) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) hdr = im.header.copy() hdr.set_data_dtype(np.uint8) hdr.set_xyzt_units('mm') imdata = im.get_data() - msk = nb.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk = binary_erosion(msk, structure=np.ones((15, 15, 10))).astype(np.uint8) thres = np.percentile(imdata[msk > 0].reshape(-1), 90.0) @@ -107,12 +107,12 @@ def bg_mask(in_file, in_mask, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_bgmask%s" % (fname, ext)) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) hdr = im.header.copy() hdr.set_data_dtype(np.uint8) hdr.set_xyzt_units('mm') imdata = im.get_data() - msk = nb.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk = 1 - binary_dilation(msk, structure=np.ones((20, 20, 20))) nb.Nifti1Image(msk.astype(np.uint8), im.affine, hdr).to_filename(out_file) return out_file diff --git a/nipype/workflows/dmri/fsl/artifacts.py b/nipype/workflows/dmri/fsl/artifacts.py index 906541829f..89462924ce 100644 --- a/nipype/workflows/dmri/fsl/artifacts.py +++ b/nipype/workflows/dmri/fsl/artifacts.py @@ -235,7 +235,7 @@ def _gen_index(in_file): import nibabel as nb import os out_file = os.path.abspath('index.txt') - vols = nb.load(in_file).get_data().shape[-1] + vols = nb.load(in_file, mmap=NUMPY_MMAP).get_data().shape[-1] np.savetxt(out_file, np.ones((vols,)).T) return out_file @@ -900,7 +900,7 @@ def _xfm_jacobian(in_xfm): def _get_zoom(in_file, enc_dir): import nibabel as nb - zooms = nb.load(in_file).header.get_zooms() + zooms = nb.load(in_file, mmap=NUMPY_MMAP).header.get_zooms() if 'y' in enc_dir: return zooms[1] diff --git a/nipype/workflows/dmri/fsl/epi.py b/nipype/workflows/dmri/fsl/epi.py index 16bcab3ab6..b174941a17 100644 --- a/nipype/workflows/dmri/fsl/epi.py +++ b/nipype/workflows/dmri/fsl/epi.py @@ -718,10 +718,10 @@ def _effective_echospacing(dwell_time, pi_factor=1.0): def _prepare_phasediff(in_file): - import nibabel as nib + import nibabel as nb import os import numpy as np - img = nib.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) max_diff = np.max(img.get_data().reshape(-1)) min_diff = np.min(img.get_data().reshape(-1)) A = (2.0 * np.pi) / (max_diff - min_diff) @@ -732,47 +732,47 @@ def _prepare_phasediff(in_file): if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_2pi.nii.gz' % name) - nib.save(nib.Nifti1Image(diff_norm, img.affine, img.header), out_file) + nb.save(nb.Nifti1Image(diff_norm, img.affine, img.header), out_file) return out_file def _dilate_mask(in_file, iterations=4): - import nibabel as nib + import nibabel as nb import scipy.ndimage as ndimage import os - img = nib.load(in_file) + img = nb.load(in_file, mmap=NUMPY_MMAP) img._data = ndimage.binary_dilation(img.get_data(), iterations=iterations) name, fext = os.path.splitext(os.path.basename(in_file)) if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_dil.nii.gz' % name) - nib.save(img, out_file) + nb.save(img, out_file) return out_file def _fill_phase(in_file): - import nibabel as nib + import nibabel as nb import os import numpy as np - img = nib.load(in_file) - dumb_img = nib.Nifti1Image(np.zeros(img.shape), img.affine, img.header) - out_nii = nib.funcs.concat_images((img, dumb_img)) + img = nb.load(in_file, mmap=NUMPY_MMAP) + dumb_img = nb.Nifti1Image(np.zeros(img.shape), img.affine, img.header) + out_nii = nb.funcs.concat_images((img, dumb_img)) name, fext = os.path.splitext(os.path.basename(in_file)) if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_fill.nii.gz' % name) - nib.save(out_nii, out_file) + nb.save(out_nii, out_file) return out_file def _vsm_remove_mean(in_file, mask_file, in_unwarped): - import nibabel as nib + import nibabel as nb import os import numpy as np import numpy.ma as ma - img = nib.load(in_file) - msk = nib.load(mask_file).get_data() + img = nb.load(in_file, mmap=NUMPY_MMAP) + msk = nb.load(mask_file, mmap=NUMPY_MMAP).get_data() img_data = img.get_data() img_data[msk == 0] = 0 vsmmag_masked = ma.masked_values(img_data.reshape(-1), 0.0) @@ -782,7 +782,7 @@ def _vsm_remove_mean(in_file, mask_file, in_unwarped): if fext == '.gz': name, _ = os.path.splitext(name) out_file = os.path.abspath('./%s_demeaned.nii.gz' % name) - nib.save(img, out_file) + nb.save(img, out_file) return out_file @@ -791,15 +791,15 @@ def _ms2sec(val): def _split_dwi(in_file): - import nibabel as nib + import nibabel as nb import os out_files = [] - frames = nib.funcs.four_to_three(nib.load(in_file)) + frames = nb.funcs.four_to_three(nb.load(in_file, mmap=NUMPY_MMAP)) name, fext = os.path.splitext(os.path.basename(in_file)) if fext == '.gz': name, _ = os.path.splitext(name) for i, frame in enumerate(frames): out_file = os.path.abspath('./%s_%03d.nii.gz' % (name, i)) - nib.save(frame, out_file) + nb.save(frame, out_file) out_files.append(out_file) return out_files diff --git a/nipype/workflows/dmri/fsl/tbss.py b/nipype/workflows/dmri/fsl/tbss.py index feede0d223..9ca9a12f98 100644 --- a/nipype/workflows/dmri/fsl/tbss.py +++ b/nipype/workflows/dmri/fsl/tbss.py @@ -11,10 +11,10 @@ def tbss1_op_string(in_files): - import nibabel as nib + import nibabel as nb op_strings = [] for infile in in_files: - img = nib.load(infile) + img = nb.load(infile, mmap=NUMPY_MMAP) dimtup = tuple(d - 2 for d in img.shape) dimtup = dimtup[0:3] op_str = '-min 1 -ero -roi 1 %d 1 %d 1 %d 0 1' % dimtup diff --git a/nipype/workflows/dmri/fsl/utils.py b/nipype/workflows/dmri/fsl/utils.py index ba1658d468..cd45749b60 100644 --- a/nipype/workflows/dmri/fsl/utils.py +++ b/nipype/workflows/dmri/fsl/utils.py @@ -215,7 +215,7 @@ def extract_bval(in_dwi, in_bval, b=0, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_tsoi%s" % (fname, ext)) - im = nb.load(in_dwi) + im = nb.load(in_dwi, mmap=NUMPY_MMAP) dwidata = im.get_data() bvals = np.loadtxt(in_bval) @@ -243,7 +243,7 @@ def hmc_split(in_file, in_bval, ref_num=0, lowbval=5.0): import os.path as op from nipype.interfaces.base import isdefined - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data() hdr = im.header.copy() bval = np.loadtxt(in_bval) @@ -294,7 +294,7 @@ def remove_comp(in_file, in_bval, volid=0, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_extract%s" % (fname, ext)) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data() hdr = im.header.copy() bval = np.loadtxt(in_bval) @@ -343,7 +343,7 @@ def recompose_dwi(in_dwi, in_bval, in_corrected, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_eccorrect%s" % (fname, ext)) - im = nb.load(in_dwi) + im = nb.load(in_dwi, mmap=NUMPY_MMAP) dwidata = im.get_data() bvals = np.loadtxt(in_bval) dwis = np.where(bvals != 0)[0].tolist() @@ -353,7 +353,7 @@ def recompose_dwi(in_dwi, in_bval, in_corrected, out_file=None): 'correction should match')) for bindex, dwi in zip(dwis, in_corrected): - dwidata[..., bindex] = nb.load(dwi).get_data() + dwidata[..., bindex] = nb.load(dwi, mmap=NUMPY_MMAP).get_data() nb.Nifti1Image(dwidata, im.affine, im.header).to_filename(out_file) return out_file @@ -405,7 +405,7 @@ def time_avg(in_file, index=[0], out_file=None): index = np.atleast_1d(index).tolist() - imgs = np.array(nb.four_to_three(nb.load(in_file)))[index] + imgs = np.array(nb.four_to_three(nb.load(in_file, mmap=NUMPY_MMAP)))[index] if len(index) == 1: data = imgs[0].get_data().astype(np.float32) else: @@ -450,7 +450,7 @@ def b0_average(in_dwi, in_bval, max_b=10.0, out_file=None): ext = ext2 + ext out_file = op.abspath("%s_avg_b0%s" % (fname, ext)) - imgs = np.array(nb.four_to_three(nb.load(in_dwi))) + imgs = np.array(nb.four_to_three(nb.load(in_dwi, mmap=NUMPY_MMAP))) bval = np.loadtxt(in_bval) index = np.argwhere(bval <= max_b).flatten().tolist() @@ -628,7 +628,7 @@ def rads2radsec(in_file, delta_te, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_radsec.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data().astype(np.float32) * (1.0 / delta_te) nb.Nifti1Image(data, im.affine, im.header).to_filename(out_file) return out_file @@ -649,12 +649,12 @@ def demean_image(in_file, in_mask=None, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_demean.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) data = im.get_data().astype(np.float32) msk = np.ones_like(data) if in_mask is not None: - msk = nb.load(in_mask).get_data().astype(np.float32) + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data().astype(np.float32) msk[msk > 0] = 1.0 msk[msk < 1] = 0.0 @@ -679,7 +679,7 @@ def add_empty_vol(in_file, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_4D.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) zim = nb.Nifti1Image(np.zeros_like(im.get_data()), im.affine, im.header) nb.funcs.concat_images([im, zim]).to_filename(out_file) @@ -702,8 +702,8 @@ def reorient_bvecs(in_dwi, old_dwi, in_bvec): bvecs = np.loadtxt(in_bvec).T new_bvecs = [] - N = nb.load(in_dwi).affine - O = nb.load(old_dwi).affine + N = nb.load(in_dwi, mmap=NUMPY_MMAP).affine + O = nb.load(old_dwi, mmap=NUMPY_MMAP).affine RS = N.dot(np.linalg.inv(O))[:3, :3] sc_idx = np.where((np.abs(RS) != 1) & (RS != 0)) S = np.ones_like(RS) @@ -726,10 +726,10 @@ def copy_hdr(in_file, in_file_hdr, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_fixhdr.nii.gz' % fname) - imref = nb.load(in_file_hdr) + imref = nb.load(in_file_hdr, mmap=NUMPY_MMAP) hdr = imref.header.copy() hdr.set_data_dtype(np.float32) - vsm = nb.load(in_file).get_data().astype(np.float32) + vsm = nb.load(in_file, mmap=NUMPY_MMAP).get_data().astype(np.float32) hdr.set_data_shape(vsm.shape) hdr.set_xyzt_units('mm') nii = nb.Nifti1Image(vsm, imref.affine, hdr) @@ -749,12 +749,12 @@ def enhance(in_file, clip_limit=0.010, in_mask=None, out_file=None): fname, _ = op.splitext(fname) out_file = op.abspath('./%s_enh.nii.gz' % fname) - im = nb.load(in_file) + im = nb.load(in_file, mmap=NUMPY_MMAP) imdata = im.get_data() imshape = im.shape if in_mask is not None: - msk = nb.load(in_mask).get_data() + msk = nb.load(in_mask, mmap=NUMPY_MMAP).get_data() msk[msk > 0] = 1 msk[msk < 1] = 0 imdata = imdata * msk diff --git a/nipype/workflows/misc/utils.py b/nipype/workflows/misc/utils.py index 1d47be1e31..06df275a1c 100644 --- a/nipype/workflows/misc/utils.py +++ b/nipype/workflows/misc/utils.py @@ -10,7 +10,7 @@ def get_vox_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header voxdims = hdr.get_zooms() return [float(voxdims[0]), float(voxdims[1]), float(voxdims[2])] @@ -20,7 +20,7 @@ def get_data_dims(volume): import nibabel as nb if isinstance(volume, list): volume = volume[0] - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) hdr = nii.header datadims = hdr.get_data_shape() return [int(datadims[0]), int(datadims[1]), int(datadims[2])] @@ -28,7 +28,7 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb - nii = nb.load(volume) + nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine @@ -49,7 +49,7 @@ def select_aparc_annot(list_of_files): def region_list_from_volume(in_file): import nibabel as nb import numpy as np - segmentation = nb.load(in_file) + segmentation = nb.load(in_file, mmap=NUMPY_MMAP) segmentationdata = segmentation.get_data() rois = np.unique(segmentationdata) region_list = list(rois) diff --git a/nipype/workflows/smri/freesurfer/autorecon1.py b/nipype/workflows/smri/freesurfer/autorecon1.py index bd93b5b722..88b9dc84c1 100644 --- a/nipype/workflows/smri/freesurfer/autorecon1.py +++ b/nipype/workflows/smri/freesurfer/autorecon1.py @@ -8,7 +8,7 @@ def checkT1s(T1_files, cw256=False): """Verifying size of inputs and setting workflow parameters""" import sys - import nibabel as nib + import nibabel as nb from nipype.utils.filemanip import filename_to_list T1_files = filename_to_list(T1_files) @@ -16,9 +16,9 @@ def checkT1s(T1_files, cw256=False): print("ERROR: No T1's Given") sys.exit(-1) - shape = nib.load(T1_files[0]).shape + shape = nb.load(T1_files[0]).shape for t1 in T1_files[1:]: - if nib.load(t1).shape != shape: + if nb.load(t1, mmap=NUMPY_MMAP).shape != shape: print("ERROR: T1s not the same size. Cannot process {0} and {1} " "together".format(T1_files[0], t1)) sys.exit(-1) From 671fc0358d55a3c1939a22be76328da79ad74db1 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 14:59:04 -0800 Subject: [PATCH 25/28] add NUMPY_MMAP import --- examples/dmri_camino_dti.py | 3 ++- examples/dmri_connectivity.py | 9 +++++---- examples/fmri_ants_openfmri.py | 2 ++ examples/fmri_spm_auditory.py | 2 ++ examples/fmri_spm_face.py | 3 ++- examples/rsfmri_vol_surface_preprocessing.py | 2 ++ examples/rsfmri_vol_surface_preprocessing_nipy.py | 1 + nipype/algorithms/confounds.py | 6 +++--- nipype/algorithms/icc.py | 1 + nipype/algorithms/metrics.py | 2 ++ nipype/algorithms/misc.py | 2 ++ nipype/algorithms/tests/test_normalize_tpms.py | 1 + nipype/algorithms/tests/test_splitmerge.py | 1 + nipype/interfaces/cmtk/cmtk.py | 2 ++ nipype/interfaces/dcmstack.py | 1 + nipype/interfaces/dipy/preprocess.py | 2 ++ nipype/interfaces/dipy/simulate.py | 1 + nipype/interfaces/freesurfer/tests/test_model.py | 2 ++ nipype/interfaces/fsl/epi.py | 2 ++ nipype/interfaces/nipy/model.py | 2 ++ nipype/interfaces/nipy/preprocess.py | 2 ++ nipype/utils/__init__.py | 1 + nipype/utils/config.py | 15 +++++++++------ nipype/workflows/dmri/dipy/denoise.py | 1 + nipype/workflows/dmri/fsl/artifacts.py | 2 ++ nipype/workflows/dmri/fsl/epi.py | 1 + nipype/workflows/dmri/fsl/tbss.py | 1 + nipype/workflows/dmri/fsl/utils.py | 2 ++ nipype/workflows/misc/utils.py | 1 + nipype/workflows/smri/freesurfer/autorecon1.py | 2 ++ 30 files changed, 60 insertions(+), 15 deletions(-) diff --git a/examples/dmri_camino_dti.py b/examples/dmri_camino_dti.py index 8395d77082..9ad6d523ae 100755 --- a/examples/dmri_camino_dti.py +++ b/examples/dmri_camino_dti.py @@ -18,6 +18,7 @@ Import necessary modules from nipype. """ +import os # system functions import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine @@ -25,7 +26,7 @@ import nipype.interfaces.fsl as fsl import nipype.interfaces.camino2trackvis as cam2trk import nipype.algorithms.misc as misc -import os # system functions +from nipype.utils import NUMPY_MMAP """ We use the following functions to scrape the voxel and data dimensions of the input images. This allows the diff --git a/examples/dmri_connectivity.py b/examples/dmri_connectivity.py index c3d904c237..db7cf5cb88 100755 --- a/examples/dmri_connectivity.py +++ b/examples/dmri_connectivity.py @@ -47,6 +47,10 @@ First, we import the necessary modules from nipype. """ +import inspect + +import os.path as op # system functions +import cmp # connectome mapper import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine @@ -56,10 +60,7 @@ import nipype.interfaces.freesurfer as fs # freesurfer import nipype.interfaces.cmtk as cmtk import nipype.algorithms.misc as misc -import inspect - -import os.path as op # system functions -import cmp # connectome mapper +from nipype.utils import NUMPY_MMAP """ We define the following functions to scrape the voxel and data dimensions of the input images. This allows the diff --git a/examples/fmri_ants_openfmri.py b/examples/fmri_ants_openfmri.py index d421d6bfb7..ba5ce3ce0c 100755 --- a/examples/fmri_ants_openfmri.py +++ b/examples/fmri_ants_openfmri.py @@ -36,6 +36,8 @@ from nipype.workflows.fmri.fsl import (create_featreg_preproc, create_modelfit_workflow, create_fixed_effects_flow) +from nipype.utils import NUMPY_MMAP + config.enable_provenance() version = 0 diff --git a/examples/fmri_spm_auditory.py b/examples/fmri_spm_auditory.py index 8c229ffd4d..9b6de644d7 100755 --- a/examples/fmri_spm_auditory.py +++ b/examples/fmri_spm_auditory.py @@ -28,6 +28,8 @@ import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification import os # system functions +from nipype.utils import NUMPY_MMAP + """ diff --git a/examples/fmri_spm_face.py b/examples/fmri_spm_face.py index 8e81bea664..2876715699 100755 --- a/examples/fmri_spm_face.py +++ b/examples/fmri_spm_face.py @@ -20,13 +20,14 @@ from __future__ import division from builtins import range +import os # system functions import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.spm as spm # spm import nipype.interfaces.matlab as mlab # how to run matlab import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification -import os # system functions +from nipype.utils import NUMPY_MMAP """ diff --git a/examples/rsfmri_vol_surface_preprocessing.py b/examples/rsfmri_vol_surface_preprocessing.py index 0fc4e56d13..d01459e149 100644 --- a/examples/rsfmri_vol_surface_preprocessing.py +++ b/examples/rsfmri_vol_surface_preprocessing.py @@ -75,6 +75,8 @@ import numpy as np import scipy as sp import nibabel as nb +from nipype.utils import NUMPY_MMAP + imports = ['import os', 'import nibabel as nb', diff --git a/examples/rsfmri_vol_surface_preprocessing_nipy.py b/examples/rsfmri_vol_surface_preprocessing_nipy.py index b4837f2725..ab1d4f14d8 100644 --- a/examples/rsfmri_vol_surface_preprocessing_nipy.py +++ b/examples/rsfmri_vol_surface_preprocessing_nipy.py @@ -75,6 +75,7 @@ import numpy as np import scipy as sp import nibabel as nb +from nipype.utils import NUMPY_MMAP imports = ['import os', 'import nibabel as nb', diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 3163e98e74..c6503c703a 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -16,21 +16,21 @@ import os import os.path as op -from distutils.version import LooseVersion import nibabel as nb import numpy as np from numpy.polynomial import Legendre -from scipy import linalg, signal +from scipy import linalg from .. import logging from ..external.due import BibTeX from ..interfaces.base import (traits, TraitedSpec, BaseInterface, BaseInterfaceInputSpec, File, isdefined, InputMultiPath) +from nipype.utils import NUMPY_MMAP + IFLOG = logging.getLogger('interface') -NUMPY_MMAP = LooseVersion(np.__version__) >= LooseVersion('1.12.0') class ComputeDVARSInputSpec(BaseInterfaceInputSpec): in_file = File(exists=True, mandatory=True, desc='functional data, after HMC') diff --git a/nipype/algorithms/icc.py b/nipype/algorithms/icc.py index b497f9b7a6..36d2cb4221 100644 --- a/nipype/algorithms/icc.py +++ b/nipype/algorithms/icc.py @@ -8,6 +8,7 @@ from scipy.linalg import pinv from ..interfaces.base import BaseInterfaceInputSpec, TraitedSpec, \ BaseInterface, traits, File +from nipype.utils import NUMPY_MMAP class ICCInputSpec(BaseInterfaceInputSpec): diff --git a/nipype/algorithms/metrics.py b/nipype/algorithms/metrics.py index 7bc3a16e6c..a184dbad20 100644 --- a/nipype/algorithms/metrics.py +++ b/nipype/algorithms/metrics.py @@ -30,6 +30,8 @@ from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File, InputMultiPath, BaseInterfaceInputSpec, isdefined) +from nipype.utils import NUMPY_MMAP + iflogger = logging.getLogger('interface') diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index fda84eef2e..b69bc7baaf 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -34,6 +34,8 @@ BaseInterfaceInputSpec, isdefined, DynamicTraitedSpec, Undefined) from ..utils.filemanip import fname_presuffix, split_filename +from nipype.utils import NUMPY_MMAP + from . import confounds iflogger = logging.getLogger('interface') diff --git a/nipype/algorithms/tests/test_normalize_tpms.py b/nipype/algorithms/tests/test_normalize_tpms.py index 2550ddbdab..19a183bee0 100644 --- a/nipype/algorithms/tests/test_normalize_tpms.py +++ b/nipype/algorithms/tests/test_normalize_tpms.py @@ -14,6 +14,7 @@ import nipype.testing as nit from nipype.algorithms.misc import normalize_tpms +from nipype.utils import NUMPY_MMAP def test_normalize_tpms(tmpdir): diff --git a/nipype/algorithms/tests/test_splitmerge.py b/nipype/algorithms/tests/test_splitmerge.py index 1ac376e5ac..e122fef077 100644 --- a/nipype/algorithms/tests/test_splitmerge.py +++ b/nipype/algorithms/tests/test_splitmerge.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from nipype.testing import example_data +from nipype.utils import NUMPY_MMAP def test_split_and_merge(tmpdir): diff --git a/nipype/interfaces/cmtk/cmtk.py b/nipype/interfaces/cmtk/cmtk.py index d60587078c..7d65af99a7 100644 --- a/nipype/interfaces/cmtk/cmtk.py +++ b/nipype/interfaces/cmtk/cmtk.py @@ -22,6 +22,8 @@ from ... import logging from ...utils.filemanip import split_filename +from ...utils import NUMPY_MMAP + from ..base import (BaseInterface, BaseInterfaceInputSpec, traits, File, TraitedSpec, Directory, OutputMultiPath, isdefined) iflogger = logging.getLogger('interface') diff --git a/nipype/interfaces/dcmstack.py b/nipype/interfaces/dcmstack.py index 584f8d3143..e9dab240f6 100644 --- a/nipype/interfaces/dcmstack.py +++ b/nipype/interfaces/dcmstack.py @@ -23,6 +23,7 @@ InputMultiPath, File, Directory, traits, BaseInterface) from .traits_extension import isdefined, Undefined +from ..utils import NUMPY_MMAP have_dcmstack = True diff --git a/nipype/interfaces/dipy/preprocess.py b/nipype/interfaces/dipy/preprocess.py index 1680e87ffc..8bc28dbdbd 100644 --- a/nipype/interfaces/dipy/preprocess.py +++ b/nipype/interfaces/dipy/preprocess.py @@ -16,6 +16,8 @@ from ... import logging from ..base import (traits, TraitedSpec, File, isdefined) from .base import DipyBaseInterface +from ...utils import NUMPY_MMAP + IFLOGGER = logging.getLogger('interface') diff --git a/nipype/interfaces/dipy/simulate.py b/nipype/interfaces/dipy/simulate.py index ac58a83f59..0331171811 100644 --- a/nipype/interfaces/dipy/simulate.py +++ b/nipype/interfaces/dipy/simulate.py @@ -13,6 +13,7 @@ import nibabel as nb from ... import logging +from ...utils import NUMPY_MMAP from ..base import (traits, TraitedSpec, BaseInterfaceInputSpec, File, InputMultiPath, isdefined) from .base import DipyBaseInterface diff --git a/nipype/interfaces/freesurfer/tests/test_model.py b/nipype/interfaces/freesurfer/tests/test_model.py index bf8c167666..28e49401e0 100644 --- a/nipype/interfaces/freesurfer/tests/test_model.py +++ b/nipype/interfaces/freesurfer/tests/test_model.py @@ -7,6 +7,8 @@ import nibabel as nb import pytest + +from nipype.utils import NUMPY_MMAP from nipype.interfaces.freesurfer import model, no_freesurfer import nipype.pipeline.engine as pe diff --git a/nipype/interfaces/fsl/epi.py b/nipype/interfaces/fsl/epi.py index 6c6a96e522..2a54ec14b4 100644 --- a/nipype/interfaces/fsl/epi.py +++ b/nipype/interfaces/fsl/epi.py @@ -21,6 +21,8 @@ import warnings from ...utils.filemanip import split_filename +from ...utils import NUMPY_MMAP + from ..base import (traits, TraitedSpec, InputMultiPath, File, isdefined) from .base import FSLCommand, FSLCommandInputSpec diff --git a/nipype/interfaces/nipy/model.py b/nipype/interfaces/nipy/model.py index 28e8cffc12..58c07b5b95 100644 --- a/nipype/interfaces/nipy/model.py +++ b/nipype/interfaces/nipy/model.py @@ -8,6 +8,8 @@ import numpy as np from ...utils.misc import package_check +from ...utils import NUMPY_MMAP + from ..base import (BaseInterface, TraitedSpec, traits, File, OutputMultiPath, BaseInterfaceInputSpec, isdefined) diff --git a/nipype/interfaces/nipy/preprocess.py b/nipype/interfaces/nipy/preprocess.py index b0fd107080..c4b3ece2f1 100644 --- a/nipype/interfaces/nipy/preprocess.py +++ b/nipype/interfaces/nipy/preprocess.py @@ -16,6 +16,8 @@ import numpy as np from ...utils.misc import package_check +from ...utils import NUMPY_MMAP + from ...utils.filemanip import split_filename, fname_presuffix from ..base import (TraitedSpec, BaseInterface, traits, BaseInterfaceInputSpec, isdefined, File, diff --git a/nipype/utils/__init__.py b/nipype/utils/__init__.py index 691947f82f..decca90323 100644 --- a/nipype/utils/__init__.py +++ b/nipype/utils/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +from nipype.utils.config import NUMPY_MMAP from nipype.utils.onetime import OneTimeProperty, setattr_on_read from nipype.utils.tmpdirs import TemporaryDirectory, InTemporaryDirectory diff --git a/nipype/utils/config.py b/nipype/utils/config.py index 3bbeb22323..7130c100e3 100644 --- a/nipype/utils/config.py +++ b/nipype/utils/config.py @@ -10,22 +10,25 @@ @author: Chris Filo Gorgolewski ''' from __future__ import print_function, division, unicode_literals, absolute_import -from builtins import str, object, open - -from future import standard_library -standard_library.install_aliases() - -import configparser import os import shutil import errno from warnings import warn from io import StringIO +from distutils.version import LooseVersion from simplejson import load, dump +import numpy as np +from builtins import str, object, open +from future import standard_library +standard_library.install_aliases() + +import configparser from ..external import portalocker +NUMPY_MMAP = LooseVersion(np.__version__) >= LooseVersion('1.12.0') + # Get home directory in platform-agnostic way homedir = os.path.expanduser('~') default_cfg = """ diff --git a/nipype/workflows/dmri/dipy/denoise.py b/nipype/workflows/dmri/dipy/denoise.py index aefb2329d4..14e85c938c 100644 --- a/nipype/workflows/dmri/dipy/denoise.py +++ b/nipype/workflows/dmri/dipy/denoise.py @@ -4,6 +4,7 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: from builtins import range +from nipype.utils import NUMPY_MMAP from ....pipeline import engine as pe from ....interfaces import utility as niu from ....interfaces import dipy diff --git a/nipype/workflows/dmri/fsl/artifacts.py b/nipype/workflows/dmri/fsl/artifacts.py index 89462924ce..237d7b2e01 100644 --- a/nipype/workflows/dmri/fsl/artifacts.py +++ b/nipype/workflows/dmri/fsl/artifacts.py @@ -4,6 +4,8 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: from __future__ import division +from nipype.utils import NUMPY_MMAP + from ....interfaces.io import JSONFileGrabber from ....interfaces import utility as niu from ....interfaces import ants diff --git a/nipype/workflows/dmri/fsl/epi.py b/nipype/workflows/dmri/fsl/epi.py index b174941a17..de023215ab 100644 --- a/nipype/workflows/dmri/fsl/epi.py +++ b/nipype/workflows/dmri/fsl/epi.py @@ -5,6 +5,7 @@ import warnings +from nipype.utils import NUMPY_MMAP from ....pipeline import engine as pe from ....interfaces import utility as niu from ....interfaces import fsl as fsl diff --git a/nipype/workflows/dmri/fsl/tbss.py b/nipype/workflows/dmri/fsl/tbss.py index 9ca9a12f98..4567b65f90 100644 --- a/nipype/workflows/dmri/fsl/tbss.py +++ b/nipype/workflows/dmri/fsl/tbss.py @@ -5,6 +5,7 @@ import os from warnings import warn +from nipype.utils import NUMPY_MMAP from ....pipeline import engine as pe from ....interfaces import utility as util from ....interfaces import fsl as fsl diff --git a/nipype/workflows/dmri/fsl/utils.py b/nipype/workflows/dmri/fsl/utils.py index cd45749b60..8078241ab3 100644 --- a/nipype/workflows/dmri/fsl/utils.py +++ b/nipype/workflows/dmri/fsl/utils.py @@ -6,6 +6,8 @@ from __future__ import print_function, division, unicode_literals, absolute_import from builtins import zip, next, range, str +from nipype.utils import NUMPY_MMAP + from ....pipeline import engine as pe from ....interfaces import utility as niu from ....interfaces import fsl diff --git a/nipype/workflows/misc/utils.py b/nipype/workflows/misc/utils.py index 06df275a1c..8d3524e2e0 100644 --- a/nipype/workflows/misc/utils.py +++ b/nipype/workflows/misc/utils.py @@ -4,6 +4,7 @@ from __future__ import print_function, division, unicode_literals, absolute_import from builtins import map, range +from nipype.utils import NUMPY_MMAP def get_vox_dims(volume): diff --git a/nipype/workflows/smri/freesurfer/autorecon1.py b/nipype/workflows/smri/freesurfer/autorecon1.py index 88b9dc84c1..1ec49740b2 100644 --- a/nipype/workflows/smri/freesurfer/autorecon1.py +++ b/nipype/workflows/smri/freesurfer/autorecon1.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- + +from nipype.utils import NUMPY_MMAP from nipype.interfaces.utility import Function,IdentityInterface import nipype.pipeline.engine as pe # pypeline engine from nipype.interfaces.freesurfer import * From 05d74ec37a16437d74fc7456797cb96802d1982d Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 15:07:51 -0800 Subject: [PATCH 26/28] more NUMPY_MMAP fixes --- nipype/algorithms/modelgen.py | 7 ++++--- nipype/workflows/fmri/fsl/preprocess.py | 6 ++++-- nipype/workflows/rsfmri/fsl/resting.py | 4 +++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/nipype/algorithms/modelgen.py b/nipype/algorithms/modelgen.py index 3cafa8c0ac..3a0b2ef8b0 100644 --- a/nipype/algorithms/modelgen.py +++ b/nipype/algorithms/modelgen.py @@ -28,6 +28,7 @@ import numpy as np from scipy.special import gammaln +from ..utils import NUMPY_MMAP from ..interfaces.base import (BaseInterface, TraitedSpec, InputMultiPath, traits, File, Bunch, BaseInterfaceInputSpec, isdefined) @@ -354,7 +355,7 @@ def _generate_standard_design(self, infolist, for i, out in enumerate(outliers): numscans = 0 for f in filename_to_list(sessinfo[i]['scans']): - shape = load(f).shape + shape = load(f, mmap=NUMPY_MMAP).shape if len(shape) == 3 or shape[3] == 1: iflogger.warning(("You are using 3D instead of 4D " "files. Are you sure this was " @@ -457,7 +458,7 @@ def _concatenate_info(self, infolist): if isinstance(f, list): numscans = len(f) elif isinstance(f, (str, bytes)): - img = load(f) + img = load(f, mmap=NUMPY_MMAP) numscans = img.shape[3] else: raise Exception('Functional input not specified correctly') @@ -781,7 +782,7 @@ def _generate_clustered_design(self, infolist): infoout[i].onsets = None infoout[i].durations = None if info.conditions: - img = load(self.inputs.functional_runs[i]) + img = load(self.inputs.functional_runs[i], mmap=NUMPY_MMAP) nscans = img.shape[3] reg, regnames = self._cond_to_regress(info, nscans) if hasattr(infoout[i], 'regressors') and infoout[i].regressors: diff --git a/nipype/workflows/fmri/fsl/preprocess.py b/nipype/workflows/fmri/fsl/preprocess.py index d3f2d25ebe..9660f19c53 100644 --- a/nipype/workflows/fmri/fsl/preprocess.py +++ b/nipype/workflows/fmri/fsl/preprocess.py @@ -28,19 +28,21 @@ def pickfirst(files): def pickmiddle(files): from nibabel import load import numpy as np + from nipype.utils import NUMPY_MMAP middlevol = [] for f in files: - middlevol.append(int(np.ceil(load(f).shape[3] / 2))) + middlevol.append(int(np.ceil(load(f, mmap=NUMPY_MMAP).shape[3] / 2))) return middlevol def pickvol(filenames, fileidx, which): from nibabel import load import numpy as np + from nipype.utils import NUMPY_MMAP if which.lower() == 'first': idx = 0 elif which.lower() == 'middle': - idx = int(np.ceil(load(filenames[fileidx]).shape[3] / 2)) + idx = int(np.ceil(load(filenames[fileidx], mmap=NUMPY_MMAP).shape[3] / 2)) elif which.lower() == 'last': idx = load(filenames[fileidx]).shape[3] - 1 else: diff --git a/nipype/workflows/rsfmri/fsl/resting.py b/nipype/workflows/rsfmri/fsl/resting.py index a51693fda5..c15ca27061 100644 --- a/nipype/workflows/rsfmri/fsl/resting.py +++ b/nipype/workflows/rsfmri/fsl/resting.py @@ -14,10 +14,12 @@ def select_volume(filename, which): """ from nibabel import load import numpy as np + from nipype.utils import NUMPY_MMAP + if which.lower() == 'first': idx = 0 elif which.lower() == 'middle': - idx = int(np.ceil(load(filename).shape[3] / 2)) + idx = int(np.ceil(load(filename, mmap=NUMPY_MMAP).shape[3] / 2)) else: raise Exception('unknown value for volume selection : %s' % which) return idx From 875ed0babe532609566f99ce6af9fd54142cc4bb Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 15:11:57 -0800 Subject: [PATCH 27/28] the last nibabel.load without NUMPY_MMAP --- examples/fmri_fsl.py | 3 ++- nipype/algorithms/rapidart.py | 9 +++++---- nipype/interfaces/spm/base.py | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/fmri_fsl.py b/examples/fmri_fsl.py index 7fddfc76c4..9772290200 100755 --- a/examples/fmri_fsl.py +++ b/examples/fmri_fsl.py @@ -106,10 +106,11 @@ def pickfirst(files): def getmiddlevolume(func): from nibabel import load + from nipype.utils import NUMPY_MMAP funcfile = func if isinstance(func, list): funcfile = func[0] - _, _, _, timepoints = load(funcfile).shape + _, _, _, timepoints = load(funcfile, mmap=NUMPY_MMAP).shape return int(timepoints / 2) - 1 preproc.connect(inputnode, ('func', getmiddlevolume), extract_ref, 't_min') diff --git a/nipype/algorithms/rapidart.py b/nipype/algorithms/rapidart.py index 084005b25b..3b3295eab0 100644 --- a/nipype/algorithms/rapidart.py +++ b/nipype/algorithms/rapidart.py @@ -29,6 +29,7 @@ from scipy import signal import scipy.io as sio +from ..utils import NUMPY_MMAP from ..interfaces.base import (BaseInterface, traits, InputMultiPath, OutputMultiPath, TraitedSpec, File, BaseInterfaceInputSpec, isdefined) @@ -352,12 +353,12 @@ def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None): # read in functional image if isinstance(imgfile, (str, bytes)): - nim = load(imgfile) + nim = load(imgfile, mmap=NUMPY_MMAP) elif isinstance(imgfile, list): if len(imgfile) == 1: - nim = load(imgfile[0]) + nim = load(imgfile[0], mmap=NUMPY_MMAP) else: - images = [load(f) for f in imgfile] + images = [load(f, mmap=NUMPY_MMAP) for f in imgfile] nim = funcs.concat_images(images) # compute global intensity signal @@ -394,7 +395,7 @@ def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None): mask[:, :, :, t0] = mask_tmp g[t0] = np.nansum(vol * mask_tmp) / np.nansum(mask_tmp) elif masktype == 'file': # uses a mask image to determine intensity - maskimg = load(self.inputs.mask_file) + maskimg = load(self.inputs.mask_file, mmap=NUMPY_MMAP) mask = maskimg.get_data() affine = maskimg.affine mask = mask > 0.5 diff --git a/nipype/interfaces/spm/base.py b/nipype/interfaces/spm/base.py index 09c9595a6b..25ab67d412 100644 --- a/nipype/interfaces/spm/base.py +++ b/nipype/interfaces/spm/base.py @@ -28,7 +28,7 @@ # Local imports from ... import logging -from ...utils import spm_docs as sd +from ...utils import spm_docs as sd, NUMPY_MMAP from ..base import (BaseInterface, traits, isdefined, InputMultiPath, BaseInterfaceInputSpec, Directory, Undefined) from ..matlab import MatlabCommand @@ -45,7 +45,7 @@ def func_is_3d(in_file): if isinstance(in_file, list): return func_is_3d(in_file[0]) else: - img = load(in_file) + img = load(in_file, mmap=NUMPY_MMAP) shape = img.shape if len(shape) == 3 or (len(shape) == 4 and shape[3] == 1): return True @@ -73,7 +73,7 @@ def scans_for_fname(fname): for sno, f in enumerate(fname): scans[sno] = '%s,1' % f return scans - img = load(fname) + img = load(fname, mmap=NUMPY_MMAP) if len(img.shape) == 3: return np.array(('%s,1' % fname,), dtype=object) else: From 20e10f9f3ff391e741c62ca4c5406ae48b53dff7 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 15:17:56 -0800 Subject: [PATCH 28/28] fix import NUMPY_MMAP location in examples --- examples/dmri_camino_dti.py | 4 +++- examples/dmri_connectivity.py | 4 +++- examples/fmri_spm_auditory.py | 2 +- examples/fmri_spm_face.py | 2 +- examples/rsfmri_vol_surface_preprocessing.py | 6 +++++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/dmri_camino_dti.py b/examples/dmri_camino_dti.py index 9ad6d523ae..3a9528bf10 100755 --- a/examples/dmri_camino_dti.py +++ b/examples/dmri_camino_dti.py @@ -26,7 +26,6 @@ import nipype.interfaces.fsl as fsl import nipype.interfaces.camino2trackvis as cam2trk import nipype.algorithms.misc as misc -from nipype.utils import NUMPY_MMAP """ We use the following functions to scrape the voxel and data dimensions of the input images. This allows the @@ -37,6 +36,7 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] nii = nb.load(volume, mmap=NUMPY_MMAP) @@ -47,6 +47,7 @@ def get_vox_dims(volume): def get_data_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] nii = nb.load(volume, mmap=NUMPY_MMAP) @@ -57,6 +58,7 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine diff --git a/examples/dmri_connectivity.py b/examples/dmri_connectivity.py index db7cf5cb88..5ebbbb77d5 100755 --- a/examples/dmri_connectivity.py +++ b/examples/dmri_connectivity.py @@ -60,7 +60,6 @@ import nipype.interfaces.freesurfer as fs # freesurfer import nipype.interfaces.cmtk as cmtk import nipype.algorithms.misc as misc -from nipype.utils import NUMPY_MMAP """ We define the following functions to scrape the voxel and data dimensions of the input images. This allows the @@ -75,6 +74,7 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] nii = nb.load(volume, mmap=NUMPY_MMAP) @@ -85,6 +85,7 @@ def get_vox_dims(volume): def get_data_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] nii = nb.load(volume, mmap=NUMPY_MMAP) @@ -95,6 +96,7 @@ def get_data_dims(volume): def get_affine(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP nii = nb.load(volume, mmap=NUMPY_MMAP) return nii.affine diff --git a/examples/fmri_spm_auditory.py b/examples/fmri_spm_auditory.py index 9b6de644d7..33c03f8bf0 100755 --- a/examples/fmri_spm_auditory.py +++ b/examples/fmri_spm_auditory.py @@ -28,7 +28,6 @@ import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification import os # system functions -from nipype.utils import NUMPY_MMAP """ @@ -122,6 +121,7 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] nii = nb.load(volume, mmap=NUMPY_MMAP) diff --git a/examples/fmri_spm_face.py b/examples/fmri_spm_face.py index 2876715699..c716e4793f 100755 --- a/examples/fmri_spm_face.py +++ b/examples/fmri_spm_face.py @@ -27,7 +27,6 @@ import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification -from nipype.utils import NUMPY_MMAP """ @@ -115,6 +114,7 @@ def get_vox_dims(volume): import nibabel as nb + from nipype.utils import NUMPY_MMAP if isinstance(volume, list): volume = volume[0] nii = nb.load(volume, mmap=NUMPY_MMAP) diff --git a/examples/rsfmri_vol_surface_preprocessing.py b/examples/rsfmri_vol_surface_preprocessing.py index d01459e149..77c7598f84 100644 --- a/examples/rsfmri_vol_surface_preprocessing.py +++ b/examples/rsfmri_vol_surface_preprocessing.py @@ -75,7 +75,6 @@ import numpy as np import scipy as sp import nibabel as nb -from nipype.utils import NUMPY_MMAP imports = ['import os', @@ -119,6 +118,7 @@ def median(in_files): """ import numpy as np import nibabel as nb + from nipype.utils import NUMPY_MMAP average = None for idx, filename in enumerate(filename_to_list(in_files)): img = nb.load(filename, mmap=NUMPY_MMAP) @@ -147,6 +147,7 @@ def bandpass_filter(files, lowpass_freq, highpass_freq, fs): from nipype.utils.filemanip import split_filename, list_to_filename import numpy as np import nibabel as nb + from nipype.utils import NUMPY_MMAP out_files = [] for filename in filename_to_list(files): path, name, ext = split_filename(filename) @@ -262,6 +263,7 @@ def extract_noise_components(realigned_file, mask_file, num_components=5, from scipy.linalg.decomp_svd import svd import numpy as np import nibabel as nb + from nipype.utils import NUMPY_MMAP import os imgseries = nb.load(realigned_file, mmap=NUMPY_MMAP) components = None @@ -331,6 +333,7 @@ def extract_subrois(timeseries_file, label_file, indices): """ from nipype.utils.filemanip import split_filename import nibabel as nb + from nipype.utils import NUMPY_MMAP import os img = nb.load(timeseries_file, mmap=NUMPY_MMAP) data = img.get_data() @@ -354,6 +357,7 @@ def combine_hemi(left, right): """ import os import numpy as np + from nipype.utils import NUMPY_MMAP lh_data = nb.load(left, mmap=NUMPY_MMAP).get_data() rh_data = nb.load(right, mmap=NUMPY_MMAP).get_data()