Merge pull request #6 from nipy/master

Re-basing code with nipype master branch

Author: pintohutch

.noserc

@@ -0,0 +1,11 @@
+[nosetests]
+verbosity=3
+
+with-coverage=1
+cover-branches=1
+cover-xml=1
+cover-xml-file=./coverage.xml
+cover-min-percentage=50
+
+
+with-xunit=1

.travis.yml

@@ -23,6 +23,8 @@ before_install:
 - if $INSTALL_DEB_DEPENDECIES; then travis_retry sudo apt-get install -qq fsl-atlases;
   fi
 - if $INSTALL_DEB_DEPENDECIES; then source /etc/fsl/fsl.sh; fi
+- if $INSTALL_DEB_DEPENDECIES; then source /etc/afni/afni.sh; fi
+- export FSLOUTPUTTYPE=NIFTI_GZ
 install:
 - conda update --yes conda
 - conda create -n testenv --yes pip python=$TRAVIS_PYTHON_VERSION

CHANGES

@@ -1,6 +1,10 @@
 Next release
 ============
 
+* ENH: Added support for PETPVC (https://github.com/nipy/nipype/pull/1335)
+* ENH: Merge S3DataSink into DataSink, added AWS documentation (https://github.com/nipy/nipype/pull/1316)
+* TST: Cache APT in CircleCI (https://github.com/nipy/nipype/pull/1333)
+* ENH: Add new flags to the BRAINSABC for new features (https://github.com/nipy/nipype/pull/1322)
 * ENH: Provides a Nipype wrapper for ANTs DenoiseImage (https://github.com/nipy/nipype/pull/1291)
 * FIX: Minor bugfix logging hash differences (https://github.com/nipy/nipype/pull/1298)
 * FIX: Use released Prov python library (https://github.com/nipy/nipype/pull/1279)

circle.yml

@@ -1,83 +1,71 @@
 dependencies:
   cache_directories:
+    - "~/.apt-cache"
     - "~/examples/data"
     - "~/examples/fsdata"
-    - "~/examples/feeds"
     - "~/mcr"
     - "~/spm12"
-    - "~/fsl"
     - "~/examples/fsl_course_data"
+  pre:
+    # Let CircleCI cache the apt archive
+    - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
+    - wget -O- http://neuro.debian.net/lists/precise.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
+    - sudo apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9
+    - sudo apt-get update
   override:
+    # Install apt packages
+    - sudo apt-get install -y fsl-core fsl-atlases fsl-mni152-templates fsl-feeds afni
+    - echo "source /etc/fsl/fsl.sh" >> $HOME/.profile
+    - echo "source /etc/afni/afni.sh" >> $HOME/.profile
+    - mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
+    # Set up python environment
     - pip install --upgrade pip
     - pip install -e .
-    - pip install matplotlib sphinx ipython boto
+    - pip install matplotlib sphinx ipython boto coverage
     - gem install fakes3
-    - if [[ ! -d ~/fsl ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.9-centos6_64.tar.gz"; tar zxvf fsl-5.0.9-centos6_64.tar.gz; mv fsl ~/fsl; fi
-    - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2"; tar jxvf nipype-tutorial.tar.bz2; mkdir ~/examples; mv nipype-tutorial/* ~/examples/; fi
-    # we download this manually because CircleCI does not cache apt
-    - if [[ ! -d ~/examples/feeds ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.9-feeds.tar.gz"; tar zxvf fsl-5.0.9-feeds.tar.gz; mv feeds ~/examples/; fi
-    - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" ; wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz"; wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz"; mkdir ~/examples/fsl_course_data; tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data; tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data; tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
+    - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
+    - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && mkdir ~/examples/fsl_course_data && tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
     - bash ~/nipype/tools/install_spm_mcr.sh
-    - mkdir -p ~/.nipype && echo "[logging]" > ~/.nipype/nipype.cfg && echo "workflow_level = DEBUG" >> ~/.nipype/nipype.cfg && echo "interface_level = DEBUG" >> ~/.nipype/nipype.cfg && echo "filemanip_level = DEBUG" >> ~/.nipype/nipype.cfg
+    - mkdir -p ~/.nipype && echo '[logging]' > ~/.nipype/nipype.cfg && echo 'workflow_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'interface_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'filemanip_level = DEBUG' >> ~/.nipype/nipype.cfg
+machine:
+  environment:
+    FSLOUTPUTTYPE: NIFTI_GZ
 test:
   override:
-    - nosetests --with-doctest --logging-level=DEBUG --verbosity=3:
+    - mkdir -p ${CIRCLE_TEST_REPORTS}/nose
+    - source $HOME/.profile; nosetests --with-doctest --xunit-file="${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml" -c ./.noserc --logging-level=DEBUG --verbosity=3:
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
           FSL_COURSE_DATA: "$HOME/examples/fsl_course_data"
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
         timeout: 2600
     - set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt
     - cat ~/log.txt && if grep -q "ERROR" ~/log.txt; then false; else true; fi
-    - python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
         pwd: ../examples
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
         timeout: 1600
-    - python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
         pwd: ../examples
-        environment:
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
-    - python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
         pwd: ../examples
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
         timeout: 1600
-    - python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
         pwd: ../examples
-        environment:
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
-    - python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
         pwd: ../examples
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
 
 general:
   artifacts:
     - "doc/_build/html"
     - "~/log.txt"
+    - "nosetests.xml"
+    - "coverage.xml"

doc/users/aws.rst

@@ -0,0 +1,102 @@
+.. _aws:
+
+============================================
+Using Nipype with Amazon Web Services (AWS)
+============================================
+Several groups have been successfully using Nipype on AWS. This procedure
+involves setting a temporary cluster using StarCluster and potentially
+transferring files to/from S3. The latter is supported by Nipype through
+DataSink and S3DataGrabber.
+
+
+Using DataSink with S3
+======================
+The DataSink class now supports sending output data directly to an AWS S3
+bucket. It does this through the introduction of several input attributes to the
+DataSink interface and by parsing the `base_directory` attribute. This class
+uses the `boto3 <https://boto3.readthedocs.org/en/latest/>`_ and
+`botocore <https://botocore.readthedocs.org/en/latest/>`_ Python packages to
+interact with AWS. To configure the DataSink to write data to S3, the user must
+set the ``base_directory`` property to an S3-style filepath. For example:
+
+::
+
+	import nipype.interfaces.io as nio
+	ds = nio.DataSink()
+	ds.inputs.base_directory = 's3://mybucket/path/to/output/dir'
+
+With the "s3://" prefix in the path, the DataSink knows that the output
+directory to send files is on S3 in the bucket "mybucket". "path/to/output/dir"
+is the relative directory path within the bucket "mybucket" where output data
+will be uploaded to (NOTE: if the relative path specified contains folders that
+don’t exist in the bucket, the DataSink will create them). The DataSink treats
+the S3 base directory exactly as it would a local directory, maintaining support
+for containers, substitutions, subfolders, "." notation, etc to route output
+data appropriately.
+
+There are four new attributes introduced with S3-compatibility: ``creds_path``,
+``encrypt_bucket_keys``, ``local_copy``, and ``bucket``.
+
+::
+
+	ds.inputs.creds_path = '/home/user/aws_creds/credentials.csv'
+	ds.inputs.encrypt_bucket_keys = True
+	ds.local_copy = '/home/user/workflow_outputs/local_backup'
+
+``creds_path`` is a file path where the user's AWS credentials file (typically
+a csv) is stored. This credentials file should contain the AWS access key id and
+secret access key and should be formatted as one of the following (these formats
+are how Amazon provides the credentials file by default when first downloaded).
+
+Root-account user:
+
+::
+
+	AWSAccessKeyID=ABCDEFGHIJKLMNOP
+	AWSSecretKey=zyx123wvu456/ABC890+gHiJk
+
+IAM-user:
+
+::
+
+	User Name,Access Key Id,Secret Access Key
+	"username",ABCDEFGHIJKLMNOP,zyx123wvu456/ABC890+gHiJk
+
+The ``creds_path`` is necessary when writing files to a bucket that has
+restricted access (almost no buckets are publicly writable). If ``creds_path``
+is not specified, the DataSink will check the ``AWS_ACCESS_KEY_ID`` and
+``AWS_SECRET_ACCESS_KEY`` environment variables and use those values for bucket
+access.
+
+``encrypt_bucket_keys`` is a boolean flag that indicates whether to encrypt the
+output data on S3, using server-side AES-256 encryption. This is useful if the
+data being output is sensitive and one desires an extra layer of security on the
+data. By default, this is turned off.
+
+``local_copy`` is a string of the filepath where local copies of the output data
+are stored in addition to those sent to S3. This is useful if one wants to keep
+a backup version of the data stored on their local computer. By default, this is
+turned off.
+
+``bucket`` is a boto3 Bucket object that the user can use to overwrite the
+bucket specified in their ``base_directory``. This can be useful if one has to
+manually create a bucket instance on their own using special credentials (or
+using a mock server like `fakes3 <https://github.com/jubos/fake-s3>`_). This is
+typically used for developers unit-testing the DataSink class. Most users do not
+need to use this attribute for actual workflows. This is an optional argument.
+
+Finally, the user needs only to specify the input attributes for any incoming
+data to the node, and the outputs will be written to their S3 bucket.
+
+::
+
+	workflow.connect(inputnode, 'subject_id', ds, 'container')
+	workflow.connect(realigner, 'realigned_files', ds, 'motion')
+
+So, for example, outputs for sub001’s realigned_file1.nii.gz will be in:
+s3://mybucket/path/to/output/dir/sub001/motion/realigned_file1.nii.gz
+
+
+Using S3DataGrabber
+======================
+Coming soon...

doc/users/index.rst

@@ -38,6 +38,7 @@
    spmmcr
    mipav
    nipypecmd
+   aws
 
 
 

nipype/algorithms/misc.py

@@ -263,6 +263,14 @@ class TSNRInputSpec(BaseInterfaceInputSpec):
     in_file = InputMultiPath(File(exists=True), mandatory=True,
                              desc='realigned 4D file or a list of 3D files')
     regress_poly = traits.Range(low=1, desc='Remove polynomials')
+    tsnr_file = File('tsnr.nii.gz', usedefault=True, hash_files=False,
+                     desc='output tSNR file')
+    mean_file = File('mean.nii.gz', usedefault=True, hash_files=False,
+                     desc='output mean file')
+    stddev_file = File('stdev.nii.gz', usedefault=True, hash_files=False,
+                       desc='output tSNR file')
+    detrended_file = File('detrend.nii.gz', usedefault=True, hash_files=False,
+                          desc='input file after detrending')
 
 
 class TSNROutputSpec(TraitedSpec):
@@ -288,24 +296,18 @@ class TSNR(BaseInterface):
     input_spec = TSNRInputSpec
     output_spec = TSNROutputSpec
 
-    def _gen_output_file_name(self, suffix=None):
-        _, base, ext = split_filename(self.inputs.in_file[0])
-        if suffix in ['mean', 'stddev']:
-            return os.path.abspath(base + "_tsnr_" + suffix + ext)
-        elif suffix in ['detrended']:
-            return os.path.abspath(base + "_" + suffix + ext)
-        else:
-            return os.path.abspath(base + "_tsnr" + ext)
-
     def _run_interface(self, runtime):
         img = nb.load(self.inputs.in_file[0])
         header = img.header.copy()
         vollist = [nb.load(filename) for filename in self.inputs.in_file]
         data = np.concatenate([vol.get_data().reshape(
-            vol.shape[:3] + (-1,)) for vol in vollist], axis=3)
+            vol.get_shape()[:3] + (-1,)) for vol in vollist], axis=3)
+        data = data.nan_to_num()
+
         if data.dtype.kind == 'i':
             header.set_data_dtype(np.float32)
             data = data.astype(np.float32)
+
         if isdefined(self.inputs.regress_poly):
             timepoints = img.shape[-1]
             X = np.ones((timepoints, 1))
@@ -318,26 +320,28 @@ def _run_interface(self, runtime):
                                              betas[1:, :, :, :], 0, 3)),
                                   0, 4)
             data = data - datahat
-            img = nb.Nifti1Image(data, img.affine, header)
-            nb.save(img, self._gen_output_file_name('detrended'))
+            img = nb.Nifti1Image(data, img.get_affine(), header)
+            nb.save(img, op.abspath(self.inputs.detrended_file))
+
         meanimg = np.mean(data, axis=3)
         stddevimg = np.std(data, axis=3)
-        tsnr = meanimg / stddevimg
-        img = nb.Nifti1Image(tsnr, img.affine, header)
-        nb.save(img, self._gen_output_file_name())
-        img = nb.Nifti1Image(meanimg, img.affine, header)
-        nb.save(img, self._gen_output_file_name('mean'))
-        img = nb.Nifti1Image(stddevimg, img.affine, header)
-        nb.save(img, self._gen_output_file_name('stddev'))
+        tsnr = np.zeros_like(meanimg)
+        tsnr[stddevimg > 1.e-3] = meanimg[stddevimg > 1.e-3] / stddevimg[stddevimg > 1.e-3]
+        img = nb.Nifti1Image(tsnr, img.get_affine(), header)
+        nb.save(img, op.abspath(self.inputs.tsnr_file))
+        img = nb.Nifti1Image(meanimg, img.get_affine(), header)
+        nb.save(img, op.abspath(self.inputs.mean_file))
+        img = nb.Nifti1Image(stddevimg, img.get_affine(), header)
+        nb.save(img, op.abspath(self.inputs.stddev_file))
         return runtime
 
     def _list_outputs(self):
         outputs = self._outputs().get()
-        outputs['tsnr_file'] = self._gen_output_file_name()
-        outputs['mean_file'] = self._gen_output_file_name('mean')
-        outputs['stddev_file'] = self._gen_output_file_name('stddev')
+        for k in ['tsnr_file', 'mean_file', 'stddev_file']:
+            outputs[k] = op.abspath(getattr(self.inputs, k))
+
         if isdefined(self.inputs.regress_poly):
-            outputs['detrended_file'] = self._gen_output_file_name('detrended')
+            outputs['detrended_file'] = op.abspath(self.inputs.detrended_file)
         return outputs
 
 

nipype/algorithms/tests/test_auto_TSNR.py

@@ -4,12 +4,24 @@
 
 
 def test_TSNR_inputs():
-    input_map = dict(ignore_exception=dict(nohash=True,
+    input_map = dict(detrended_file=dict(hash_files=False,
+    usedefault=True,
+    ),
+    ignore_exception=dict(nohash=True,
     usedefault=True,
     ),
     in_file=dict(mandatory=True,
     ),
+    mean_file=dict(hash_files=False,
+    usedefault=True,
+    ),
     regress_poly=dict(),
+    stddev_file=dict(hash_files=False,
+    usedefault=True,
+    ),
+    tsnr_file=dict(hash_files=False,
+    usedefault=True,
+    ),
     )
     inputs = TSNR.input_spec()