[DOC] add documentation & doc tests to fsl preprocess

.circle/codecov.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# This script pull all coverage files into the $CIRCLE_TEST_REPORTS folder
+# and sends data to codecov.
+#
+
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
+
+mkdir -p ${CIRCLE_TEST_REPORTS}/
+for report in $( ls ~/scratch/*.xml ); do
+    rname=$( basename $report )
+    cp ${report} ${CIRCLE_TEST_REPORTS}/${rname:: -4}_${CIRCLE_NODE_INDEX}.xml
+done
+
+# Send coverage data to codecov.io
+curl -so codecov.io https://codecov.io/bash
+chmod 755 codecov.io
+
+find "${CIRCLE_TEST_REPORTS}/" -name 'coverage*.xml' -print0 | \
+  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F unittests
+find "${CIRCLE_TEST_REPORTS}/" -name 'smoketests*.xml' -print0 | \
+  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F smoketests

docker/files/tests.sh → .circle/tests.sh

@@ -1,9 +1,12 @@
 #!/bin/bash
+#
+# Balance nipype testing workflows across CircleCI build nodes
+#
 
-set -o nounset
-set -o xtrace
-
-export CODECOV_TOKEN=ac172a50-8e66-42e5-8822-5373fcf54686
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
 
 if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
   echo "These tests were designed to be run at 4x parallelism."
@@ -14,15 +17,15 @@ fi
 # They may need to be rebalanced in the future.
 case ${CIRCLE_NODE_INDEX} in
   0)
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 && \
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline
-    ;;
-  1)
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d && \
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d && \
     docker run --rm -it -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 && \
     docker run --rm -it -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 && \
-    docker run --rm -it -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh
+    docker run --rm -it -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh && \
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d && \
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d
+    ;;
+  1)
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 && \
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline
     ;;
   2)
     docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 && \
@@ -34,14 +37,3 @@ case ${CIRCLE_NODE_INDEX} in
     docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow
     ;;
 esac
-
-# Put the artifacts in place
-bash docker/files/teardown.sh
-
-# Send coverage data to codecov.io
-curl -so codecov.io https://codecov.io/bash
-chmod 755 codecov.io
-find "${CIRCLE_TEST_REPORTS}/pytest" -name 'coverage*.xml' -print0 | \
-  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F unittests
-find "${CIRCLE_TEST_REPORTS}/pytest" -name 'smoketests*.xml' -print0 | \
-  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F smoketests

.dockerignore

@@ -23,11 +23,13 @@ src/
 # other
 docs/**/*
 docs/
+.circle/**/*
+.circle/
+circle.yml
 .coverage
 .coveragerc
 codecov.yml
 rtd_requirements.txt
-circle.yml
 Vagrantfile
 .travis.yml
 .noserc

CHANGES

@@ -1,6 +1,7 @@
 Upcoming Release
 =====================
 
+* ENH: Added non-steady state detector for EPI data (https://github.com/nipy/nipype/pull/1839)
 * ENH: Enable new BBRegister init options for FSv6+ (https://github.com/nipy/nipype/pull/1811)
 * REF: Splits nipype.interfaces.utility into base, csv, and wrappers (https://github.com/nipy/nipype/pull/1828)
 * FIX: Makespec now runs with nipype in current directory (https://github.com/nipy/nipype/pull/1813)

README.rst

@@ -78,8 +78,8 @@ Support and Communication
 -------------------------
 
 If you have a problem or would like to ask a question about how to do something in Nipype please open an issue to
-`NeuroStars.org <http://neurostars.org>`_ with a *nipype* tag. `NeuroStars.org <http://neurostars.org>`_  is a 
-platform similar to StackOverflow but dedicated to neuroinformatics. 
+`NeuroStars.org <http://neurostars.org>`_ with a *nipype* tag. `NeuroStars.org <http://neurostars.org>`_  is a
+platform similar to StackOverflow but dedicated to neuroinformatics.
 
 To participate in the Nipype development related discussions please use the following mailing list::
 
@@ -117,16 +117,3 @@ Currently Nipype consists of the following files and directories:
   setup.py
     Script for building and installing NIPYPE.
 
-License information
--------------------
-
-We use the 3-clause BSD license; the full license is in the file ``LICENSE`` in
-the nipype distribution.
-
-There are interfaces to some GNU code but these are entirely optional.
-
-All trademarks referenced herein are property of their respective
-holders.
-
-Copyright (c) 2009-2015, NIPY Developers
-All rights reserved.

circle.yml

@@ -8,6 +8,7 @@ machine:
     DATA_NIPYPE_FSL_COURSE: "${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
     DATA_NIPYPE_FSL_FEEDS: "${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
     SCRATCH: "$HOME/scratch"
+    CODECOV_TOKEN: "ac172a50-8e66-42e5-8822-5373fcf54686"
   services:
     - docker
 
@@ -32,34 +33,39 @@ dependencies:
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
     - docker images
     - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
-    - e=1 && for i in {1..5}; do docker build -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 21600
-    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - e=1 && for i in {1..5}; do docker build --rm=false -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 1600
-    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - e=1 && for i in {1..5}; do docker build --rm=false -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 1600
     - docker save -o $HOME/docker/cache.tar nipype/nipype:latest nipype/nipype_test:py27 nipype/nipype_test:py35 :
         timeout: 6000
 
 test:
   override:
-    - bash docker/files/tests.sh :
+    - bash .circle/tests.sh :
         timeout: 7200
         parallel: true
+  post:
+    # Send coverage data to codecov.io
+    - bash .circle/codecov.sh
 
 general:
   artifacts:
-    - "~/docs"
-    - "~/logs"
+    - "~/scratch/docs"
+    - "~/scratch/logs"
 
 deployment:
   production:
     tag: /.*/
     commands:
+      # Deploy to docker hub      
       - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi :
           timeout: 21600
       - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi :
           timeout: 21600
+
 # Automatic deployment to Pypi:
 #       - printf "[distutils]\nindex-servers =\n    pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc
 #       - python setup.py sdist upload -r pypi

nipype/algorithms/confounds.py

@@ -579,6 +579,79 @@ def _list_outputs(self):
             outputs['detrended_file'] = op.abspath(self.inputs.detrended_file)
         return outputs
 
+
+class NonSteadyStateDetectorInputSpec(BaseInterfaceInputSpec):
+    in_file = File(exists=True, mandatory=True, desc='4D NIFTI EPI file')
+
+
+class NonSteadyStateDetectorOutputSpec(TraitedSpec):
+    n_volumes_to_discard = traits.Int(desc='Number of non-steady state volumes'
+                                           'detected in the beginning of the scan.')
+
+
+class NonSteadyStateDetector(BaseInterface):
+    """
+    Returns the number of non-steady state volumes detected at the beginning
+    of the scan.
+    """
+
+    input_spec = NonSteadyStateDetectorInputSpec
+    output_spec = NonSteadyStateDetectorOutputSpec
+
+    def _run_interface(self, runtime):
+        in_nii = nb.load(self.inputs.in_plots)
+        global_signal = in_nii.get_data()[:,:,:,:50].mean(axis=0).mean(axis=0).mean(axis=0)
+
+        self._results = {
+            'out_file': _is_outlier(global_signal)
+        }
+
+        return runtime
+
+    def _list_outputs(self):
+        return self._results
+
+def _is_outlier(points, thresh=3.5):
+    """
+    Returns a boolean array with True if points are outliers and False
+    otherwise.
+
+    Parameters:
+    -----------
+        points : An numobservations by numdimensions array of observations
+        thresh : The modified z-score to use as a threshold. Observations with
+            a modified z-score (based on the median absolute deviation) greater
+            than this value will be classified as outliers.
+
+    Returns:
+    --------
+        mask : A numobservations-length boolean array.
+
+    References:
+    ----------
+        Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and
+        Handle Outliers", The ASQC Basic References in Quality Control:
+        Statistical Techniques, Edward F. Mykytka, Ph.D., Editor.
+    """
+    if len(points.shape) == 1:
+        points = points[:, None]
+    median = np.median(points, axis=0)
+    diff = np.sum((points - median) ** 2, axis=-1)
+    diff = np.sqrt(diff)
+    med_abs_deviation = np.median(diff)
+
+    modified_z_score = 0.6745 * diff / med_abs_deviation
+
+    timepoints_to_discard = 0
+    for i in range(len(modified_z_score)):
+        if modified_z_score[i] <= thresh:
+            break
+        else:
+            timepoints_to_discard += 1
+
+    return timepoints_to_discard
+
+
 def regress_poly(degree, data, remove_mean=True, axis=-1):
     ''' returns data with degree polynomial regressed out.
     Be default it is calculated along the last axis (usu. time).

nipype/algorithms/tests/test_auto_ComputeDVARS.py

@@ -17,6 +17,8 @@ def test_ComputeDVARS_inputs():
     ),
     in_mask=dict(mandatory=True,
     ),
+    intensity_normalization=dict(usedefault=True,
+    ),
     remove_zerovariance=dict(usedefault=True,
     ),
     save_all=dict(usedefault=True,

nipype/algorithms/tests/test_auto_FramewiseDisplacement.py

@@ -19,6 +19,8 @@ def test_FramewiseDisplacement_inputs():
     ),
     out_file=dict(usedefault=True,
     ),
+    parameter_source=dict(mandatory=True,
+    ),
     radius=dict(usedefault=True,
     ),
     save_plot=dict(usedefault=True,

nipype/algorithms/tests/test_auto_ErrorMap.py → nipype/algorithms/tests/test_auto_NonSteadyStateDetector.py

@@ -1,34 +1,26 @@
 # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT
 from __future__ import unicode_literals
-from ..metrics import ErrorMap
+from ..confounds import NonSteadyStateDetector
 
 
-def test_ErrorMap_inputs():
+def test_NonSteadyStateDetector_inputs():
     input_map = dict(ignore_exception=dict(nohash=True,
     usedefault=True,
     ),
-    in_ref=dict(mandatory=True,
+    in_file=dict(mandatory=True,
     ),
-    in_tst=dict(mandatory=True,
-    ),
-    mask=dict(),
-    metric=dict(mandatory=True,
-    usedefault=True,
-    ),
-    out_map=dict(),
     )
-    inputs = ErrorMap.input_spec()
+    inputs = NonSteadyStateDetector.input_spec()
 
     for key, metadata in list(input_map.items()):
         for metakey, value in list(metadata.items()):
             assert getattr(inputs.traits()[key], metakey) == value
 
 
-def test_ErrorMap_outputs():
-    output_map = dict(distance=dict(),
-    out_map=dict(),
+def test_NonSteadyStateDetector_outputs():
+    output_map = dict(n_volumes_to_discard=dict(),
     )
-    outputs = ErrorMap.output_spec()
+    outputs = NonSteadyStateDetector.output_spec()
 
     for key, metadata in list(output_map.items()):
         for metakey, value in list(metadata.items()):