[testing] Start using docker tags for circleCI

.dockerignore

@@ -0,0 +1,5 @@
+.git/
+*.pyc
+*.egg-info
+__pycache__
+

CHANGES

@@ -1,8 +1,8 @@
 Upcoming release 0.13
 =====================
 
-* ENH: Implement missing inputs/outputs in FSL AvScale
-       (https://github.com/nipy/nipype/pull/1563)
+* TST: reduce the size of docker images & use tags for images (https://github.com/nipy/nipype/pull/1564)
+* ENH: Implement missing inputs/outputs in FSL AvScale (https://github.com/nipy/nipype/pull/1563)
 
 
 Release 0.12.1 (August 3, 2016)

circle.yml

@@ -5,33 +5,41 @@ machine:
 dependencies:
   cache_directories:
     - "~/docker"
+    - "~/examples"
+    - "~/.apt-cache"
 
   pre:
     - mkdir -p "~/scratch/nose"
+    - mkdir -p "~/examples"
+    # Let CircleCI cache the apt archive
+    - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
+    - sudo apt-get -y update && sudo apt-get install -y wget bzip2
 
   override:
     - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
-    - docker build -t nipype/testbench:latest . :
+    - docker build -f docker/nipype_test_py27/Dockerfile -t nipype/nipype_test:py27 . :
         timeout: 1600
-    - mkdir -p ~/docker; docker save nipype/testbench:latest > ~/docker/image.tar :
+    - mkdir -p ~/docker; docker save nipype/nipype_test:py27 > ~/docker/image.tar :
         timeout: 1600
+    - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget -q -O nipype-tutorial.tar.bz2 https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2 && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi
+    - if [[ ! -d ~/examples/feeds ]]; then wget -q -O fsl-feeds.tar.gz https://googledrive.com/host/0BxI12kyv2olZNXBONlJKV0Y1Tm8 && tar xzf fsl-feeds.tar.gz -C ~/examples/; fi
 
 test:
   override:
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch --entrypoint="/usr/bin/run_builddocs.sh" nipype/testbench
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench test_spm Linear /root/examples/ workflow3d :
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /root/src/nipype/doc nipype/nipype_test:py27 /usr/bin/run_builddocs.sh
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d :
         timeout: 1600
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench test_spm Linear /root/examples/ workflow4d :
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d :
         timeout: 1600
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_fsl_feeds Linear /root/examples/ l1pipeline
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_dartel Linear /root/examples/ level1 :
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /root/examples/ l1pipeline
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 :
         timeout: 1600
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_dartel Linear /root/examples/ l2pipeline :
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline :
         timeout: 1600
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_fsl_reuse Linear /root/examples/ level1_workflow
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_nested Linear /root/examples/ level1
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_nested Linear /root/examples/ l2pipeline
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch --entrypoint="/usr/bin/run_nosetests.sh" nipype/testbench :
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested Linear /root/examples/ level1
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v $(pwd)/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested Linear /root/examples/ l2pipeline
+    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_nosetests.sh :
         timeout: 2600
   post:
     - bash docker/circleci/teardown.sh

doc/Makefile

@@ -35,9 +35,9 @@ htmlonly:
 
 api:
 	rm -rf api/generated
-	python ../tools/build_modref_templates.py
+	python -u ../tools/build_modref_templates.py
 	rm -rf interfaces/generated
-	python ../tools/build_interface_docs.py
+	python -u ../tools/build_interface_docs.py
 	@echo "Build API docs finished."
 
 html: clean examples2rst api htmlonly
@@ -76,7 +76,7 @@ doctest:
 	      "results in _build/doctest/output.txt."
 
 gitwash-update:
-	python ../tools/gitwash_dumper.py devel nipype \
+	python -u ../tools/gitwash_dumper.py devel nipype \
             --repo-name=nipype \
             --github-user=nipy \
             --project-url=http://nipy.org/nipype \

docker/circleci/run_builddocs.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
-for i in /etc/profile.d/*.sh; do
-    source $i
-done
-source activate nipypetests-2.7
+set -e
+set -x
+set -u
 
 mkdir -p /scratch/docs
-set -o pipefail && cd /root/src/nipype/doc && make html 2>&1 | tee /scratch/builddocs.log
+make html 2>&1 | tee /scratch/builddocs.log
 cp -r /root/src/nipype/doc/_build/html/* /scratch/docs/
 chmod 777 -R /scratch/docs
 chmod 777 /scratch/builddocs.log

docker/circleci/run_examples.sh

@@ -1,13 +1,15 @@
 #!/bin/bash
-for i in /etc/profile.d/*.sh; do
-    source $i
-done
+set -e
+set -x
+set -u
 
 mkdir -p /root/.nipype
+mkdir -p /scratch/logs
 echo '[logging]' > /root/.nipype/nipype.cfg
 echo 'workflow_level = DEBUG' >> /root/.nipype/nipype.cfg
 echo 'interface_level = DEBUG' >> /root/.nipype/nipype.cfg
 echo 'filemanip_level = DEBUG' >> /root/.nipype/nipype.cfg
+echo 'log_to_file = true' >> /root/.nipype/nipype.cfg
+echo 'log_directory = /scratch/logs/' >> /root/.nipype/nipype.cfg
 
-source activate nipypetests-2.7
 python /root/src/nipype/tools/run_examples.py $@

docker/circleci/run_nosetests.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
-for i in /etc/profile.d/*.sh; do
-    source $i
-done
-source activate nipypetests-2.7
+set -e
+set -x
+set -u
+
 cd /root/src/nipype
 mkdir -p /scratch/nose
 nosetests -c /root/src/nipype/.noserc --xunit-file="/scratch/nosetests.xml" --cover-xml-file="/scratch/coverage.xml"

docker/circleci/teardown.sh

@@ -13,4 +13,4 @@ sudo mv ~/scratch/coverage.xml ~/coverage.xml
 mkdir -p ~/docs
 sudo mv ~/scratch/docs/* ~/docs/
 mkdir -p ~/logs
-sudo mv ~/scratch/logs/* ~/logs/
+sudo mv $(pwd)/scratch/logs/* ~/logs/

docker/nipype_test_base/Dockerfile

@@ -0,0 +1,83 @@
+# Copyright (c) 2016, The developers of nipype
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of crn_base nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM neurodebian:latest
+MAINTAINER Nipype developers
+
+# Preparations
+RUN ln -snf /bin/bash /bin/sh
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN sed -i -e 's,main$,main contrib non-free,g' /etc/apt/sources.list.d/neurodebian.sources.list && \
+    apt-get -y update && \
+    apt-get install -y curl \
+                       git \
+                       xvfb \
+                       bzip2 \
+                       unzip \
+                       apt-utils \
+                       fusefat \
+                       graphviz \
+                       make \
+                       fsl-core && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    echo ". /etc/fsl/fsl.sh" >> /etc/bash.bashrc
+
+ENV FSLDIR=/usr/share/fsl/5.0
+ENV FSLOUTPUTTYPE=NIFTI_GZ
+ENV PATH=/usr/lib/fsl/5.0:$PATH
+ENV FSLMULTIFILEQUIT=TRUE
+ENV POSSUMDIR=/usr/share/fsl/5.0
+ENV LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH
+ENV FSLTCLSH=/usr/bin/tclsh
+ENV FSLWISH=/usr/bin/wish
+
+# Install Matlab: from the good old install_spm_mcr.sh of @chrisfilo
+WORKDIR /opt
+
+RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \
+    echo "agreeToLicense=yes" >> mcr_options.txt && \
+    echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \
+    echo "mode=silent" >> mcr_options.txt && \
+    mkdir -p matlab_installer && \
+    curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \
+         -o matlab_installer/installer.zip && \
+    unzip matlab_installer/installer.zip -d matlab_installer/ && \
+    matlab_installer/install -inputFile mcr_options.txt && \
+    rm -rf matlab_installer mcr_options.txt
+
+ENV SPMMCRCMD "/opt/spm12/run_spm12.sh /opt/mcr/v85/ script"
+ENV FORCE_SPMMCR 1
+
+# Install SPM
+RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \
+    unzip spm12.zip && \
+    rm -rf spm12.zip
+
+CMD ["/bin/bash"]
+

docker/nipype_test_py27/Dockerfile

@@ -0,0 +1,63 @@
+# Copyright (c) 2016, The developers of nipype
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of crn_base nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM nipype/nipype_test:base
+MAINTAINER The nipype developers https://github.com/nipy/nipype
+# Preparations
+RUN ln -snf /bin/bash /bin/sh
+WORKDIR /root
+
+# Install miniconda
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh && \
+    /bin/bash Miniconda-latest-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda-latest-Linux-x86_64.sh
+
+ENV PATH /usr/local/miniconda/bin:$PATH
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+# Add conda-forge channel in conda
+RUN conda config --add channels conda-forge && \
+    conda install -y lockfile nipype matplotlib sphinx boto coverage
+
+RUN mkdir -p /root/src/nipype
+COPY . /root/src/nipype
+
+RUN cd /root/src/nipype && \
+    pip install dipy && \
+    pip install -r requirements.txt && \
+    pip install -e .
+
+
+COPY docker/circleci/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+ENV SHELL /bin/bash
+
+CMD ["/bin/bash"]

docker/nipype_test_py34/Dockerfile

@@ -0,0 +1,64 @@
+# Copyright (c) 2016, The developers of nipype
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of crn_base nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM nipype/nipype_test:base
+MAINTAINER The nipype developers https://github.com/nipy/nipype
+# Preparations
+RUN ln -snf /bin/bash /bin/sh
+WORKDIR /root
+
+# Install miniconda
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-latest-Linux-x86_64.sh
+
+ENV PATH /usr/local/miniconda/bin:$PATH
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+# Add conda-forge channel in conda
+RUN conda config --add channels conda-forge && \
+    conda update -y conda && \
+    conda update --all -y python=3.4 && \
+    conda install -y lockfile nipype
+
+RUN mkdir -p /root/src/nipype
+COPY . /root/src/nipype
+
+RUN cd /root/src/nipype && \
+    pip install -r requirements.txt && \
+    pip install -e .
+
+
+COPY docker/circleci/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+ENV SHELL /bin/bash
+
+CMD ["/bin/bash"]