From da3b7256edf3dbf7da74ee3bf652262fe0424d60 Mon Sep 17 00:00:00 2001 From: mwaskom Date: Mon, 5 Aug 2013 23:51:35 -0700 Subject: [PATCH 01/20] Adding code initial code for new data grabber --- nipype/interfaces/io.py | 115 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 2e1be6e01e..466af598f5 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -19,6 +19,7 @@ """ import glob import os +import os.path as op import shutil import re import tempfile @@ -551,6 +552,120 @@ def _list_outputs(self): outputs[key] = outputs[key][0] return outputs + +class DataGrabber2InputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): + + base_directory = Directory(exists=True, + desc="Root path common to templates.") + templates = traits.Dict(mandatory=True, + key_traits=traits.Str, + value_traits=traits.Str, + desc="String formatting templates with {} syntax that form "\ + "output file names given the values of the interface inputs.") + sort_filelist = traits.Bool(True, usedefault=True, + desc="When matching mutliple files, return them in sorted order.") + raise_on_empty = traits.Bool(True, usedefault=True, + desc="Raise an exception if a template pattern matches no files.") + force_lists = traits.Bool(False, usedefault=True, + desc="Return all values as lists even when matching a single file.") + + +class DataGrabber2(IOBase): + """Flexibly collect data from disk to feed into workflows. + + This interface uses the {}-based string formatting syntax to plug + values (possibly known only at workflow execution time) into string + templates and collect files from persistant storage. These templates + can also be combined with glob wildcards. The names used in the + formatting template should correspond to the input fields given + when you instatiate the interfacee, and the outputs are formed by the + keys in the `templates` dictionary. + + Examples + -------- + + >>> from nipype import DataGrabber2 + >>> import os + >>> dg = DataGrabber2(infields=["subject_id"], + ... base_directory=os.environ["SUBJECTS_DIR"]) + ... templates={"T1": "{subject_id}/mri/T1.mgz", + ... "aseg": "{subject_id}/mri/aseg.mgz") + >>> print dg._outputs() + + T1 = + aseg = + + """ + input_spec = DataGrabber2InputSpec + output_spec = DynamicTraitedSpec + _always_run = True + + def __init__(self, infields, **kwargs): + """Create an instance with specific input fields. + + Parameters + ---------- + infields : list of strings + Names of dynamic input fields to add to the interface. These + inputs can be used as names in the format templates to build + output filenames. + + """ + super(DataGrabber2, self).__init__(**kwargs) + self._infields = infields + self._outfields = list(self.inputs.templates) + + # Manually handle the mandatory inputs + if not isdefined(self.inputs.templates): + raise ValueError("'templates' is a mandatory input.") + + # Add the dynamic input fields + undefined_traits = {} + for field in infields: + self.inputs.add_trait(field, traits.Any) + undefined_traits[field] = Undefined + self.inputs.trait_set(trait_change_notify=False, **undefined_traits) + + def _list_outputs(self): + """Find the files and expose them as interface outputs.""" + outputs = {} + info = {k: v for k, v in self.inputs.__dict__.items() + if k in self._infields} + + for field, template in self.inputs.templates.items(): + + # Build the full template path + if isdefined(self.inputs.base_directory): + template = op.abspath(op.join( + self.inputs.base_directory, template)) + else: + template = op.abspath(template) + + # Fill in the template and glob for files + filled_template = template.format(**info) + filelist = glob.glob(filled_template) + + # Handle the case where nothing matched + if not filelist: + msg = "No files were found matching %s template" % field + if self.inputs.raise_on_empty: + raise IOError(msg) + else: + warn(msg) + + # Possibly sort the list + if self.inputs.sort_filelist: + filelist.sort() + + # Handle whether this must be a list or not + if not self.inputs.force_lists: + filelist = list_to_filename(filelist) + + outputs[field] = filelist + + return outputs + + class DataFinderInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): root_paths = traits.Either(traits.List(), traits.Str(), From d985f67766324150b26495f72db591c7458f9549 Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 00:00:30 -0700 Subject: [PATCH 02/20] PEP8 --- nipype/interfaces/io.py | 63 +++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 466af598f5..6388778194 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -529,9 +529,9 @@ def _list_outputs(self): filledtemplate = template if argtuple: try: - filledtemplate = template%tuple(argtuple) + filledtemplate = template % tuple(argtuple) except TypeError as e: - raise TypeError(e.message + ": Template %s failed to convert with args %s"%(template, str(tuple(argtuple)))) + raise TypeError(e.message + ": Template %s failed to convert with args %s" % (template, str(tuple(argtuple)))) outfiles = glob.glob(filledtemplate) if len(outfiles) == 0: msg = 'Output key: %s Template: %s returned no files' % (key, filledtemplate) @@ -666,18 +666,18 @@ def _list_outputs(self): return outputs -class DataFinderInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): +class DataFinderInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): root_paths = traits.Either(traits.List(), traits.Str(), mandatory=True,) - match_regex = traits.Str('(.+)', + match_regex = traits.Str('(.+)', usedefault=True, desc=("Regular expression for matching " "paths.")) ignore_regexes = traits.List(desc=("List of regular expressions, " "if any match the path it will be " "ignored.") - ) + ) max_depth = traits.Int(desc="The maximum depth to search beneath " "the root_paths") min_depth = traits.Int(desc="The minimum depth to search beneath " @@ -688,23 +688,19 @@ class DataFinderInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): class DataFinder(IOBase): - """Search for paths that match a given regular expression. Allows a less + """Search for paths that match a given regular expression. Allows a less proscriptive approach to gathering input files compared to DataGrabber. - Will recursively search any subdirectories by default. This can be limited - with the min/max depth options. - - Matched paths are available in the output 'out_paths'. Any named groups of - captured text from the regular expression are also available as ouputs of + Will recursively search any subdirectories by default. This can be limited + with the min/max depth options. + Matched paths are available in the output 'out_paths'. Any named groups of + captured text from the regular expression are also available as ouputs of the same name. - Examples -------- >>> from nipype.interfaces.io import DataFinder - - Look for Nifti files in directories with "ep2d_fid" or "qT1" in the name, + Look for Nifti files in directories with "ep2d_fid" or "qT1" in the name, starting in the current directory. - >>> df = DataFinder() >>> df.inputs.root_paths = '.' >>> df.inputs.match_regex = '.+/(?P.+(qT1|ep2d_fid_T1).+)/(?P.+)\.nii.gz' @@ -714,13 +710,11 @@ class DataFinder(IOBase): './018-ep2d_fid_T1_Gd2/acquisition.nii.gz', './016-ep2d_fid_T1_Gd1/acquisition.nii.gz', './013-ep2d_fid_T1_pre/acquisition.nii.gz'] - >>> print result.outputs.series_dir # doctest: +SKIP ['027-ep2d_fid_T1_Gd4', '018-ep2d_fid_T1_Gd2', '016-ep2d_fid_T1_Gd1', '013-ep2d_fid_T1_pre'] - >>> print result.outputs.basename # doctest: +SKIP ['acquisition', 'acquisition', @@ -728,31 +722,27 @@ class DataFinder(IOBase): 'acquisition'] """ - input_spec = DataFinderInputSpec output_spec = DynamicTraitedSpec _always_run = True - + def _match_path(self, target_path): #Check if we should ignore the path for ignore_re in self.ignore_regexes: if ignore_re.search(target_path): return - #Check if we can match the path match = self.match_regex.search(target_path) if not match is None: match_dict = match.groupdict() - if self.result is None: - self.result = {'out_paths' : []} + self.result = {'out_paths': []} for key in match_dict.keys(): self.result[key] = [] - self.result['out_paths'].append(target_path) for key, val in match_dict.iteritems(): self.result[key].append(val) - + def _run_interface(self, runtime): #Prepare some of the inputs if isinstance(self.inputs.root_paths, str): @@ -770,33 +760,27 @@ def _run_interface(self, runtime): self.ignore_regexes = [] else: self.ignore_regexes = \ - [re.compile(regex) + [re.compile(regex) for regex in self.inputs.ignore_regexes] - self.result = None for root_path in self.inputs.root_paths: #Handle tilda/env variables and remove extra seperators root_path = os.path.normpath(os.path.expandvars(os.path.expanduser(root_path))) - #Check if the root_path is a file if os.path.isfile(root_path): if min_depth == 0: self._match_path(root_path) continue - - #Walk through directory structure checking paths + #Walk through directory structure checking paths for curr_dir, sub_dirs, files in os.walk(root_path): #Determine the current depth from the root_path - curr_depth = (curr_dir.count(os.sep) - + curr_depth = (curr_dir.count(os.sep) - root_path.count(os.sep)) - - #If the max path depth has been reached, clear sub_dirs + #If the max path depth has been reached, clear sub_dirs #and files - if (not max_depth is None and - curr_depth >= max_depth): + if not max_depth is not None and curr_depth >= max_depth: sub_dirs[:] = [] files = [] - #Test the path for the curr_dir and all files if curr_depth >= min_depth: self._match_path(curr_dir) @@ -804,22 +788,21 @@ def _run_interface(self, runtime): for infile in files: full_path = os.path.join(curr_dir, infile) self._match_path(full_path) - - if (self.inputs.unpack_single and + if (self.inputs.unpack_single and len(self.result['out_paths']) == 1 - ): + ): for key, vals in self.result.iteritems(): self.result[key] = vals[0] - if not self.result: raise RuntimeError("Regular expression did not match any files!") return runtime - + def _list_outputs(self): outputs = self._outputs().get() outputs.update(self.result) return outputs + class FSSourceInputSpec(BaseInterfaceInputSpec): subjects_dir = Directory(mandatory=True, desc='Freesurfer subjects directory.') From c5f6fcd25ab1b5d0e521cbbb33bb095e34d5cf35 Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 00:11:04 -0700 Subject: [PATCH 03/20] Addding tests for new data grabber --- nipype/interfaces/io.py | 3 ++- nipype/interfaces/tests/test_io.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 6388778194..06c60d10ff 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -647,7 +647,8 @@ def _list_outputs(self): # Handle the case where nothing matched if not filelist: - msg = "No files were found matching %s template" % field + msg = "No files were found matching %s template: %s" % ( + field, template) if self.inputs.raise_on_empty: raise IOError(msg) else: diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 5246f2e299..5759f9a792 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -3,8 +3,10 @@ import os import glob import shutil +import os.path as op from tempfile import mkstemp, mkdtemp +import nipype from nipype.testing import assert_equal, assert_true, assert_false import nipype.interfaces.io as nio from nipype.interfaces.base import Undefined @@ -15,6 +17,26 @@ def test_datagrabber(): yield assert_equal, dg.inputs.base_directory, Undefined yield assert_equal, dg.inputs.template_args,{'outfiles': []} + +def test_datagrabber2(): + base_dir = op.dirname(nipype.__file__) + templates = {"model": "interfaces/{package}/model.py"} + dg = nio.DataGrabber2(["package"], templates=templates, + base_directory=base_dir) + dg.inputs.package = "fsl" + res = dg.run() + wanted = op.join(op.dirname(nipype.__file__), "interfaces/fsl/model.py") + yield assert_equal, res.outputs.model, wanted + + dg = nio.DataGrabber2(["package"], templates=templates, + base_directory=base_dir, + force_lists=True) + dg.inputs.package = "spm" + res = dg.run() + wanted = op.join(op.dirname(nipype.__file__), "interfaces/spm/model.py") + yield assert_equal, res.outputs.model, [wanted] + + def test_datasink(): ds = nio.DataSink() yield assert_true, ds.inputs.parameterization @@ -26,6 +48,7 @@ def test_datasink(): ds = nio.DataSink(infields=['test']) yield assert_true, 'test' in ds.inputs.copyable_trait_names() + def test_datasink_substitutions(): indir = mkdtemp(prefix='-Tmp-nipype_ds_subs_in') outdir = mkdtemp(prefix='-Tmp-nipype_ds_subs_out') From 5fea91def14e692d2e74e66aec5837f50e30d37f Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 00:25:47 -0700 Subject: [PATCH 04/20] Addding tests for new data grabber --- nipype/interfaces/tests/test_io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 5759f9a792..5d71642a3c 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -20,7 +20,8 @@ def test_datagrabber(): def test_datagrabber2(): base_dir = op.dirname(nipype.__file__) - templates = {"model": "interfaces/{package}/model.py"} + templates = {"model": "interfaces/{package}/model.py", + "preprocess": "interfaces/{package}/preprocess.py"} dg = nio.DataGrabber2(["package"], templates=templates, base_directory=base_dir) dg.inputs.package = "fsl" From f32be886c9a075b19bc5c9f853d5d29f741c2209 Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 00:28:35 -0700 Subject: [PATCH 05/20] Fix typo --- nipype/interfaces/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 06c60d10ff..f06fe24964 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -578,7 +578,7 @@ class DataGrabber2(IOBase): templates and collect files from persistant storage. These templates can also be combined with glob wildcards. The names used in the formatting template should correspond to the input fields given - when you instatiate the interfacee, and the outputs are formed by the + when you instatiate the interface, and the outputs are formed by the keys in the `templates` dictionary. Examples From 537a7e2ff3edb238e850202e24901878396e909d Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 00:28:44 -0700 Subject: [PATCH 06/20] Improving test a bit --- nipype/interfaces/tests/test_io.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 5d71642a3c..a9c17438a4 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -21,7 +21,7 @@ def test_datagrabber(): def test_datagrabber2(): base_dir = op.dirname(nipype.__file__) templates = {"model": "interfaces/{package}/model.py", - "preprocess": "interfaces/{package}/preprocess.py"} + "preprocess": "interfaces/{package}/pre*.py"} dg = nio.DataGrabber2(["package"], templates=templates, base_directory=base_dir) dg.inputs.package = "fsl" @@ -34,8 +34,9 @@ def test_datagrabber2(): force_lists=True) dg.inputs.package = "spm" res = dg.run() - wanted = op.join(op.dirname(nipype.__file__), "interfaces/spm/model.py") - yield assert_equal, res.outputs.model, [wanted] + wanted = op.join(op.dirname(nipype.__file__), + "interfaces/spm/preprocess.py") + yield assert_equal, res.outputs.preprocess, [wanted] def test_datasink(): From 010f121574922d5e137f77bcae6ea4edae138e58 Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 10:27:26 -0700 Subject: [PATCH 07/20] Updating CHANGES --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index e1e7355106..9101fdc0a4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ Next release ============ +* ENH: DataGrabber2: a streamlined version of DataGrabber * ENH: New interfaces: spm.ResliceToReference * FIX: Deals properly with 3d files in SPM Realign From 90e85360e34aa2d8026c3f2d0d02efb12ce677be Mon Sep 17 00:00:00 2001 From: mwaskom Date: Tue, 6 Aug 2013 23:55:31 -0700 Subject: [PATCH 08/20] Fix typo --- nipype/interfaces/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index f06fe24964..d6a845dae5 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -578,7 +578,7 @@ class DataGrabber2(IOBase): templates and collect files from persistant storage. These templates can also be combined with glob wildcards. The names used in the formatting template should correspond to the input fields given - when you instatiate the interface, and the outputs are formed by the + when you instantiate the interface, and the outputs are formed by the keys in the `templates` dictionary. Examples From 049139d2ac837d713d67f1a5f66c437429e01d4a Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 15:41:44 -0700 Subject: [PATCH 09/20] Changing name to SelectFiles --- nipype/interfaces/io.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index d6a845dae5..5ed9674465 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -553,7 +553,7 @@ def _list_outputs(self): return outputs -class DataGrabber2InputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): +class SelectFilesInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): base_directory = Directory(exists=True, desc="Root path common to templates.") @@ -570,7 +570,7 @@ class DataGrabber2InputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): desc="Return all values as lists even when matching a single file.") -class DataGrabber2(IOBase): +class SelectFiles(IOBase): """Flexibly collect data from disk to feed into workflows. This interface uses the {}-based string formatting syntax to plug @@ -584,19 +584,19 @@ class DataGrabber2(IOBase): Examples -------- - >>> from nipype import DataGrabber2 + >>> from nipype import SelectFiles >>> import os - >>> dg = DataGrabber2(infields=["subject_id"], - ... base_directory=os.environ["SUBJECTS_DIR"]) - ... templates={"T1": "{subject_id}/mri/T1.mgz", - ... "aseg": "{subject_id}/mri/aseg.mgz") + >>> dg = SelectFiles(infields=["subject_id"], + ... base_directory=os.environ["SUBJECTS_DIR"]) + ... templates={"T1": "{subject_id}/mri/T1.mgz", + ... "aseg": "{subject_id}/mri/aseg.mgz") >>> print dg._outputs() T1 = aseg = """ - input_spec = DataGrabber2InputSpec + input_spec = SelectFilesInputSpec output_spec = DynamicTraitedSpec _always_run = True @@ -611,7 +611,7 @@ def __init__(self, infields, **kwargs): output filenames. """ - super(DataGrabber2, self).__init__(**kwargs) + super(SelectFiles, self).__init__(**kwargs) self._infields = infields self._outfields = list(self.inputs.templates) From 5e38643635fb81edf68de28d352ae3219bae3ee2 Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 15:41:57 -0700 Subject: [PATCH 10/20] Importing SelectFiles up the hierarchy --- nipype/__init__.py | 2 +- nipype/interfaces/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nipype/__init__.py b/nipype/__init__.py index 20a05c291b..c9f9684f2a 100644 --- a/nipype/__init__.py +++ b/nipype/__init__.py @@ -17,7 +17,7 @@ from pipeline import Node, MapNode, Workflow from interfaces import (fsl, spm, freesurfer, afni, ants, slicer, dipy, nipy, - mrtrix, camino, DataGrabber, DataSink, + mrtrix, camino, DataGrabber, DataSink, SelectFiles, IdentityInterface, Rename, Function, Select, Merge) diff --git a/nipype/interfaces/__init__.py b/nipype/interfaces/__init__.py index 56d6e8f376..b81deb9d70 100644 --- a/nipype/interfaces/__init__.py +++ b/nipype/interfaces/__init__.py @@ -7,6 +7,6 @@ """ __docformat__ = 'restructuredtext' -from io import DataGrabber, DataSink +from io import DataGrabber, DataSink, SelectFiles from utility import IdentityInterface, Rename, Function, Select, Merge import fsl, spm, freesurfer, afni, ants, slicer, dipy, nipy, mrtrix, camino From 632fd41bd33eac76340de6f91ff7eb8979113d07 Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 15:46:55 -0700 Subject: [PATCH 11/20] Updating tests and changes with new name --- CHANGES | 2 +- nipype/interfaces/io.py | 9 +++++++-- nipype/interfaces/tests/test_io.py | 12 ++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGES b/CHANGES index 9101fdc0a4..35ea60f708 100644 --- a/CHANGES +++ b/CHANGES @@ -1,7 +1,7 @@ Next release ============ -* ENH: DataGrabber2: a streamlined version of DataGrabber +* ENH: SelectFiles: a streamlined version of DataGrabber * ENH: New interfaces: spm.ResliceToReference * FIX: Deals properly with 3d files in SPM Realign diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 5ed9674465..565b5a7cbd 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -609,16 +609,21 @@ def __init__(self, infields, **kwargs): Names of dynamic input fields to add to the interface. These inputs can be used as names in the format templates to build output filenames. + templates : dictionary + Dictionary mappying string fields to string tempalate values. + See SelectFiles.help() for more information. """ super(SelectFiles, self).__init__(**kwargs) - self._infields = infields - self._outfields = list(self.inputs.templates) # Manually handle the mandatory inputs if not isdefined(self.inputs.templates): raise ValueError("'templates' is a mandatory input.") + self._infields = infields + self._outfields = list(self.inputs.templates) + for + # Add the dynamic input fields undefined_traits = {} for field in infields: diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index a9c17438a4..e80be9e24c 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -18,20 +18,20 @@ def test_datagrabber(): yield assert_equal, dg.inputs.template_args,{'outfiles': []} -def test_datagrabber2(): +def test_selectfiles(): base_dir = op.dirname(nipype.__file__) templates = {"model": "interfaces/{package}/model.py", "preprocess": "interfaces/{package}/pre*.py"} - dg = nio.DataGrabber2(["package"], templates=templates, - base_directory=base_dir) + dg = nio.SelectFiles(["package"], templates=templates, + base_directory=base_dir) dg.inputs.package = "fsl" res = dg.run() wanted = op.join(op.dirname(nipype.__file__), "interfaces/fsl/model.py") yield assert_equal, res.outputs.model, wanted - dg = nio.DataGrabber2(["package"], templates=templates, - base_directory=base_dir, - force_lists=True) + dg = nio.SelectFiles(["package"], templates=templates, + base_directory=base_dir, + force_lists=True) dg.inputs.package = "spm" res = dg.run() wanted = op.join(op.dirname(nipype.__file__), From e85b023898320f99a7e2d4fab9a463dc1da4e5fa Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 15:49:43 -0700 Subject: [PATCH 12/20] Removing errant for statement --- nipype/interfaces/io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 565b5a7cbd..b5d7a763ce 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -622,7 +622,6 @@ def __init__(self, infields, **kwargs): self._infields = infields self._outfields = list(self.inputs.templates) - for # Add the dynamic input fields undefined_traits = {} From cefcb7f85dda3b3c79aced858468963b60868c3e Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 16:04:31 -0700 Subject: [PATCH 13/20] Another SelectFiles test --- nipype/interfaces/tests/test_io.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index e80be9e24c..1d6ffdc74a 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -32,6 +32,8 @@ def test_selectfiles(): dg = nio.SelectFiles(["package"], templates=templates, base_directory=base_dir, force_lists=True) + yield assert_equal, set(dg._outputs().get().keys()), {"model", "preprocess"} + dg.inputs.package = "spm" res = dg.run() wanted = op.join(op.dirname(nipype.__file__), From 94077ceafa7a5e8954a0823776d09e6ee3e56d77 Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 16:04:51 -0700 Subject: [PATCH 14/20] Adding _add_output_traits in SelectFiles --- nipype/interfaces/io.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index b5d7a763ce..ba3c2f1c9d 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -630,6 +630,10 @@ def __init__(self, infields, **kwargs): undefined_traits[field] = Undefined self.inputs.trait_set(trait_change_notify=False, **undefined_traits) + def _add_output_traits(self, base): + """Add the dynamic output fields""" + return add_traits(base, self.inputs.templates.keys()) + def _list_outputs(self): """Find the files and expose them as interface outputs.""" outputs = {} From 4b29046a14432cf0ae49c7bdab7cccfbf3ba2c0f Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Wed, 7 Aug 2013 19:58:17 -0700 Subject: [PATCH 15/20] Changing SelectFiles doctest --- nipype/interfaces/io.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index ba3c2f1c9d..25a3376d0c 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -584,16 +584,12 @@ class SelectFiles(IOBase): Examples -------- - >>> from nipype import SelectFiles - >>> import os + >>> from nipype import SelectFiles, Node >>> dg = SelectFiles(infields=["subject_id"], - ... base_directory=os.environ["SUBJECTS_DIR"]) - ... templates={"T1": "{subject_id}/mri/T1.mgz", - ... "aseg": "{subject_id}/mri/aseg.mgz") - >>> print dg._outputs() - - T1 = - aseg = + ... templates={"T1": "{subject_id}/struct/T1.nii", + ... "epi": "{subject_id}/func/epi.nii"}) + >>> Node(dg, "selectfiles").outputs.get() + {'T1': , 'epi': } """ input_spec = SelectFilesInputSpec From ef0aba74c0d3f2d9a9224594b0207d57ea6ae2c2 Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Fri, 9 Aug 2013 11:44:33 -0700 Subject: [PATCH 16/20] Update SelectFiles to infer infields from templates --- nipype/interfaces/io.py | 49 +++++++++++++++--------------- nipype/interfaces/tests/test_io.py | 10 +++--- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 25a3376d0c..e5d7b5d882 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -18,6 +18,7 @@ """ import glob +import string import os import os.path as op import shutil @@ -557,11 +558,6 @@ class SelectFilesInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): base_directory = Directory(exists=True, desc="Root path common to templates.") - templates = traits.Dict(mandatory=True, - key_traits=traits.Str, - value_traits=traits.Str, - desc="String formatting templates with {} syntax that form "\ - "output file names given the values of the interface inputs.") sort_filelist = traits.Bool(True, usedefault=True, desc="When matching mutliple files, return them in sorted order.") raise_on_empty = traits.Bool(True, usedefault=True, @@ -585,10 +581,11 @@ class SelectFiles(IOBase): -------- >>> from nipype import SelectFiles, Node - >>> dg = SelectFiles(infields=["subject_id"], - ... templates={"T1": "{subject_id}/struct/T1.nii", - ... "epi": "{subject_id}/func/epi.nii"}) - >>> Node(dg, "selectfiles").outputs.get() + >>> templates={"T1": "{subject_id}/struct/T1.nii", + ... "epi": "{subject_id}/func/f[0,1].nii"} + >>> dg = Node(SelectFiles(templates), "selectfiles") + >>> dg.inputs.subject_id = "subj1" + >>> dg.outputs.get() {'T1': , 'epi': } """ @@ -596,28 +593,32 @@ class SelectFiles(IOBase): output_spec = DynamicTraitedSpec _always_run = True - def __init__(self, infields, **kwargs): + def __init__(self, templates, **kwargs): """Create an instance with specific input fields. Parameters ---------- - infields : list of strings - Names of dynamic input fields to add to the interface. These - inputs can be used as names in the format templates to build - output filenames. templates : dictionary - Dictionary mappying string fields to string tempalate values. - See SelectFiles.help() for more information. + Mapping string keys to string tempalate values. + The keys become output fields on the interface. + The templates should use {}-formatting syntax, where + the keys in curly braces become inputs fields on the interface. + Format strings can also use glob wildcards to match multiple + files. """ super(SelectFiles, self).__init__(**kwargs) - # Manually handle the mandatory inputs - if not isdefined(self.inputs.templates): - raise ValueError("'templates' is a mandatory input.") + # Infer the infields and outfields from the template + infields = [] + for name, template in templates.iteritems(): + for _, field_name, _, _ in string.Formatter().parse(template): + if field_name is not None and field_name not in infields: + infields.append(field_name) self._infields = infields - self._outfields = list(self.inputs.templates) + self._outfields = list(templates) + self._templates = templates # Add the dynamic input fields undefined_traits = {} @@ -628,15 +629,15 @@ def __init__(self, infields, **kwargs): def _add_output_traits(self, base): """Add the dynamic output fields""" - return add_traits(base, self.inputs.templates.keys()) + return add_traits(base, self._templates.keys()) def _list_outputs(self): """Find the files and expose them as interface outputs.""" outputs = {} - info = {k: v for k, v in self.inputs.__dict__.items() - if k in self._infields} + info = dict([(k, v) for k, v in self.inputs.__dict__.items() + if k in self._infields]) - for field, template in self.inputs.templates.items(): + for field, template in self._templates.iteritems(): # Build the full template path if isdefined(self.inputs.base_directory): diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 1d6ffdc74a..8f2fdb9c7c 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -22,17 +22,19 @@ def test_selectfiles(): base_dir = op.dirname(nipype.__file__) templates = {"model": "interfaces/{package}/model.py", "preprocess": "interfaces/{package}/pre*.py"} - dg = nio.SelectFiles(["package"], templates=templates, - base_directory=base_dir) + dg = nio.SelectFiles(templates, base_directory=base_dir) + yield assert_equal, dg._infields, ["package"] + yield assert_equal, sorted(dg._outfields), ["model", "preprocess"] dg.inputs.package = "fsl" res = dg.run() wanted = op.join(op.dirname(nipype.__file__), "interfaces/fsl/model.py") yield assert_equal, res.outputs.model, wanted - dg = nio.SelectFiles(["package"], templates=templates, + dg = nio.SelectFiles(templates, base_directory=base_dir, force_lists=True) - yield assert_equal, set(dg._outputs().get().keys()), {"model", "preprocess"} + outfields = sorted(dg._outputs().get()) + yield assert_equal, outfields, ["model", "preprocess"] dg.inputs.package = "spm" res = dg.run() From 748f8c7c61757ff5211d8acfa6828a864f443eb9 Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Fri, 9 Aug 2013 12:59:25 -0700 Subject: [PATCH 17/20] Adding example of string converter in SelectFiles --- nipype/interfaces/io.py | 7 +++++++ nipype/interfaces/tests/test_io.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index e5d7b5d882..5edef75691 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -588,6 +588,13 @@ class SelectFiles(IOBase): >>> dg.outputs.get() {'T1': , 'epi': } + The same thing with dynamic grabbing of specific files: + + >>> templates["epi"] = "{subject_id}/func/f{run!s}.nii" + >>> dg = Node(SelectFiles(templates), "selectfiles") + >>> dg.inputs.subject_id = "subj1" + >>> dg.inputs.run = [2, 4] + """ input_spec = SelectFilesInputSpec output_spec = DynamicTraitedSpec diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 8f2fdb9c7c..d0e65e5789 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -42,6 +42,13 @@ def test_selectfiles(): "interfaces/spm/preprocess.py") yield assert_equal, res.outputs.preprocess, [wanted] + templates = {"converter": "interfaces/dcm{to!s}nii.py"} + dg = nio.SelectFiles(templates, base_directory=base_dir) + dg.inputs.to = 2 + res = dg.run() + wanted = op.join(base_dir, "interfaces/dcm2nii.py") + yield assert_equal, res.outputs.converter, wanted + def test_datasink(): ds = nio.DataSink() From 70a2a06c09c56b91e5b59016fd5fc80afa4a661b Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Fri, 9 Aug 2013 13:01:25 -0700 Subject: [PATCH 18/20] PEP8 --- nipype/interfaces/tests/test_io.py | 33 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index d0e65e5789..f71b896932 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -11,11 +11,12 @@ import nipype.interfaces.io as nio from nipype.interfaces.base import Undefined + def test_datagrabber(): dg = nio.DataGrabber() yield assert_equal, dg.inputs.template, Undefined yield assert_equal, dg.inputs.base_directory, Undefined - yield assert_equal, dg.inputs.template_args,{'outfiles': []} + yield assert_equal, dg.inputs.template_args, {'outfiles': []} def test_selectfiles(): @@ -56,7 +57,7 @@ def test_datasink(): yield assert_equal, ds.inputs.base_directory, Undefined yield assert_equal, ds.inputs.strip_dir, Undefined yield assert_equal, ds.inputs._outputs, {} - ds = nio.DataSink(base_directory = 'foo') + ds = nio.DataSink(base_directory='foo') yield assert_equal, ds.inputs.base_directory, 'foo' ds = nio.DataSink(infields=['test']) yield assert_true, 'test' in ds.inputs.copyable_trait_names() @@ -72,52 +73,58 @@ def test_datasink_substitutions(): open(f, 'w') ds = nio.DataSink( parametrization=False, - base_directory = outdir, - substitutions = [('ababab', 'ABABAB')], + base_directory=outdir, + substitutions=[('ababab', 'ABABAB')], # end archoring ($) is used to assure operation on the filename # instead of possible temporary directories names matches # Patterns should be more comprehendable in the real-world usage # cases since paths would be quite more sensible - regexp_substitutions = [(r'xABABAB(\w*)\.n$', r'a-\1-b.n'), - ('(.*%s)[-a]([^%s]*)$' % ((os.path.sep,)*2), - r'\1!\2')] ) + regexp_substitutions=[(r'xABABAB(\w*)\.n$', r'a-\1-b.n'), + ('(.*%s)[-a]([^%s]*)$' % ((os.path.sep,) * 2), + r'\1!\2')]) setattr(ds.inputs, '@outdir', files) ds.run() yield assert_equal, \ sorted([os.path.basename(x) for x in glob.glob(os.path.join(outdir, '*'))]), \ - ['!-yz-b.n', 'ABABAB.n'] # so we got re used 2nd and both patterns + ['!-yz-b.n', 'ABABAB.n'] # so we got re used 2nd and both patterns shutil.rmtree(indir) shutil.rmtree(outdir) + def _temp_analyze_files(): """Generate temporary analyze file pair.""" - fd, orig_img = mkstemp(suffix = '.img', dir=mkdtemp()) + fd, orig_img = mkstemp(suffix='.img', dir=mkdtemp()) orig_hdr = orig_img[:-4] + '.hdr' fp = file(orig_hdr, 'w+') fp.close() return orig_img, orig_hdr + def test_datasink_copydir(): orig_img, orig_hdr = _temp_analyze_files() outdir = mkdtemp() pth, fname = os.path.split(orig_img) - ds = nio.DataSink(base_directory = outdir, parameterization=False) - setattr(ds.inputs,'@outdir',pth) + ds = nio.DataSink(base_directory=outdir, parameterization=False) + setattr(ds.inputs, '@outdir', pth) ds.run() - file_exists = lambda: os.path.exists(os.path.join(outdir, pth.split(os.path.sep)[-1], fname)) + sep = os.path.sep + file_exists = lambda: os.path.exists(os.path.join(outdir, + pth.split(sep)[-1], + fname)) yield assert_true, file_exists() shutil.rmtree(pth) orig_img, orig_hdr = _temp_analyze_files() pth, fname = os.path.split(orig_img) ds.inputs.remove_dest_dir = True - setattr(ds.inputs,'outdir',pth) + setattr(ds.inputs, 'outdir', pth) ds.run() yield assert_false, file_exists() shutil.rmtree(outdir) shutil.rmtree(pth) + def test_freesurfersource(): fss = nio.FreeSurferSource() yield assert_equal, fss.inputs.hemi, 'both' From bc6adbe97d288e1ffffbeb888e1824be78f80eed Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Fri, 9 Aug 2013 13:20:10 -0700 Subject: [PATCH 19/20] Tweak SelectFiles docstring up reflect changes --- nipype/interfaces/io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 5edef75691..4ffb445e49 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -572,17 +572,17 @@ class SelectFiles(IOBase): This interface uses the {}-based string formatting syntax to plug values (possibly known only at workflow execution time) into string templates and collect files from persistant storage. These templates - can also be combined with glob wildcards. The names used in the - formatting template should correspond to the input fields given - when you instantiate the interface, and the outputs are formed by the - keys in the `templates` dictionary. + can also be combined with glob wildcards. The field names in the + formatting template (i.e. the terms in braces) will become inputs + fields on the interface, and the keys in the templates dictionary + will form the output fields. Examples -------- >>> from nipype import SelectFiles, Node >>> templates={"T1": "{subject_id}/struct/T1.nii", - ... "epi": "{subject_id}/func/f[0,1].nii"} + ... "epi": "{subject_id}/func/f[0, 1].nii"} >>> dg = Node(SelectFiles(templates), "selectfiles") >>> dg.inputs.subject_id = "subj1" >>> dg.outputs.get() From 89cd205fbfcf3841c2277d6debc3cbfefc8cd682 Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Fri, 9 Aug 2013 13:31:41 -0700 Subject: [PATCH 20/20] Tweak SelectFiles __init__ docstring for clarity --- nipype/interfaces/io.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 4ffb445e49..a319ca9a76 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -606,12 +606,14 @@ def __init__(self, templates, **kwargs): Parameters ---------- templates : dictionary - Mapping string keys to string tempalate values. + Mapping from string keys to string template values. The keys become output fields on the interface. The templates should use {}-formatting syntax, where - the keys in curly braces become inputs fields on the interface. + the names in curly braces become inputs fields on the interface. Format strings can also use glob wildcards to match multiple - files. + files. At runtime, the values of the interface inputs will be + plugged into these templates, and the resulting strings will be + used to select files. """ super(SelectFiles, self).__init__(**kwargs)