From 15440fd21d2cb28823442ad846801d351336c112 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 2 May 2017 19:05:03 -0400 Subject: [PATCH 1/6] fix: refactor compcor to not assign to self.inputs and re-use common functions --- nipype/algorithms/confounds.py | 398 +++++++++--------- nipype/algorithms/tests/test_auto_ACompCor.py | 8 +- nipype/algorithms/tests/test_auto_TCompCor.py | 18 +- nipype/algorithms/tests/test_compcor.py | 13 +- 4 files changed, 232 insertions(+), 205 deletions(-) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 0a74fcd57f..4ceb222482 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -223,11 +223,13 @@ class FramewiseDisplacementInputSpec(BaseInterfaceInputSpec): figsize = traits.Tuple(traits.Float(11.7), traits.Float(2.3), usedefault=True, desc='output figure size') + class FramewiseDisplacementOutputSpec(TraitedSpec): out_file = File(desc='calculated FD per timestep') out_figure = File(desc='output image file') fd_average = traits.Float(desc='average FD') + class FramewiseDisplacement(BaseInterface): """ Calculate the :abbr:`FD (framewise displacement)` as in [Power2012]_. @@ -299,35 +301,41 @@ def _run_interface(self, runtime): def _list_outputs(self): return self._results + class CompCorInputSpec(BaseInterfaceInputSpec): realigned_file = File(exists=True, mandatory=True, - desc='already realigned brain image (4D)') - mask_file = InputMultiPath(File(exists=True, deprecated='0.13', - new_name='mask_files', - desc='One or more mask files that determines ROI (3D)')) - mask_files = InputMultiPath(File(exists=True, - desc='One or more mask files that determines ROI (3D)')) + desc='already realigned brain image (4D)') + mask_files = InputMultiPath(File(exists=True), + desc=('One or more mask files that determines ' + 'ROI (3D)')) merge_method = traits.Enum('union', 'intersect', 'none', xor=['mask_index'], - requires=['mask_files'], - desc='Merge method if multiple masks are present - `union` aggregates ' - 'all masks, `intersect` computes the truth value of all masks, `none` ' - 'performs CompCor on each mask individually') - mask_index = traits.Range(0, xor=['merge_method'], requires=['mask_files'], - desc='Position of mask in `mask_files` to use - first is the default') + requires=['mask_files'], + desc=('Merge method if multiple masks are ' + 'present - `union` aggregates all masks, ' + '`intersect` computes the truth value of ' + 'all masks, `none` performs CompCor on ' + 'each mask individually')) + mask_index = traits.Range(low=0, xor=['merge_method'], + requires=['mask_files'], + desc=('Position of mask in `mask_files` to use - ' + 'first is the default')) components_file = File('components_file.txt', exists=False, usedefault=True, - desc='filename to store physiological components') + desc='Filename to store physiological components') num_components = traits.Int(6, usedefault=True) # 6 for BOLD, 4 for ASL use_regress_poly = traits.Bool(True, usedefault=True, - desc='use polynomial regression pre-component extraction') + desc=('use polynomial regression ' + 'pre-component extraction')) regress_poly_degree = traits.Range(low=1, default=1, usedefault=True, - desc='the degree polynomial to use') - header = traits.Str( - desc='the desired header for the output tsv file (one column).' - 'If undefined, will default to "CompCor"') + desc='the degree polynomial to use') + header = traits.Str(desc=('the desired header for the output tsv file (one ' + 'column). If undefined, will default to ' + '"CompCor"')) + class CompCorOutputSpec(TraitedSpec): components_file = File(exists=True, - desc='text file containing the noise components') + desc='text file containing the noise components') + class CompCor(BaseInterface): """ @@ -360,108 +368,61 @@ class CompCor(BaseInterface): 'tags': ['method', 'implementation'] }] - def _run_interface(self, runtime, tCompCor_mask=False): + def __init__(self, *args, **kwargs): + ''' exactly the same as compcor except the header ''' + super(CompCor, self).__init__(*args, **kwargs) + self._header = 'CompCor' - imgseries = nb.load(self.inputs.realigned_file, - mmap=NUMPY_MMAP).get_data() - components = None - - if isdefined(self.inputs.mask_files) or isdefined(self.inputs.mask_file): - if (not isdefined(self.inputs.mask_index) and - not isdefined(self.inputs.merge_method)): - self.inputs.mask_index = 0 - - if isdefined(self.inputs.mask_index): - if self.inputs.mask_index < len(self.inputs.mask_files): - self.inputs.mask_files = [ - self.inputs.mask_files[self.inputs.mask_index]] - else: - self.inputs.mask_files = self.inputs.mask_files[0] - if not tCompCor_mask: - RuntimeWarning('Mask index exceeded number of masks, using ' - 'mask {} instead'.format(self.inputs.mask_files[0])) - - for mask_file in self.inputs.mask_files: - mask = nb.load(mask_file, mmap=NUMPY_MMAP).get_data() - - if imgseries.shape[:3] != mask.shape: - raise ValueError('Inputs for CompCor, func {} and mask {}, ' - 'do not have matching spatial dimensions ' - '({} and {}, respectively)'.format( - self.inputs.realigned_file, mask_file, - imgseries.shape[:3], mask.shape)) - - if (isdefined(self.inputs.merge_method) and - self.inputs.merge_method != 'none' and - len(self.inputs.mask_files) > 1): - if mask_file == self.inputs.mask_files[0]: - new_mask = mask - continue - else: - if self.inputs.merge_method == 'union': - new_mask = np.logical_or(new_mask, mask).astype(int) - elif self.inputs.merge_method == 'intersect': - new_mask = np.logical_and(new_mask, mask).astype(int) - - if mask_file != self.inputs.mask_files[-1]: - continue - else: # merge complete - mask = new_mask - - voxel_timecourses = imgseries[mask > 0] - # Zero-out any bad values - voxel_timecourses[np.isnan(np.sum(voxel_timecourses, axis=1)), :] = 0 + def _run_interface(self, runtime): + mask_images = [] + if isdefined(self.inputs.mask_files): + mask_images = combine_mask_files(self.inputs.mask_files, + self.inputs.merge_method, + self.inputs.mask_index) - # from paper: - # "The constant and linear trends of the columns in the matrix M were - # removed [prior to ...]" degree = (self.inputs.regress_poly_degree if self.inputs.use_regress_poly else 0) - voxel_timecourses = regress_poly(degree, voxel_timecourses) - # "Voxel time series from the noise ROI (either anatomical or tSTD) were - # placed in a matrix M of size Nxm, with time along the row dimension - # and voxels along the column dimension." - M = voxel_timecourses.T + imgseries = nb.load(self.inputs.realigned_file, + mmap=NUMPY_MMAP) - # "[... were removed] prior to column-wise variance normalization." - M = M / self._compute_tSTD(M, 1.) + if len(imgseries.shape) != 4: + raise ValueError('tCompCor expected a 4-D nifti file. Input {} has ' + '{} dimensions (shape {})'.format( + self.inputs.realigned_file, len(imgseries.shape), + imgseries.shape)) - # "The covariance matrix C = MMT was constructed and decomposed into its - # principal components using a singular value decomposition." - u, _, _ = linalg.svd(M, full_matrices=False) - if components is None: - components = u[:, :self.inputs.num_components] - else: - components = np.hstack((components, - u[:, :self.inputs.num_components])) + if len(mask_images) == 0: + img = nb.Nifti1Image(np.ones(imgseries.shape[:3], dtype=np.bool), + affine=imgseries.affine, + header=imgseries.get_header()) + mask_images = [img] + + mask_images = self._process_masks(mask_images, imgseries.get_data()) + + components = compute_noise_components(imgseries.get_data(), + mask_images, degree, + self.inputs.num_components) components_file = os.path.join(os.getcwd(), self.inputs.components_file) - self._set_header() np.savetxt(components_file, components, fmt=b"%.10f", delimiter='\t', header=self._make_headers(components.shape[1]), comments='') return runtime + def _process_masks(self, mask_images, timeseries=None): + return mask_images + def _list_outputs(self): outputs = self._outputs().get() outputs['components_file'] = os.path.abspath(self.inputs.components_file) return outputs - def _compute_tSTD(self, M, x, axis=0): - stdM = np.std(M, axis=axis) - # set bad values to x - stdM[stdM == 0] = x - stdM[np.isnan(stdM)] = x - return stdM - - def _set_header(self, header='CompCor'): - self.inputs.header = (self.inputs.header if isdefined(self.inputs.header) - else header) - def _make_headers(self, num_col): headers = [] + header = self.inputs.header if isdefined(self.inputs.header) else \ + self._header for i in range(num_col): - headers.append(self.inputs.header + str(i)) + headers.append(header + '{:02d}'.format(i)) return '\t'.join(headers) @@ -475,7 +436,7 @@ class ACompCor(CompCor): def __init__(self, *args, **kwargs): ''' exactly the same as compcor except the header ''' super(ACompCor, self).__init__(*args, **kwargs) - self._set_header('aCompCor') + self._header = 'aCompCor' class TCompCorInputSpec(CompCorInputSpec): @@ -490,10 +451,13 @@ class TCompCorInputSpec(CompCorInputSpec): 'That is, the 2% of voxels ' 'with the highest variance are used.') + class TCompCorOutputSpec(CompCorInputSpec): # and all the fields in CompCorInputSpec - high_variance_masks = OutputMultiPath(File(exists=True, - desc="voxels excedding the variance threshold")) + high_variance_masks = OutputMultiPath(File(exists=True), + desc=("voxels excedding the variance " + "threshold")) + class TCompCor(CompCor): """ @@ -515,108 +479,42 @@ class TCompCor(CompCor): input_spec = TCompCorInputSpec output_spec = TCompCorOutputSpec - def _run_interface(self, runtime): - - _out_masks = [] - img = nb.load(self.inputs.realigned_file, mmap=NUMPY_MMAP) - imgseries = img.get_data() - aff = img.affine - - - if imgseries.ndim != 4: - raise ValueError('tCompCor expected a 4-D nifti file. Input {} has ' - '{} dimensions (shape {})'.format( - self.inputs.realigned_file, imgseries.ndim, - imgseries.shape)) - - if isdefined(self.inputs.mask_files): - if (not isdefined(self.inputs.mask_index) and - not isdefined(self.inputs.merge_method)): - self.inputs.mask_index = 0 - if isdefined(self.inputs.mask_index): - if self.inputs.mask_index < len(self.inputs.mask_files): - self.inputs.mask_files = [ - self.inputs.mask_files[self.inputs.mask_index]] - else: - RuntimeWarning('Mask index exceeded number of masks, using ' - 'mask {} instead'.format(self.inputs.mask_files[0])) - self.inputs.mask_files = self.inputs.mask_files[0] - - for i, mask_file in enumerate(self.inputs.mask_files, 1): - in_mask = nb.load(mask_file, mmap=NUMPY_MMAP).get_data() - if (isdefined(self.inputs.merge_method) and - self.inputs.merge_method != 'none' and - len(self.inputs.mask_files) > 1): - if mask_file == self.inputs.mask_files[0]: - new_mask = in_mask - continue - else: - if self.inputs.merge_method == 'union': - new_mask = np.logical_or(new_mask, - in_mask).astype(int) - elif self.inputs.merge_method == 'intersect': - new_mask = np.logical_and(new_mask, - in_mask).astype(int) - if mask_file != self.inputs.mask_files[-1]: - continue - else: # merge complete - in_mask = new_mask - - imgseries = imgseries[in_mask != 0, :] - - # From the paper: - # "For each voxel time series, the temporal standard deviation is - # defined as the standard deviation of the time series after the removal - # of low-frequency nuisance terms (e.g., linear and quadratic drift)." - imgseries = regress_poly(2, imgseries) - - # "To construct the tSTD noise ROI, we sorted the voxels by their - # temporal standard deviation ..." - tSTD = self._compute_tSTD(imgseries, 0, axis=-1) - - # use percentile_threshold to pick voxels - threshold_std = np.percentile(tSTD, 100. * - (1. - self.inputs.percentile_threshold)) - mask = tSTD >= threshold_std - - mask_data = np.zeros_like(in_mask) - mask_data[in_mask != 0] = mask - # save mask - if self.inputs.merge_method == 'none': - mask_file = os.path.abspath('mask{}.nii'.format(i)) - else: - mask_file = os.path.abspath('mask.nii') - nb.Nifti1Image(mask_data, aff).to_filename(mask_file) - IFLOG.debug('tCompcor computed and saved mask of shape {} to ' - 'mask_file {}'.format(mask.shape, mask_file)) - _out_masks.append(mask_file) - self._set_header('tCompCor') - - else: + def __init__(self, *args, **kwargs): + ''' exactly the same as compcor except the header ''' + super(TCompCor, self).__init__(*args, **kwargs) + self._header = 'tCompCor' + self._mask_files = [] + + def _process_masks(self, mask_images, timeseries=None): + out_images = [] + self._mask_files = [] + for i, img in enumerate(mask_images): + mask = img.get_data().astype(np.bool) + imgseries = timeseries[mask, :] imgseries = regress_poly(2, imgseries) - tSTD = self._compute_tSTD(imgseries, 0, axis=-1) - threshold_std = np.percentile(tSTD, 100. * - (1. - self.inputs.percentile_threshold)) - mask = tSTD >= threshold_std - mask_data = mask.astype(int) + tSTD = _compute_tSTD(imgseries, 0, axis=-1) + threshold_std = np.percentile(tSTD, np.round(100. * + (1. - self.inputs.percentile_threshold)).astype(int)) + mask_data = np.zeros_like(mask) + mask_data[mask != 0] = tSTD >= threshold_std + out_image = nb.Nifti1Image(mask_data, affine=img.affine, + header=img.get_header()) # save mask - mask_file = os.path.abspath('mask.nii') - nb.Nifti1Image(mask_data, aff).to_filename(mask_file) + mask_file = os.path.abspath('mask_{:03d}.nii.gz'.format(i)) + out_image.to_filename(mask_file) IFLOG.debug('tCompcor computed and saved mask of shape {} to ' 'mask_file {}'.format(mask.shape, mask_file)) - _out_masks.append(mask_file) - self._set_header('tCompCor') - - self.inputs.mask_files = _out_masks - super(TCompCor, self)._run_interface(runtime, tCompCor_mask=True) - return runtime + self._mask_files.append(mask_file) + out_images.append(out_image) + return out_images def _list_outputs(self): outputs = super(TCompCor, self)._list_outputs() - outputs['high_variance_masks'] = self.inputs.mask_files + outputs['high_variance_masks'] = self._mask_files return outputs + class TSNRInputSpec(BaseInterfaceInputSpec): in_file = InputMultiPath(File(exists=True), mandatory=True, desc='realigned 4D file or a list of 3D files') @@ -928,3 +826,113 @@ def regress_poly(degree, data, remove_mean=True, axis=-1): # Back to original shape return regressed_data.reshape(datashape) + +def combine_mask_files(mask_files, mask_method=None, mask_index=None): + """Combines input mask files into a single nibabel image + + A helper function for CompCor + + mask_files: a list + one or more binary mask files + mask_method: enum ('union', 'intersect', 'none') + determines how to combine masks + mask_index: an integer + determines which file to return (mutually exclusive with mask_method) + + returns: a list of nibabel images + """ + + if isdefined(mask_index) or not isdefined(mask_method): + if not isdefined(mask_index): + mask_index = 0 + if mask_index < len(mask_files): + mask = nb.load(mask_files[mask_index], mmap=NUMPY_MMAP) + return [mask] + raise ValueError(('mask_index {0} must be less than number of mask ' + 'files {1}').format(mask_index, len(mask_files))) + masks = [] + if mask_method == 'none': + for filename in mask_files: + masks.append(nb.load(filename, mmap=NUMPY_MMAP)) + return masks + + if mask_method == 'union': + mask = None + for filename in mask_files: + img = nb.load(filename, mmap=NUMPY_MMAP) + if mask is None: + mask = img.get_data() > 0 + np.logical_or(mask, img.get_data() > 0, mask) + img = nb.Nifti1Image(mask, img.affine, header=img.get_header()) + return [img] + + if mask_method == 'intersect': + mask = None + for filename in mask_files: + img = nb.load(filename, mmap=NUMPY_MMAP) + if mask is None: + mask = img.get_data() > 0 + np.logical_and(mask, img.get_data() > 0, mask) + img = nb.Nifti1Image(mask, img.affine, header=img.get_header()) + return [img] + + +def compute_noise_components(imgseries, mask_images, degree, num_components): + """Compute the noise components from the imgseries for each mask + + imgseries: a nibabel img + mask_images: a list of nibabel images + degree: order of polynomial used to remove trends from the timeseries + num_components: number of noise components to return + + returns: + + components: a numpy array + + """ + components = None + for img in mask_images: + mask = img.get_data().astype(np.bool) + if imgseries.shape[:3] != mask.shape: + raise ValueError('Inputs for CompCor, timeseries and mask, ' + 'do not have matching spatial dimensions ' + '({} and {}, respectively)'.format( + imgseries.shape[:3], mask.shape)) + + voxel_timecourses = imgseries[mask, :] + + # Zero-out any bad values + voxel_timecourses[np.isnan(np.sum(voxel_timecourses, axis=1)), :] = 0 + + # from paper: + # "The constant and linear trends of the columns in the matrix M were + # removed [prior to ...]" + voxel_timecourses = regress_poly(degree, voxel_timecourses) + + # "Voxel time series from the noise ROI (either anatomical or tSTD) were + # placed in a matrix M of size Nxm, with time along the row dimension + # and voxels along the column dimension." + M = voxel_timecourses.T + + # "[... were removed] prior to column-wise variance normalization." + M = M / _compute_tSTD(M, 1.) + + # "The covariance matrix C = MMT was constructed and decomposed into its + # principal components using a singular value decomposition." + u, _, _ = linalg.svd(M, full_matrices=False) + if components is None: + components = u[:, :num_components] + else: + components = np.hstack((components, + u[:, :num_components])) + if components is None and num_components > 0: + raise ValueError('No components found') + return components + + +def _compute_tSTD(M, x, axis=0): + stdM = np.std(M, axis=axis) + # set bad values to x + stdM[stdM == 0] = x + stdM[np.isnan(stdM)] = x + return stdM diff --git a/nipype/algorithms/tests/test_auto_ACompCor.py b/nipype/algorithms/tests/test_auto_ACompCor.py index 575e3b7e04..4dbf02a9b6 100644 --- a/nipype/algorithms/tests/test_auto_ACompCor.py +++ b/nipype/algorithms/tests/test_auto_ACompCor.py @@ -10,7 +10,13 @@ def test_ACompCor_inputs(): ignore_exception=dict(nohash=True, usedefault=True, ), - mask_file=dict(), + mask_files=dict(), + mask_index=dict(requires=['mask_files'], + xor=['merge_method'], + ), + merge_method=dict(requires=['mask_files'], + xor=['mask_index'], + ), num_components=dict(usedefault=True, ), realigned_file=dict(mandatory=True, diff --git a/nipype/algorithms/tests/test_auto_TCompCor.py b/nipype/algorithms/tests/test_auto_TCompCor.py index 644acf9b05..ba682f57ad 100644 --- a/nipype/algorithms/tests/test_auto_TCompCor.py +++ b/nipype/algorithms/tests/test_auto_TCompCor.py @@ -10,7 +10,13 @@ def test_TCompCor_inputs(): ignore_exception=dict(nohash=True, usedefault=True, ), - mask_file=dict(), + mask_files=dict(), + mask_index=dict(requires=['mask_files'], + xor=['merge_method'], + ), + merge_method=dict(requires=['mask_files'], + xor=['mask_index'], + ), num_components=dict(usedefault=True, ), percentile_threshold=dict(usedefault=True, @@ -33,11 +39,17 @@ def test_TCompCor_outputs(): output_map = dict(components_file=dict(usedefault=True, ), header=dict(), - high_variance_mask=dict(), + high_variance_masks=dict(), ignore_exception=dict(nohash=True, usedefault=True, ), - mask_file=dict(), + mask_files=dict(), + mask_index=dict(requires=['mask_files'], + xor=['merge_method'], + ), + merge_method=dict(requires=['mask_files'], + xor=['mask_index'], + ), num_components=dict(usedefault=True, ), realigned_file=dict(mandatory=True, diff --git a/nipype/algorithms/tests/test_compcor.py b/nipype/algorithms/tests/test_compcor.py index 9c58735472..2e1d29a1cd 100644 --- a/nipype/algorithms/tests/test_compcor.py +++ b/nipype/algorithms/tests/test_compcor.py @@ -70,7 +70,7 @@ def test_tcompcor_no_percentile(self): ccinterface = TCompCor(realigned_file=self.realigned_file) ccinterface.run() - mask = nb.load('mask.nii').get_data() + mask = nb.load('mask_000.nii.gz').get_data() num_nonmasked_voxels = np.count_nonzero(mask) assert num_nonmasked_voxels == 1 @@ -89,7 +89,7 @@ def test_tcompcor_asymmetric_dim(self): 'asymmetric.nii') TCompCor(realigned_file=asymmetric_data).run() - assert nb.load('mask.nii').get_data().shape == asymmetric_shape[:3] + assert nb.load('mask_000.nii.gz').get_data().shape == asymmetric_shape[:3] def test_compcor_bad_input_shapes(self): shape_less_than = (1, 2, 2, 5) # dim 0 is < dim 0 of self.mask_files (2) @@ -112,17 +112,17 @@ def test_tcompcor_merge_intersect_masks(self): mask_files=self.mask_files, merge_method=method).run() if method == 'union': - assert np.array_equal(nb.load('mask.nii').get_data(), + assert np.array_equal(nb.load('mask_000.nii.gz').get_data(), ([[[0,0],[0,0]],[[0,0],[1,0]]])) if method == 'intersect': - assert np.array_equal(nb.load('mask.nii').get_data(), + assert np.array_equal(nb.load('mask_000.nii.gz').get_data(), ([[[0,0],[0,0]],[[0,1],[0,0]]])) def test_tcompcor_index_mask(self): TCompCor(realigned_file=self.realigned_file, mask_files=self.mask_files, mask_index=1).run() - assert np.array_equal(nb.load('mask.nii').get_data(), + assert np.array_equal(nb.load('mask_000.nii.gz').get_data(), ([[[0,0],[0,0]],[[0,1],[0,0]]])) def run_cc(self, ccinterface, expected_components, expected_header='CompCor'): @@ -142,7 +142,8 @@ def run_cc(self, ccinterface, expected_components, expected_header='CompCor'): components_data = [line.split('\t') for line in components_file] header = components_data.pop(0) # the first item will be '#', we can throw it out - expected_header = [expected_header + str(i) for i in range(expected_n_components)] + expected_header = [expected_header + '{:02d}'.format(i) for i in + range(expected_n_components)] for i, heading in enumerate(header): assert expected_header[i] in heading From c0a3c7fda5eb52ca35be552d4ea855d2048800cc Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 2 May 2017 19:05:52 -0400 Subject: [PATCH 2/6] fix: clean auto tests and white spaces --- doc/documentation.rst | 2 +- doc/users/config_file.rst | 2 +- nipype/algorithms/confounds.py | 36 +++++++++---------- nipype/interfaces/afni/preprocess.py | 4 +-- nipype/interfaces/afni/utils.py | 2 +- nipype/interfaces/fsl/tests/test_auto_Eddy.py | 3 -- .../interfaces/fsl/tests/test_auto_TOPUP.py | 10 +++--- nipype/interfaces/fsl/utils.py | 10 +++--- 8 files changed, 33 insertions(+), 36 deletions(-) diff --git a/doc/documentation.rst b/doc/documentation.rst index 39e3cadb08..ec89d8c061 100644 --- a/doc/documentation.rst +++ b/doc/documentation.rst @@ -29,7 +29,7 @@ Previous versions: `0.12.0 `_ `0.11.0 :maxdepth: 2 users/index - + .. toctree:: :maxdepth: 1 diff --git a/doc/users/config_file.rst b/doc/users/config_file.rst index 6d70e614e7..7d55cc522d 100644 --- a/doc/users/config_file.rst +++ b/doc/users/config_file.rst @@ -145,7 +145,7 @@ Execution crashfiles allow interactive debugging and rerunning of nodes, while text crashfiles allow portability across machines and shorter load time. (possible values: ``pklz`` and ``txt``; default value: ``pklz``) - + Example ~~~~~~~ diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 4ceb222482..b8e907954d 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -306,28 +306,28 @@ class CompCorInputSpec(BaseInterfaceInputSpec): realigned_file = File(exists=True, mandatory=True, desc='already realigned brain image (4D)') mask_files = InputMultiPath(File(exists=True), - desc=('One or more mask files that determines ' + desc=('One or more mask files that determines ' 'ROI (3D)')) merge_method = traits.Enum('union', 'intersect', 'none', xor=['mask_index'], requires=['mask_files'], - desc=('Merge method if multiple masks are ' - 'present - `union` aggregates all masks, ' - '`intersect` computes the truth value of ' - 'all masks, `none` performs CompCor on ' + desc=('Merge method if multiple masks are ' + 'present - `union` aggregates all masks, ' + '`intersect` computes the truth value of ' + 'all masks, `none` performs CompCor on ' 'each mask individually')) mask_index = traits.Range(low=0, xor=['merge_method'], requires=['mask_files'], - desc=('Position of mask in `mask_files` to use - ' + desc=('Position of mask in `mask_files` to use - ' 'first is the default')) components_file = File('components_file.txt', exists=False, usedefault=True, desc='Filename to store physiological components') num_components = traits.Int(6, usedefault=True) # 6 for BOLD, 4 for ASL use_regress_poly = traits.Bool(True, usedefault=True, - desc=('use polynomial regression ' + desc=('use polynomial regression ' 'pre-component extraction')) regress_poly_degree = traits.Range(low=1, default=1, usedefault=True, desc='the degree polynomial to use') - header = traits.Str(desc=('the desired header for the output tsv file (one ' + header = traits.Str(desc=('the desired header for the output tsv file (one ' 'column). If undefined, will default to ' '"CompCor"')) @@ -350,7 +350,7 @@ class CompCor(BaseInterface): >>> ccinterface.inputs.num_components = 1 >>> ccinterface.inputs.use_regress_poly = True >>> ccinterface.inputs.regress_poly_degree = 2 - + """ input_spec = CompCorInputSpec output_spec = CompCorOutputSpec @@ -473,7 +473,7 @@ class TCompCor(CompCor): >>> ccinterface.inputs.use_regress_poly = True >>> ccinterface.inputs.regress_poly_degree = 2 >>> ccinterface.inputs.percentile_threshold = .03 - + """ input_spec = TCompCorInputSpec @@ -829,16 +829,16 @@ def regress_poly(degree, data, remove_mean=True, axis=-1): def combine_mask_files(mask_files, mask_method=None, mask_index=None): """Combines input mask files into a single nibabel image - + A helper function for CompCor - + mask_files: a list one or more binary mask files mask_method: enum ('union', 'intersect', 'none') determines how to combine masks mask_index: an integer determines which file to return (mutually exclusive with mask_method) - + returns: a list of nibabel images """ @@ -879,16 +879,16 @@ def combine_mask_files(mask_files, mask_method=None, mask_index=None): def compute_noise_components(imgseries, mask_images, degree, num_components): """Compute the noise components from the imgseries for each mask - + imgseries: a nibabel img mask_images: a list of nibabel images degree: order of polynomial used to remove trends from the timeseries num_components: number of noise components to return - + returns: - - components: a numpy array - + + components: a numpy array + """ components = None for img in mask_images: diff --git a/nipype/interfaces/afni/preprocess.py b/nipype/interfaces/afni/preprocess.py index 3556e7e589..54e9411f98 100644 --- a/nipype/interfaces/afni/preprocess.py +++ b/nipype/interfaces/afni/preprocess.py @@ -2417,7 +2417,7 @@ class QwarpPlusMinusOutputSpec(TraitedSpec): class QwarpPlusMinus(CommandLine): - """A version of 3dQwarp for performing field susceptibility correction + """A version of 3dQwarp for performing field susceptibility correction using two images with opposing phase encoding directions. For complete details, see the `3dQwarp Documentation. @@ -2434,7 +2434,7 @@ class QwarpPlusMinus(CommandLine): >>> qwarp.cmdline # doctest: +ALLOW_UNICODE '3dQwarp -prefix Qwarp.nii.gz -plusminus -base sub-01_dir-RL_epi.nii.gz -nopadWARP -source sub-01_dir-LR_epi.nii.gz' >>> res = warp.run() # doctest: +SKIP - + """ _cmd = '3dQwarp -prefix Qwarp.nii.gz -plusminus' input_spec = QwarpPlusMinusInputSpec diff --git a/nipype/interfaces/afni/utils.py b/nipype/interfaces/afni/utils.py index cb544a7e46..10e180ee7e 100644 --- a/nipype/interfaces/afni/utils.py +++ b/nipype/interfaces/afni/utils.py @@ -695,7 +695,7 @@ class FWHMx(AFNICommandBase): _cmd = '3dFWHMx' input_spec = FWHMxInputSpec output_spec = FWHMxOutputSpec - + references_ = [{'entry': BibTeX('@article{CoxReynoldsTaylor2016,' 'author={R.W. Cox, R.C. Reynolds, and P.A. Taylor},' 'title={AFNI and clustering: false positive rates redux},' diff --git a/nipype/interfaces/fsl/tests/test_auto_Eddy.py b/nipype/interfaces/fsl/tests/test_auto_Eddy.py index e47633fb07..c5f521045f 100644 --- a/nipype/interfaces/fsl/tests/test_auto_Eddy.py +++ b/nipype/interfaces/fsl/tests/test_auto_Eddy.py @@ -88,9 +88,6 @@ def test_Eddy_inputs(): def test_Eddy_outputs(): output_map = dict(out_corrected=dict(), out_movement_rms=dict(), - out_outlier_map=dict(), - out_outlier_n_sd_map=dict(), - out_outlier_n_sqr_sd_map=dict(), out_outlier_report=dict(), out_parameter=dict(), out_restricted_movement_rms=dict(), diff --git a/nipype/interfaces/fsl/tests/test_auto_TOPUP.py b/nipype/interfaces/fsl/tests/test_auto_TOPUP.py index 01f50670ba..28083c6dc0 100644 --- a/nipype/interfaces/fsl/tests/test_auto_TOPUP.py +++ b/nipype/interfaces/fsl/tests/test_auto_TOPUP.py @@ -54,10 +54,6 @@ def test_TOPUP_inputs(): name_source=['in_file'], name_template='%s_field', ), - out_warp_prefix=dict(argstr='--dfout=%s', - hash_files=False, - usedefault=True, - ), out_jac_prefix=dict(argstr='--jacout=%s', hash_files=False, usedefault=True, @@ -68,6 +64,10 @@ def test_TOPUP_inputs(): name_source=['in_file'], name_template='%s_topup.log', ), + out_warp_prefix=dict(argstr='--dfout=%s', + hash_files=False, + usedefault=True, + ), output_type=dict(), readout_times=dict(mandatory=True, requires=['encoding_direction'], @@ -104,10 +104,10 @@ def test_TOPUP_outputs(): out_enc_file=dict(), out_field=dict(), out_fieldcoef=dict(), + out_jacs=dict(), out_logfile=dict(), out_movpar=dict(), out_warps=dict(), - out_jacs=dict(), ) outputs = TOPUP.output_spec() diff --git a/nipype/interfaces/fsl/utils.py b/nipype/interfaces/fsl/utils.py index 42f22ea761..7bb95f49eb 100644 --- a/nipype/interfaces/fsl/utils.py +++ b/nipype/interfaces/fsl/utils.py @@ -73,7 +73,7 @@ class RobustFOVInputSpec(FSLCommandInputSpec): brainsize = traits.Int(desc=('size of brain in z-dimension (default ' '170mm/150mm)'), argstr='-b %d') - out_transform = File(desc=("Transformation matrix in_file to out_roi " + out_transform = File(desc=("Transformation matrix in_file to out_roi " "output name"), argstr="-m %s", name_source=['in_file'], hash_files=False, @@ -83,17 +83,17 @@ class RobustFOVInputSpec(FSLCommandInputSpec): class RobustFOVOutputSpec(TraitedSpec): out_roi = File(exists=True, desc="ROI volume output name") out_transform = File(exists=True, - desc=("Transformation matrix in_file to out_roi " + desc=("Transformation matrix in_file to out_roi " "output name")) class RobustFOV(FSLCommand): """Automatically crops an image removing lower head and neck. - - Interface is stable 5.0.0 to 5.0.9, but default brainsize changed from + + Interface is stable 5.0.0 to 5.0.9, but default brainsize changed from 150mm to 170mm. """ - + _cmd = 'robustfov' input_spec = RobustFOVInputSpec output_spec = RobustFOVOutputSpec From ca205efe9cb976763c92c211acdf33755c2c01a1 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 3 May 2017 17:01:53 -0400 Subject: [PATCH 3/6] fix: ensure outputspec is derived from outputspec --- nipype/algorithms/confounds.py | 6 +++--- nipype/algorithms/tests/test_compcor.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index b8e907954d..b4eacf3920 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -319,7 +319,7 @@ class CompCorInputSpec(BaseInterfaceInputSpec): requires=['mask_files'], desc=('Position of mask in `mask_files` to use - ' 'first is the default')) - components_file = File('components_file.txt', exists=False, usedefault=True, + components_file = traits.Str('components_file.txt', usedefault=True, desc='Filename to store physiological components') num_components = traits.Int(6, usedefault=True) # 6 for BOLD, 4 for ASL use_regress_poly = traits.Bool(True, usedefault=True, @@ -452,8 +452,8 @@ class TCompCorInputSpec(CompCorInputSpec): 'with the highest variance are used.') -class TCompCorOutputSpec(CompCorInputSpec): - # and all the fields in CompCorInputSpec +class TCompCorOutputSpec(CompCorOutputSpec): + # and all the fields in CompCorOutputSpec high_variance_masks = OutputMultiPath(File(exists=True), desc=("voxels excedding the variance " "threshold")) diff --git a/nipype/algorithms/tests/test_compcor.py b/nipype/algorithms/tests/test_compcor.py index 2e1d29a1cd..ec31def701 100644 --- a/nipype/algorithms/tests/test_compcor.py +++ b/nipype/algorithms/tests/test_compcor.py @@ -8,7 +8,6 @@ import pytest from ...testing import utils from ..confounds import CompCor, TCompCor, ACompCor -from ...interfaces.base import Undefined class TestCompCor(): From 3551a97870f9bac7c767062f1f8ca80f98b913e3 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 3 May 2017 17:36:50 -0400 Subject: [PATCH 4/6] fix: clean up tests and help messages --- nipype/algorithms/confounds.py | 30 +++++++++++++-------- nipype/algorithms/tests/test_compcor.py | 36 +++++++++++++++++-------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index b4eacf3920..4591694f31 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -307,18 +307,21 @@ class CompCorInputSpec(BaseInterfaceInputSpec): desc='already realigned brain image (4D)') mask_files = InputMultiPath(File(exists=True), desc=('One or more mask files that determines ' - 'ROI (3D)')) + 'ROI (3D). When more that one file is ' + 'provided `merge_method` or ' + '`merge_index` must be provided')) merge_method = traits.Enum('union', 'intersect', 'none', xor=['mask_index'], requires=['mask_files'], desc=('Merge method if multiple masks are ' - 'present - `union` aggregates all masks, ' - '`intersect` computes the truth value of ' - 'all masks, `none` performs CompCor on ' + 'present - `union` uses voxels included in' + ' at least one input mask, `intersect` ' + 'uses only voxels present in all input ' + 'masks, `none` performs CompCor on ' 'each mask individually')) mask_index = traits.Range(low=0, xor=['merge_method'], requires=['mask_files'], desc=('Position of mask in `mask_files` to use - ' - 'first is the default')) + 'first is the default.')) components_file = traits.Str('components_file.txt', usedefault=True, desc='Filename to store physiological components') num_components = traits.Int(6, usedefault=True) # 6 for BOLD, 4 for ASL @@ -327,9 +330,9 @@ class CompCorInputSpec(BaseInterfaceInputSpec): 'pre-component extraction')) regress_poly_degree = traits.Range(low=1, default=1, usedefault=True, desc='the degree polynomial to use') - header = traits.Str(desc=('the desired header for the output tsv file (one ' - 'column). If undefined, will default to ' - '"CompCor"')) + header_prefix = traits.Str(desc=('the desired header for the output tsv ' + 'file (one column). If undefined, will ' + 'default to "CompCor"')) class CompCorOutputSpec(TraitedSpec): @@ -419,8 +422,8 @@ def _list_outputs(self): def _make_headers(self, num_col): headers = [] - header = self.inputs.header if isdefined(self.inputs.header) else \ - self._header + header = self.inputs.header_prefix if \ + isdefined(self.inputs.header_prefix) else self._header for i in range(num_col): headers.append(header + '{:02d}'.format(i)) return '\t'.join(headers) @@ -844,7 +847,12 @@ def combine_mask_files(mask_files, mask_method=None, mask_index=None): if isdefined(mask_index) or not isdefined(mask_method): if not isdefined(mask_index): - mask_index = 0 + if len(mask_files) == 1: + mask_index = 0 + else: + raise ValueError(('When more than one mask file is provided, ' + 'one of merge_method or mask_index must be ' + 'set')) if mask_index < len(mask_files): mask = nb.load(mask_files[mask_index], mmap=NUMPY_MMAP) return [mask] diff --git a/nipype/algorithms/tests/test_compcor.py b/nipype/algorithms/tests/test_compcor.py index ec31def701..adb495f90f 100644 --- a/nipype/algorithms/tests/test_compcor.py +++ b/nipype/algorithms/tests/test_compcor.py @@ -47,11 +47,13 @@ def test_compcor(self): ['-0.1246655485', '-0.1235705610']] self.run_cc(CompCor(realigned_file=self.realigned_file, - mask_files=self.mask_files), + mask_files=self.mask_files, + mask_index=0), expected_components) self.run_cc(ACompCor(realigned_file=self.realigned_file, mask_files=self.mask_files, + mask_index=0, components_file='acc_components_file'), expected_components, 'aCompCor') @@ -63,7 +65,8 @@ def test_tcompcor(self): ['0.4566907310', '0.6983205193'], ['-0.7132557407', '0.1340170559'], ['0.5022537643', '-0.5098322262'], - ['-0.1342351356', '0.1407855119']], 'tCompCor') + ['-0.1342351356', '0.1407855119']], + 'tCompCor') def test_tcompcor_no_percentile(self): ccinterface = TCompCor(realigned_file=self.realigned_file) @@ -75,12 +78,14 @@ def test_tcompcor_no_percentile(self): def test_compcor_no_regress_poly(self): self.run_cc(CompCor(realigned_file=self.realigned_file, - mask_files=self.mask_files, - use_regress_poly=False), [['0.4451946442', '-0.7683311482'], - ['-0.4285129505', '-0.0926034137'], - ['0.5721540256', '0.5608764842'], - ['-0.5367548139', '0.0059943226'], - ['-0.0520809054', '0.2940637551']]) + mask_files=self.mask_files, + mask_index=0, + use_regress_poly=False), + [['0.4451946442', '-0.7683311482'], + ['-0.4285129505', '-0.0926034137'], + ['0.5721540256', '0.5608764842'], + ['-0.5367548139', '0.0059943226'], + ['-0.0520809054', '0.2940637551']]) def test_tcompcor_asymmetric_dim(self): asymmetric_shape = (2, 3, 4, 5) @@ -96,7 +101,8 @@ def test_compcor_bad_input_shapes(self): for data_shape in (shape_less_than, shape_more_than): data_file = utils.save_toy_nii(np.zeros(data_shape), 'temp.nii') - interface = CompCor(realigned_file=data_file, mask_files=self.mask_files) + interface = CompCor(realigned_file=data_file, + mask_files=self.mask_files[0]) with pytest.raises(ValueError, message="Dimension mismatch"): interface.run() def test_tcompcor_bad_input_dim(self): @@ -124,6 +130,12 @@ def test_tcompcor_index_mask(self): assert np.array_equal(nb.load('mask_000.nii.gz').get_data(), ([[[0,0],[0,0]],[[0,1],[0,0]]])) + def test_tcompcor_multi_mask_no_index(self): + interface = TCompCor(realigned_file=self.realigned_file, + mask_files=self.mask_files) + with pytest.raises(ValueError, message='more than one mask file'): + interface.run() + def run_cc(self, ccinterface, expected_components, expected_header='CompCor'): # run ccresult = ccinterface.run() @@ -136,11 +148,13 @@ def run_cc(self, ccinterface, expected_components, expected_header='CompCor'): assert ccinterface.inputs.num_components == 6 with open(ccresult.outputs.components_file, 'r') as components_file: - expected_n_components = min(ccinterface.inputs.num_components, self.fake_data.shape[3]) + expected_n_components = min(ccinterface.inputs.num_components, + self.fake_data.shape[3]) components_data = [line.split('\t') for line in components_file] - header = components_data.pop(0) # the first item will be '#', we can throw it out + # the first item will be '#', we can throw it out + header = components_data.pop(0) expected_header = [expected_header + '{:02d}'.format(i) for i in range(expected_n_components)] for i, heading in enumerate(header): From 783c6a7707022f508db8dee58c3da121b28212dd Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 3 May 2017 17:39:38 -0400 Subject: [PATCH 5/6] fix: typo --- nipype/algorithms/confounds.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nipype/algorithms/confounds.py b/nipype/algorithms/confounds.py index 4591694f31..d32f2fadb7 100644 --- a/nipype/algorithms/confounds.py +++ b/nipype/algorithms/confounds.py @@ -458,8 +458,8 @@ class TCompCorInputSpec(CompCorInputSpec): class TCompCorOutputSpec(CompCorOutputSpec): # and all the fields in CompCorOutputSpec high_variance_masks = OutputMultiPath(File(exists=True), - desc=("voxels excedding the variance " - "threshold")) + desc=(("voxels exceeding the variance" + " threshold"))) class TCompCor(CompCor): @@ -755,6 +755,7 @@ def plot_confound(tseries, figsize, name, units=None, ax.set_yticklabels([]) return fig + def is_outlier(points, thresh=3.5): """ Returns a boolean array with True if points are outliers and False From 85b56925e44fabfcebc7bbd2d53eadf842b4a093 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 3 May 2017 17:56:45 -0400 Subject: [PATCH 6/6] fix: update autotests --- nipype/algorithms/tests/test_auto_ACompCor.py | 2 +- nipype/algorithms/tests/test_auto_TCompCor.py | 24 ++----------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/nipype/algorithms/tests/test_auto_ACompCor.py b/nipype/algorithms/tests/test_auto_ACompCor.py index 4dbf02a9b6..7867b259ed 100644 --- a/nipype/algorithms/tests/test_auto_ACompCor.py +++ b/nipype/algorithms/tests/test_auto_ACompCor.py @@ -6,7 +6,7 @@ def test_ACompCor_inputs(): input_map = dict(components_file=dict(usedefault=True, ), - header=dict(), + header_prefix=dict(), ignore_exception=dict(nohash=True, usedefault=True, ), diff --git a/nipype/algorithms/tests/test_auto_TCompCor.py b/nipype/algorithms/tests/test_auto_TCompCor.py index ba682f57ad..47bb550da3 100644 --- a/nipype/algorithms/tests/test_auto_TCompCor.py +++ b/nipype/algorithms/tests/test_auto_TCompCor.py @@ -6,7 +6,7 @@ def test_TCompCor_inputs(): input_map = dict(components_file=dict(usedefault=True, ), - header=dict(), + header_prefix=dict(), ignore_exception=dict(nohash=True, usedefault=True, ), @@ -36,28 +36,8 @@ def test_TCompCor_inputs(): def test_TCompCor_outputs(): - output_map = dict(components_file=dict(usedefault=True, - ), - header=dict(), + output_map = dict(components_file=dict(), high_variance_masks=dict(), - ignore_exception=dict(nohash=True, - usedefault=True, - ), - mask_files=dict(), - mask_index=dict(requires=['mask_files'], - xor=['merge_method'], - ), - merge_method=dict(requires=['mask_files'], - xor=['mask_index'], - ), - num_components=dict(usedefault=True, - ), - realigned_file=dict(mandatory=True, - ), - regress_poly_degree=dict(usedefault=True, - ), - use_regress_poly=dict(usedefault=True, - ), ) outputs = TCompCor.output_spec()