Skip to content

[ENH] Filter layout #510

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Remi-Gau merged 5 commits into from
Jan 22, 2023
Merged

[ENH] Filter layout #510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e2029f4
add filter to layout to only index certain subjects, sessions, modali…
Remi-Gau Jan 22, 2023
0855b9a
use filtering of layout
Remi-Gau Jan 22, 2023
9e05c94
fix when no filter
Remi-Gau Jan 22, 2023
bbfb6c8
use regexp
Remi-Gau Jan 22, 2023
d67d110
fix when to use regex
Remi-Gau Jan 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 73 additions & 3 deletions +bids/layout.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
% 'use_schema', true, ...
% 'index_derivatives', false, ...
% 'index_dependencies', true, ...
% 'filter', struct([]), ...
% 'tolerant', true, ...
% 'verbose', false)
%
Expand All @@ -31,13 +32,29 @@
% associated TSV files for each file...)
% :type index_dependencies: logical
%
% :param filter: if ``true`` this will index the dependencies (json files,
% associated TSV files for each file...)
% :type filter: struct with optional fields ``sub``, ``ses``, ``modality``.
% Regular expression can be used for ``sub`` and ``ses``.
%
% :param tolerant: Set to ``true`` to turn validation errors into warnings
% :type tolerant: logical
%
% :param verbose: Set to ``true`` to get more feedback
% :type verbose: logical
%
%
% Example::
%
% BIDS = bids.layout(fullfile(get_test_data_dir(), '7t_trt'), ...
% 'use_schema', true, ...
% 'verbose', true, ...
% 'index_derivatives', false, ...
% 'filter', struct('sub', {{'^0[12]'}}, ...
% 'modality', {{'anat', 'func'}}, ...
% 'ses', {{'1', '2'}}));
%
%

% (C) Copyright 2016-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging
%
Expand All @@ -51,6 +68,7 @@
default_index_derivatives = false;
default_index_dependencies = true;
default_tolerant = true;
default_filter = struct([]);
default_use_schema = true;
default_verbose = false;

Expand All @@ -61,6 +79,7 @@
addOptional(args, 'root', default_root, is_dir_or_struct);
addParameter(args, 'index_derivatives', default_index_derivatives);
addParameter(args, 'index_dependencies', default_index_dependencies);
addParameter(args, 'filter', default_filter, @isstruct);
addParameter(args, 'tolerant', default_tolerant);
addParameter(args, 'use_schema', default_use_schema);
addParameter(args, 'verbose', default_verbose);
Expand All @@ -70,6 +89,7 @@
root = args.Results.root;
index_derivatives = args.Results.index_derivatives;
index_dependencies = args.Results.index_dependencies;
filter = args.Results.filter;
tolerant = args.Results.tolerant;
use_schema = args.Results.use_schema;
verbose = args.Results.verbose;
Expand Down Expand Up @@ -141,6 +161,10 @@

for iSub = 1:numel(subjects)

if exclude_subject(filter, strrep(subjects{iSub}, 'sub-', ''))
continue
end

if verbose
fprintf(1, ' Indexing subject: %s [', subjects{iSub});
end
Expand All @@ -151,13 +175,18 @@
'^ses-.*$'));

for iSess = 1:numel(sessions)

if exclude_session(filter, strrep(sessions{iSess}, 'ses-', ''))
continue
end

if isempty(BIDS.subjects)
BIDS.subjects = parse_subject(BIDS.pth, subjects{iSub}, sessions{iSess}, ...
schema, tolerant, verbose);
schema, filter, tolerant, verbose);

else
new_subject = parse_subject(BIDS.pth, subjects{iSub}, sessions{iSess}, ...
schema, tolerant, verbose);
schema, filter, tolerant, verbose);
[BIDS.subjects, new_subject] = bids.internal.match_structure_fields(BIDS.subjects, ...
new_subject);
% TODO: this can be added to "match_structure_fields"
Expand All @@ -184,6 +213,34 @@

end

function value = exclude(filter, entity, label)
value = false;
% skip if not included in filter
if ~isfield(filter, entity)
return
end
% use regex when filter is a cell of numel 1
if numel(filter.(entity)) == 1
if cellfun('isempty', regexp(label, filter.(entity)))
value = true;
end
return
end
% otherwise we just check all elements
if ~ismember(label, filter.(entity))
value = true;
return
end
end

function value = exclude_subject(filter, sub_label)
value = exclude(filter, 'sub', sub_label);
end

function value = exclude_session(filter, ses_label)
value = exclude(filter, 'ses', ses_label);
end

function BIDS = index_root_directory(BIDS)
% index json and tsv files in the root directory
files_to_exclude = {'participants', ... already done
Expand Down Expand Up @@ -235,7 +292,7 @@
end
end

function subject = parse_subject(pth, subjname, sesname, schema, tolerant, verbose)
function subject = parse_subject(pth, subjname, sesname, schema, filter, tolerant, verbose)
%
% Parse a subject's directory
%
Expand Down Expand Up @@ -271,6 +328,11 @@
% so the parsing is unconstrained
for iModality = 1:numel(modalities)

if isfield(filter, 'modality') && ...
~ismember(modalities{iModality}, filter.modality)
continue
end

switch modalities{iModality}

case {'anat', ...
Expand Down Expand Up @@ -670,8 +732,16 @@
end

for iIntended = 1:numel(intended)

dest = fullfile(BIDS.pth, BIDS.subjects(info_src.sub_idx).name, ...
intended{iIntended});
% TODO: need to better take care of URI
if strfind(intended{iIntended}, ':')
tmp = strsplit(intended{iIntended}, '/');
this_intended = strjoin(tmp(2:end), '/');
dest = fullfile(BIDS.pth, BIDS.subjects(info_src.sub_idx).name, this_intended);
end

% only throw warning for non-datalad dataset
% to avoid excessive warning as sym link are not files
if ~exist(dest, 'file')
Expand Down
45 changes: 43 additions & 2 deletions tests/tests_layout/test_layout.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,53 @@
initTestSuite;
end

function test_layout_filter()

verbose = false;

BIDS = bids.layout(fullfile(get_test_data_dir(), '7t_trt'), ...
'verbose', verbose, ...
'filter', struct('sub', {{'01', '02'}}, ...
'modality', {{'anat', 'func'}}, ...
'ses', {{'1', '2'}}));

subjects = bids.query(BIDS, 'subjects');
assertEqual(subjects, {'01', '02'});

subjects = bids.query(BIDS, 'modalities');
assertEqual(subjects, {'anat', 'func'});

subjects = bids.query(BIDS, 'sessions');
assertEqual(subjects, {'1', '2'});

end

function test_layout_filter_regex()

verbose = false;

BIDS = bids.layout(fullfile(get_test_data_dir(), '7t_trt'), ...
'verbose', verbose, ...
'filter', struct('sub', {{'^.*0[12]'}}, ...
'modality', {{'anat', 'func'}}, ...
'ses', {{'[1]'}}));

subjects = bids.query(BIDS, 'subjects');
assertEqual(subjects, {'01', '02'});

subjects = bids.query(BIDS, 'modalities');
assertEqual(subjects, {'anat', 'func'});

subjects = bids.query(BIDS, 'sessions');
assertEqual(subjects, {'1'});

end

function test_layout_empty_subject_folder_allowed_when_schemaless()

verbose = false;

mkdir tmp;
mkdir tmp/sub-01;
bids.util.mkdir(fullfile(pwd, 'tmp/sub-01'));
bids.layout(fullfile(pwd, 'tmp'), 'use_schema', false, 'verbose', verbose);
rmdir(fullfile(pwd, 'tmp'), 's');
end
Expand Down