Add support for modalities, add classification to files

api/api.py

@@ -7,6 +7,7 @@
 from .handlers.confighandler      import Config, Version
 from .handlers.containerhandler   import ContainerHandler
 from .handlers.devicehandler      import DeviceHandler
+from .handlers.modalityhandler    import ModalityHandler
 from .handlers.grouphandler       import GroupHandler
 from .handlers.listhandler        import AnalysesHandler, ListHandler, FileListHandler, NotesListHandler, PermissionsListHandler, TagsListHandler
 from .handlers.reporthandler      import ReportHandler
@@ -169,6 +170,14 @@ def prefix(path, routes):
             route('/<device_id:[^/]+>', DeviceHandler,                 m=['GET']),
         ]),
 
+        # Modalities
+
+        route( '/modalities',               ModalityHandler, h='get_all',    m=['GET']),
+        route( '/modalities',               ModalityHandler,                 m=['POST']),
+        prefix('/modalities', [
+            route('/<modality_name:[^/]+>', ModalityHandler,                 m=['GET', 'PUT', 'DELETE']),
+        ]),
+
 
         # Groups
 
@@ -245,7 +254,7 @@ def prefix(path, routes):
                 route('/packfile',                              FileListHandler, h='packfile',       m=['POST']),
                 route('/packfile-end',                          FileListHandler, h='packfile_end'),
                 route('/<list_name:files>',                     FileListHandler,                     m=['POST']),
-                route('/<list_name:files>/<name:{fname}>',      FileListHandler,                     m=['GET', 'DELETE']),
+                route('/<list_name:files>/<name:{fname}>',      FileListHandler,                     m=['GET', 'DELETE', 'PUT']),
                 route('/<list_name:files>/<name:{fname}>/info', FileListHandler, h='get_info',       m=['GET']),
 
 

api/auth/listauth.py

@@ -20,6 +20,7 @@ def default_sublist(handler, container):
     access = _get_access(handler.uid, handler.user_site, container)
     def g(exec_op):
         def f(method, _id, query_params=None, payload=None, exclude_params=None):
+            log.debug('Im actually in here')
             if method == 'GET' and container.get('public', False):
                 min_access = -1
             elif method == 'GET':

api/config.py

@@ -159,6 +159,7 @@ def apply_env_variables(config):
     'file.json',
     'group-new.json',
     'group-update.json',
+    'modality.json',
     'note.json',
     'packfile.json',
     'permission.json',

api/dao/liststorage.py

@@ -3,6 +3,7 @@
 import datetime
 
 from .. import config
+from .. import util
 from . import consistencychecker, containerutil
 from . import APIStorageException, APIConflictException
 from .containerstorage import SessionStorage, AcquisitionStorage
@@ -352,3 +353,32 @@ def inflate_job_info(analysis):
 
         analysis['job'] = job
         return analysis
+
+
+class FileStorage(ListStorage):
+
+    def update_file(self, _id, query_params, payload, replace_fields=False):
+        mod_elem = {}
+        update = {}
+
+        # If we want to add to the classification lists rather than replace
+        # the entirity of the classification map, use $addToSet.
+        # This allows some endpoints to only make additive changes
+        if not replace_fields:
+            classification = payload.pop('classification', None)
+            if classification:
+                add_to_set = {}
+                for k,array in classification.items():
+                    add_to_set[self.list_name + '.$.classification.' + k] = array
+                update['$addToSet'] = add_to_set
+            payload = util.mongo_dict(payload)
+
+        for k,v in payload.items():
+            mod_elem[self.list_name + '.$.' + k] = v
+        query = {
+            '_id': _id,
+            self.list_name: {'$elemMatch': query_params}
+        }
+        update['$set'] = mod_elem
+
+        return self.dbc.update_one(query, update)

api/filetypes.json

@@ -1,28 +1,29 @@
 {
-    "bval":         [ ".bval", ".bvals" ],
-    "bvec":         [ ".bvec", ".bvecs" ],
-    "dicom":        [ ".dcm", ".dcm.zip", ".dicom.zip" ],
-    "ismrmrd":      [ ".h5", ".hdf5" ],
-    "parrec":       [ ".parrec.zip", ".par-rec.zip" ],
-    "gephysio":     [ ".gephysio.zip" ],
-    "MATLAB data":  [ ".mat" ],
-    "MGH data":     [ ".mgh", ".mgz", ".mgh.gz" ],
-    "nifti":        [ ".nii.gz", ".nii" ],
-    "pfile":        [ ".7.gz", ".7", ".7.zip" ],
-    "PsychoPy data":  [ ".psydat" ],
-    "qa":           [ ".qa.png", ".qa.json", ".qa.html" ],
+    "BVAL":             [ ".bval", ".bvals" ],
+    "BVEC":             [ ".bvec", ".bvecs" ],
+    "DICOM":            [ ".dcm", ".dcm.zip", ".dicom.zip" ],
+    "GE Physio":        [ ".gephysio.zip" ],
+    "ISMRMRD":          [ ".h5", ".hdf5" ],
+    "MATLAB Data":      [ ".mat" ],
+    "MGH Data":         [ ".mgh", ".mgz", ".mgh.gz" ],
+    "NIfTI":            [ ".nii.gz", ".nii" ],
+    "PAR/REC":          [ ".parrec.zip", ".par-rec.zip" ],
+    "PFile":            [ ".7.gz", ".7", ".7.zip" ],
+    "PsychoPy Data":    [ ".psydat" ],
+    "QC":               [ ".qa.png", ".qa.json", ".qa.html", ".qc.png", ".qc.json", ".qc.html" ],
 
-    "archive":      [ ".zip", ".tbz2", ".tar.gz", ".tbz", ".tar.bz2", ".tgz", ".tar", ".txz", ".tar.xz" ],
-    "document":     [ ".docx", ".doc" ],
-    "image":        [ ".jpg", ".tif", ".jpeg", ".gif", ".bmp", ".png", ".tiff" ],
-    "markup":       [ ".html", ".htm", ".xml"  ],
-    "markdown":     [ ".md", ".markdown"  ],
-    "log":          [ ".log" ],
-    "pdf":          [ ".pdf" ],
-    "presentation": [ ".ppt", ".pptx" ],
-    "source code":  [ ".c", ".py", ".cpp", ".js", ".m", ".json", ".java", ".php", ".css", ".toml", ".yaml", ".yml" ],
-    "spreadsheet":  [ ".xls", ".xlsx" ],
-    "tabular data": [ ".csv.gz", ".csv" ],
-    "text":         [ ".txt" ],
-    "video":        [ ".mpeg", ".mpg", ".mov", ".mp4", ".m4v", ".mts" ]
+    "Archive":          [ ".zip", ".tbz2", ".tar.gz", ".tbz", ".tar.bz2", ".tgz", ".tar", ".txz", ".tar.xz" ],
+    "Audio":            [ ".mp3", ".wav", ".wave" ],
+    "Document":         [ ".docx", ".doc" ],
+    "Image":            [ ".jpg", ".tif", ".jpeg", ".gif", ".bmp", ".png", ".tiff" ],
+    "Log File":         [ ".log" ],
+    "Markdown":         [ ".md", ".markdown" ],
+    "Markup":           [ ".html", ".htm", ".xml" ],
+    "PDF":              [ ".pdf" ],
+    "Plain Text":       [ ".txt" ],
+    "Presentation":     [ ".ppt", ".pptx" ],
+    "Source Code":      [ ".c", ".py", ".cpp", ".js", ".m", ".json", ".java", ".php", ".css", ".toml", ".yaml", ".yml" ],
+    "Spreadsheet":      [ ".xls", ".xlsx" ],
+    "Tabular Data":     [ ".csv.gz", ".csv", ".tsv.gz", ".tsv" ],
+    "Video":            [ ".mpeg", ".mpg", ".mov", ".mp4", ".m4v", ".mts" ]
 }

api/handlers/listhandler.py

@@ -18,6 +18,7 @@
 from ..dao import liststorage
 from ..dao import APIStorageException
 from ..dao import hierarchy
+from .modalityhandler import ModalityHandler
 from ..web.request import log_access, AccessType
 
 
@@ -38,7 +39,7 @@ def initialize_list_configurations():
             'input_schema_file': 'tag.json'
         },
         'files': {
-            'storage': liststorage.ListStorage,
+            'storage': liststorage.FileStorage,
             'permchecker': listauth.default_sublist,
             'use_object_id': True,
             'storage_schema_file': 'file.json',
@@ -502,6 +503,38 @@ def post(self, cont_name, list_name, **kwargs):
 
         return upload.process_upload(self.request, upload.Strategy.targeted, container_type=cont_name, id_=_id, origin=self.origin)
 
+    def put(self, cont_name, list_name, **kwargs):
+        _id = kwargs.pop('cid')
+        permchecker, storage, mongo_validator, payload_validator, keycheck = self._initialize_request(cont_name, list_name, _id, query_params=kwargs)
+
+        payload = self.request.json_body
+        payload_validator(payload, 'PUT')
+        if not set(payload.keys()).issubset({'info', 'modality', 'classification'}):
+            self.abort(400, 'Can only update info, modality an classification keys on file.')
+
+        classification = payload.get('classification')
+        if classification:
+            modality = payload.get('modality')
+            if not modality:
+                file_obj = storage.exec_op('GET', _id, query_params=kwargs)
+                modality = file_obj.get('modality')
+
+            if not ModalityHandler.check_classification(modality, classification):
+                self.abort(400, 'Classification does not match allowable values for modality {}.'.format(modality))
+
+        rf = self.is_true('replace_fields')
+
+        try:
+            keycheck(mongo_validator(permchecker(noop)))('PUT', _id=_id, query_params=kwargs, payload=payload)
+            result = storage.update_file(_id, kwargs, payload, replace_fields=rf)
+        except APIStorageException as e:
+            self.abort(400, e.message)
+        # abort if the query of the update wasn't able to find any matching documents
+        if result.matched_count == 0:
+            self.abort(404, 'Element not updated in list {} of container {} {}'.format(storage.list_name, storage.cont_name, _id))
+        else:
+            return {'modified':result.modified_count}
+
     def delete(self, cont_name, list_name, **kwargs):
         # Overriding base class delete to audit action before completion
         _id = kwargs.pop('cid')

api/handlers/modalityhandler.py

@@ -0,0 +1,112 @@
+from ..web import base
+from .. import config
+from ..auth import require_login, require_superuser
+from ..dao import containerstorage, APINotFoundException
+# from ..validators import validate_data
+
+log = config.log
+
+
+class ModalityHandler(base.RequestHandler):
+
+    def __init__(self, request=None, response=None):
+        super(ModalityHandler, self).__init__(request, response)
+        self.storage = containerstorage.ContainerStorage('modalities', use_object_id=False)
+
+    @require_login
+    def get(self, modality_name):
+        return self.storage.get_container(modality_name)
+
+    @require_login
+    def get_all(self):
+        return self.storage.get_all_el(None, None, None)
+
+    @require_superuser
+    def post(self):
+        payload = self.request.json_body
+        # Clean this up when validate_data method is fixed to use new schemas
+        # POST unnecessary, used to avoid run-time modification of schema
+        #validate_data(payload, 'modality.json', 'input', 'POST', optional=True)
+
+        result = self.storage.create_el(payload)
+        if result.acknowledged:
+            return {'_id': result.inserted_id}
+        else:
+            self.abort(400, 'Modality not inserted')
+
+    @require_superuser
+    def put(self, modality_name):
+        payload = self.request.json_body
+        # Clean this up when validate_data method is fixed to use new schemas
+        # POST unnecessary, used to avoid run-time modification of schema
+        #validate_data(payload, 'modality.json', 'input', 'POST', optional=True)
+
+        result = self.storage.update_el(modality_name, payload)
+        if result.matched_count == 1:
+            return {'modified': result.modified_count}
+        else:
+            raise APINotFoundException('Modality with name {} not found, modality not updated'.format(modality_name))
+
+    @require_superuser
+    def delete(self, modality_name):
+        result = self.storage.delete_el(modality_name)
+        if result.deleted_count == 1:
+            return {'deleted': result.deleted_count}
+        else:
+            raise APINotFoundException('Modality with name {} not found, modality not deleted'.format(modality_name))
+
+    @staticmethod
+    def check_classification(modality_name, classification_map):
+        """
+        Given a modality name and a proposed classification map,
+        ensure:
+          - that a modality exists with that name and has a classification
+            map
+          - all keys in the classification_map exist in the
+            `classifications` map on the modality object
+          - all the values in the arrays in the classification_map
+            exist in the modality's classifications map
+
+        For example:
+            Modality = {
+                "_id" = "Example_modality",
+                "classifications": {
+                    "Example1": ["Blue", "Green"]
+                    "Example2": ["one", "two"]
+                }
+            }
+
+        Returns True:
+            classification_map = {
+                "Example1": ["Blue"],
+                "custom":   ["anything"]
+            }
+
+        Returns False:
+            classification_map = {
+                "Example1": ["Red"], # "Red" is not allowed
+                "Example2": ["one", "two"]
+            }
+        """
+        try:
+            modality = containerstorage.ContainerStorage('modalities', use_object_id=False).get_container(modality_name)
+        except APINotFoundException:
+            if classification_map.keys() == ['custom']:
+                # for unknown modalities allow only list of custom values
+                return True
+            else:
+                return False
+
+        classifications = modality.get('classifications')
+        if not classifications:
+            return False
+
+        for k,array in classification_map.iteritems():
+            if k == 'custom':
+                # any unique value is allowed in custom list
+                continue
+            possible_values = classifications.get(k, [])
+            if not set(array).issubset(set(possible_values)):
+                return False
+
+        return True

api/jobs/rules.py

@@ -73,16 +73,16 @@ def lower(x):
     elif match_type == 'file.name':
         return fnmatch.fnmatch(file_['name'].lower(), match_param.lower())
 
-    # Match any of the file's measurements
-    elif match_type == 'file.measurements':
-        try:
-            if match_param:
-                return match_param.lower() in map(lower, file_.get('measurements', []))
-            else:
-                return False
-        except KeyError:
-            _log_file_key_error(file_, container, 'has no measurements key')
+    # Match any of the file's classifications
+    elif match_type == 'file.classification':
+        for v in file_.get('classification', {}).itervalues():
+            try:
+                if match_param and match_param.lower() in map(lower, v):
+                    return True
+            except KeyError:
+                _log_file_key_error(file_, container, 'has no measurements key')
             return False
+        return False
 
     # Match the container having any file (including this one) with this type
     elif match_type == 'container.has-type':

api/placer.py

@@ -242,15 +242,16 @@ def check(self):
                 job = Job.get(self.context.get('job_id'))
                 input_names = [{'name': v.name} for v in job.inputs.itervalues()]
 
-                measurement = self.metadata.get(self.container_type, {}).pop('measurement', None)
+                classification = self.metadata.get(self.container_type, {}).pop('measurement', None)
                 info = self.metadata.get(self.container_type,{}).pop('metadata', None)
                 modality = self.metadata.get(self.container_type, {}).pop('instrument', None)
-                if measurement or info or modality:
+                if classification or info or modality:
                     files_ = self.metadata[self.container_type].get('files', [])
                     files_ += input_names
                     for f in files_:
-                        if measurement:
-                            f['measurements'] = [measurement]
+                        if classification:
+                            custom = {'custom': [classification]}
+                            f['classification'] = custom
                         if info:
                             f['info'] = info
                         if modality:
@@ -502,7 +503,7 @@ def finalize(self):
 
             # OPPORTUNITY: packfile endpoint could be extended someday to take additional metadata.
             'modality': None,
-            'measurements': [],
+            'classification': {},
             'tags': [],
             'info': {},
 

api/upload.py

@@ -127,7 +127,7 @@ def process_upload(request, strategy, container_type=None, id_=None, origin=None
 
             'type': None,
             'modality': None,
-            'measurements': [],
+            'classification': {},
             'tags': [],
             'info': {}
         }