Update dm to use classification

Author: nagem

api/dao/liststorage.py

@@ -3,6 +3,7 @@
 import datetime
 
 from .. import config
+from .. import util
 from . import consistencychecker, containerutil
 from . import APIStorageException, APIConflictException
 from .containerstorage import SessionStorage, AcquisitionStorage
@@ -46,7 +47,7 @@ def get_container(self, _id, query_params=None):
         log.debug('query {}'.format(query))
         return self.dbc.find_one(query, projection)
 
-    def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_params=None, replace_classification=False):
+    def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_params=None):
         """
         Generic method to exec an operation.
         The request is dispatched to the corresponding private methods.
@@ -63,7 +64,7 @@ def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_par
         if action == 'DELETE':
             return self._delete_el(_id, query_params)
         if action == 'PUT':
-            return self._update_el(_id, query_params, payload, exclude_params, replace_classification=replace_classification)
+            return self._update_el(_id, query_params, payload, exclude_params)
         if action == 'POST':
             return self._create_el(_id, payload, exclude_params)
         raise ValueError('action should be one of GET, POST, PUT, DELETE')
@@ -346,20 +347,21 @@ def inflate_job_info(analysis):
 
 class FileStorage(ListStorage):
 
-    def _update_el(self, _id, query_params, payload, exclude_params, replace_classification=False):
+    def update_file(self, _id, query_params, payload, replace_fields=False):
         mod_elem = {}
         update = {}
 
         # If we want to add to the classification lists rather than replace
         # the entirity of the classification map, use $addToSet.
         # This allows some endpoints to only make additive changes
-        if not replace_classification:
-            classification = payload.get('classification')
+        if not replace_fields:
+            classification = payload.pop('classification', None)
             if classification:
                 add_to_set = {}
                 for k,array in classification.items():
                     add_to_set[self.list_name + '.$.classification.' + k] = array
                 update['$addToSet'] = add_to_set
+            payload = util.mongo_dict(payload)
 
         for k,v in payload.items():
             mod_elem[self.list_name + '.$.' + k] = v

api/handlers/listhandler.py

@@ -521,10 +521,11 @@ def put(self, cont_name, list_name, **kwargs):
             if not ModalityHandler.check_classification(modality, classification):
                 self.abort(400, 'Classification does not match allowable values for modality {}.'.format(modality))
 
-        rc = self.is_true('replace_classification')
+        rf = self.is_true('replace_fields')
 
         try:
-            result = keycheck(mongo_validator(permchecker(storage.exec_op)))('PUT', _id=_id, query_params=kwargs, payload=payload)
+            keycheck(mongo_validator(permchecker(noop)))('PUT', _id=_id, query_params=kwargs, payload=payload)
+            result = storage.update_file(_id, kwargs, payload, replace_fields=rf)
         except APIStorageException as e:
             self.abort(400, e.message)
         # abort if the query of the update wasn't able to find any matching documents

api/handlers/modalityhandler.py

@@ -1,11 +1,8 @@
-import datetime as dt
-
 from ..web import base
 from .. import config
-from .. import util
-from ..auth import require_drone, require_login, require_superuser
+from ..auth import require_login, require_superuser
 from ..dao import containerstorage, APINotFoundException
-from ..validators import validate_data
+# from ..validators import validate_data
 
 log = config.log
 

api/placer.py

@@ -242,15 +242,16 @@ def check(self):
                 job = Job.get(self.context.get('job_id'))
                 input_names = [{'name': v.name} for v in job.inputs.itervalues()]
 
-                measurement = self.metadata.get(self.container_type, {}).pop('measurement', None)
+                classification = self.metadata.get(self.container_type, {}).pop('measurement', None)
                 info = self.metadata.get(self.container_type,{}).pop('metadata', None)
                 modality = self.metadata.get(self.container_type, {}).pop('instrument', None)
-                if measurement or info or modality:
+                if classification or info or modality:
                     files_ = self.metadata[self.container_type].get('files', [])
                     files_ += input_names
                     for f in files_:
-                        if measurement:
-                            f['measurements'] = [measurement]
+                        if classification:
+                            custom = {'custom': [classification]}
+                            f['classification'] = custom
                         if info:
                             f['info'] = info
                         if modality:
@@ -502,7 +503,7 @@ def finalize(self):
 
             # OPPORTUNITY: packfile endpoint could be extended someday to take additional metadata.
             'modality': None,
-            'measurements': [],
+            'classification': {},
             'tags': [],
             'info': {},
 

api/upload.py

@@ -119,7 +119,7 @@ def process_upload(request, strategy, container_type=None, id_=None, origin=None
 
             'type': None,
             'modality': None,
-            'measurements': [],
+            'classification': {},
             'tags': [],
             'info': {}
         }

bin/database.py

@@ -14,7 +14,7 @@
 from api.jobs import gears
 from api.types import Origin
 
-CURRENT_DATABASE_VERSION = 21 # An int that is bumped when a new schema change is made
+CURRENT_DATABASE_VERSION = 22 # An int that is bumped when a new schema change is made
 
 def get_db_version():
 
@@ -636,6 +636,58 @@ def dm_v2_updates(cont_list, cont_name):
     dm_v2_updates(config.db.acquisitions.find(query), 'acquisitions')
 
 
+def upgrade_to_22():
+    """
+    Change file `measurement` field to `classification` map
+    Place all measurements in `custom` key on map
+    """
+
+    def update_project_template(template):
+        for a in template.get('acquisitions', []):
+            new_file_templates = []
+            for f in a.get('files', []):
+                if f.get('measurement'):
+                    measurements = f.pop('measurement')
+                    f['classification'] = {'custom': measurement}
+
+        return template
+
+
+    def change_to_classification(cont_list, cont_name):
+        for container in cont_list:
+
+            if cont_name == 'projects' and container.get('template'):
+                new_template = update_project_template(json.loads(container.get('template')))
+                update['$set'] = {'template': new_template}
+
+            query = {'_id': container['_id']}
+            update = {}
+
+            files = container.get('files')
+            if files is not None:
+                updated_files = []
+                for file_ in files:
+                    if 'measurements' in file_:
+                        measurements = file_.pop('measurements', [])
+                        custom = {'custom': measurements}
+                        file_['classification'] = custom
+
+                    updated_files.append(file_)
+                if update.get('$set'):
+                    update['$set']['files'] =  updated_files
+                else:
+                    update['$set'] = {'files': updated_files}
+
+            result = config.db[cont_name].update_one(query, update)
+
+    query = {'files.measurements': { '$exists': True}}
+
+    change_to_classification(config.db.collections.find(query), 'collections')
+    change_to_classification(config.db.projects.find(query), 'projects')
+    change_to_classification(config.db.sessions.find(query), 'sessions')
+    change_to_classification(config.db.acquisitions.find(query), 'acquisitions')
+
+
 def upgrade_schema():
     """
     Upgrades db to the current schema version

raml/examples/file_info_list.json

@@ -8,7 +8,7 @@
       "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
       "name": "cortical_surface_right_hemisphere.obj",
       "tags": [],
-      "measurements": [],
+      "classification": {},
       "modified": "2016-10-18T15:26:35.701000+00:00",
       "instrument": null,
       "size": 21804112,
@@ -23,7 +23,7 @@
       "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
       "name": "cortical_surface_right_hemisphere.obj",
       "tags": [],
-      "measurements": [],
+      "classification": {},
       "modified": "2016-10-18T17:45:17.776000+00:00",
       "instrument": null,
       "metadata": {},

raml/examples/output/acquisition-list.json

@@ -9,7 +9,7 @@
           "name": "Admin Import"
         },
         "mimetype": "application/zip",
-        "measurements": [],
+        "classification": {},
         "hash": "v0-sha384-dd3c97bfe0ad1fcba75ae6718c6e81038c59af4f447f5db194d52732efa4f955b28455db02eb64cad3e4e55f11e3679f",
         "name": "4784_1_1_localizer_dicom.zip",
         "tags": [],
@@ -49,7 +49,7 @@
           "name": "Admin Import"
         },
         "mimetype": "application/zip",
-        "measurements": [],
+        "classification": {},
         "hash": "v0-sha384-ca055fb36845db86e4278cf6e185f8674d11a96f4b29af27e401fc495cc82ef6b53a5729c3757713064649dc71c8c725",
         "name": "4784_3_1_t1_dicom.zip",
         "tags": [],
@@ -89,7 +89,7 @@
           "name": "Admin Import"
         },
         "mimetype": "application/zip",
-        "measurements": [],
+        "classification": {},
         "hash": "v0-sha384-537e42b1dd8f1feef9844fbfb4f60461361e71cafa7055556097e9d0b9f7fac68c8f234ed126af9412bd43a548948847",
         "name": "4784_5_1_fmri_dicom.zip",
         "tags": [],

raml/examples/output/acquisition.json

@@ -8,7 +8,7 @@
         "name": "Admin Import"
       },
       "mimetype": "application/zip",
-      "measurements": [],
+      "classification": {},
       "hash": "v0-sha384-dd3c97bfe0ad1fcba75ae6718c6e81038c59af4f447f5db194d52732efa4f955b28455db02eb64cad3e4e55f11e3679f",
       "name": "4784_1_1_localizer_dicom.zip",
       "tags": [],

raml/examples/output/analysis-item.json

@@ -8,7 +8,7 @@
         "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
         "name": "cortical_surface_right_hemisphere.obj",
         "tags": [],
-        "measurements": [],
+        "classification": {},
         "modified": "2016-10-18T15:26:35.701000+00:00",
         "instrument": null,
         "input": true,
@@ -24,7 +24,7 @@
         "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
         "name": "cortical_surface_right_hemisphere.obj",
         "tags": [],
-        "measurements": [],
+        "classification": {},
         "modified": "2016-10-18T17:45:17.776000+00:00",
         "instrument": null,
         "output": true,

raml/schemas/definitions/analysis.json

@@ -49,7 +49,7 @@
 			                        {"type":"null"}
 			                    ]
 			                },
-			                "measurements": {"$ref":"../definitions/file.json#/definitions/measurements"},
+			                "classification": {"$ref":"../definitions/file.json#/definitions/classification"},
 			                "tags": {"$ref":"../definitions/file.json#/definitions/tags"},
 			                "info": {"$ref":"../definitions/file.json#/definitions/info"},
 			                "origin":{"$ref":"../definitions/file.json#/definitions/origin"},

raml/schemas/definitions/file.json

@@ -5,10 +5,8 @@
         "file-type":      { "type": "string" },
         "mimetype":       { "type": "string" },
         "modality":       { "type": "string" },
-        "measurements": {
-          "items": { "type": "string"},
-          "type": "array",
-          "uniqueItems": true
+        "classification": {
+          "type": "object"
         },
         "tags": {
           "items": { "type": "string"},
@@ -37,7 +35,7 @@
               "type":           {"$ref":"#/definitions/file-type"},
               "mimetype":       {"$ref":"#/definitions/mimetype"},
               "modality":       {"$ref":"#/definitions/modality"},
-              "measurements":   {"$ref":"#/definitions/measurements"},
+              "classification":   {"$ref":"#/definitions/classification"},
               "tags": {"$ref":"#/definitions/tags"},
               "info": {"$ref":"#/definitions/info"}
           },
@@ -55,7 +53,7 @@
                       {"type":"null"}
                   ]
               },
-              "measurements": {"$ref":"#/definitions/measurements"},
+              "classification": {"$ref":"#/definitions/classification"},
               "tags": {"$ref":"#/definitions/tags"},
               "info": {"$ref":"#/definitions/info"},
               "origin":{"$ref":"#/definitions/origin"},

raml/schemas/mongo/file.json

@@ -10,10 +10,8 @@
     "size":           { "type": "integer" },
     "hash":           { "type": "string" },
     "modality":       { "type": "string" },
-    "measurements": {
-      "items": { "type": "string"},
-      "type": "array",
-      "uniqueItems": true
+    "classification": {
+      "type": "object"
     },
     "tags": {
       "items": { "type": "string"},