Update dm to use classification

Author: nagem

api/placer.py

@@ -242,15 +242,16 @@ def check(self):
                 job = Job.get(self.context.get('job_id'))
                 input_names = [{'name': v.name} for v in job.inputs.itervalues()]
 
-                measurement = self.metadata.get(self.container_type, {}).pop('measurement', None)
+                classification = self.metadata.get(self.container_type, {}).pop('measurement', None)
                 info = self.metadata.get(self.container_type,{}).pop('metadata', None)
                 modality = self.metadata.get(self.container_type, {}).pop('instrument', None)
-                if measurement or info or modality:
+                if classification or info or modality:
                     files_ = self.metadata[self.container_type].get('files', [])
                     files_ += input_names
                     for f in files_:
-                        if measurement:
-                            f['measurements'] = [measurement]
+                        if classification:
+                            custom = {'custom': [classification]}
+                            f['classification'] = custom
                         if info:
                             f['info'] = info
                         if modality:
@@ -502,7 +503,7 @@ def finalize(self):
 
             # OPPORTUNITY: packfile endpoint could be extended someday to take additional metadata.
             'modality': None,
-            'measurements': [],
+            'classification': {},
             'tags': [],
             'info': {},
 

api/upload.py

@@ -119,7 +119,7 @@ def process_upload(request, strategy, container_type=None, id_=None, origin=None
 
             'type': None,
             'modality': None,
-            'measurements': [],
+            'classification': {},
             'tags': [],
             'info': {}
         }

bin/database.py

@@ -14,7 +14,7 @@
 from api.jobs import gears
 from api.types import Origin
 
-CURRENT_DATABASE_VERSION = 21 # An int that is bumped when a new schema change is made
+CURRENT_DATABASE_VERSION = 22 # An int that is bumped when a new schema change is made
 
 def get_db_version():
 
@@ -636,6 +636,58 @@ def dm_v2_updates(cont_list, cont_name):
     dm_v2_updates(config.db.acquisitions.find(query), 'acquisitions')
 
 
+def upgrade_to_22():
+    """
+    Change file `measurement` field to `classification` map
+    Place all measurements in `custom` key on map
+    """
+
+    def update_project_template(template):
+        for a in template.get('acquisitions', []):
+            new_file_templates = []
+            for f in a.get('files', []):
+                if f.get('measurement'):
+                    measurements = f.pop('measurement')
+                    f['classification'] = {'custom': measurement}
+
+        return template
+
+
+    def change_to_classification(cont_list, cont_name):
+        for container in cont_list:
+
+            if cont_name == 'projects' and container.get('template'):
+                new_template = update_project_template(json.loads(container.get('template')))
+                update['$set'] = {'template': new_template}
+
+            query = {'_id': container['_id']}
+            update = {}
+
+            files = container.get('files')
+            if files is not None:
+                updated_files = []
+                for file_ in files:
+                    if 'measurements' in file_:
+                        measurements = file_.pop('measurements', [])
+                        custom = {'custom': measurements}
+                        file_['classification'] = custom
+
+                    updated_files.append(file_)
+                if update.get('$set'):
+                    update['$set']['files'] =  updated_files
+                else:
+                    update['$set'] = {'files': updated_files}
+
+            result = config.db[cont_name].update_one(query, update)
+
+    query = {'files.measurements': { '$exists': True}}
+
+    change_to_classification(config.db.collections.find(query), 'collections')
+    change_to_classification(config.db.projects.find(query), 'projects')
+    change_to_classification(config.db.sessions.find(query), 'sessions')
+    change_to_classification(config.db.acquisitions.find(query), 'acquisitions')
+
+
 def upgrade_schema():
     """
     Upgrades db to the current schema version

raml/examples/file_info_list.json

@@ -8,7 +8,7 @@
       "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
       "name": "cortical_surface_right_hemisphere.obj",
       "tags": [],
-      "measurements": [],
+      "classification": {},
       "modified": "2016-10-18T15:26:35.701000+00:00",
       "instrument": null,
       "size": 21804112,
@@ -23,7 +23,7 @@
       "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
       "name": "cortical_surface_right_hemisphere.obj",
       "tags": [],
-      "measurements": [],
+      "classification": {},
       "modified": "2016-10-18T17:45:17.776000+00:00",
       "instrument": null,
       "metadata": {},

raml/examples/output/acquisition-list.json

@@ -9,7 +9,7 @@
           "name": "Admin Import"
         },
         "mimetype": "application/zip",
-        "measurements": [],
+        "classification": {},
         "hash": "v0-sha384-dd3c97bfe0ad1fcba75ae6718c6e81038c59af4f447f5db194d52732efa4f955b28455db02eb64cad3e4e55f11e3679f",
         "name": "4784_1_1_localizer_dicom.zip",
         "tags": [],
@@ -49,7 +49,7 @@
           "name": "Admin Import"
         },
         "mimetype": "application/zip",
-        "measurements": [],
+        "classification": {},
         "hash": "v0-sha384-ca055fb36845db86e4278cf6e185f8674d11a96f4b29af27e401fc495cc82ef6b53a5729c3757713064649dc71c8c725",
         "name": "4784_3_1_t1_dicom.zip",
         "tags": [],
@@ -89,7 +89,7 @@
           "name": "Admin Import"
         },
         "mimetype": "application/zip",
-        "measurements": [],
+        "classification": {},
         "hash": "v0-sha384-537e42b1dd8f1feef9844fbfb4f60461361e71cafa7055556097e9d0b9f7fac68c8f234ed126af9412bd43a548948847",
         "name": "4784_5_1_fmri_dicom.zip",
         "tags": [],

raml/examples/output/acquisition.json

@@ -8,7 +8,7 @@
         "name": "Admin Import"
       },
       "mimetype": "application/zip",
-      "measurements": [],
+      "classification": {},
       "hash": "v0-sha384-dd3c97bfe0ad1fcba75ae6718c6e81038c59af4f447f5db194d52732efa4f955b28455db02eb64cad3e4e55f11e3679f",
       "name": "4784_1_1_localizer_dicom.zip",
       "tags": [],

raml/examples/output/analysis-item.json

@@ -8,7 +8,7 @@
         "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
         "name": "cortical_surface_right_hemisphere.obj",
         "tags": [],
-        "measurements": [],
+        "classification": {},
         "modified": "2016-10-18T15:26:35.701000+00:00",
         "instrument": null,
         "input": true,
@@ -24,7 +24,7 @@
         "hash": "v0-sha384-12188e00a26650b2baa3f0195337dcf504f4362bb2136eef0cdbefb57159356b1355a0402fca0ab5ab081f21c305e5c2",
         "name": "cortical_surface_right_hemisphere.obj",
         "tags": [],
-        "measurements": [],
+        "classification": {},
         "modified": "2016-10-18T17:45:17.776000+00:00",
         "instrument": null,
         "output": true,

raml/schemas/definitions/analysis.json

@@ -49,7 +49,7 @@
 			                        {"type":"null"}
 			                    ]
 			                },
-			                "measurements": {"$ref":"../definitions/file.json#/definitions/measurements"},
+			                "classification": {"$ref":"../definitions/file.json#/definitions/classification"},
 			                "tags": {"$ref":"../definitions/file.json#/definitions/tags"},
 			                "info": {"$ref":"../definitions/file.json#/definitions/info"},
 			                "origin":{"$ref":"../definitions/file.json#/definitions/origin"},

raml/schemas/definitions/file.json

@@ -5,10 +5,8 @@
         "file-type":      { "type": "string" },
         "mimetype":       { "type": "string" },
         "modality":       { "type": "string" },
-        "measurements": {
-          "items": { "type": "string"},
-          "type": "array",
-          "uniqueItems": true
+        "classification": {
+          "type": "object"
         },
         "tags": {
           "items": { "type": "string"},
@@ -37,7 +35,7 @@
               "type":           {"$ref":"#/definitions/file-type"},
               "mimetype":       {"$ref":"#/definitions/mimetype"},
               "modality":       {"$ref":"#/definitions/modality"},
-              "measurements":   {"$ref":"#/definitions/measurements"},
+              "classification":   {"$ref":"#/definitions/classification"},
               "tags": {"$ref":"#/definitions/tags"},
               "info": {"$ref":"#/definitions/info"}
           },
@@ -55,7 +53,7 @@
                       {"type":"null"}
                   ]
               },
-              "measurements": {"$ref":"#/definitions/measurements"},
+              "classification": {"$ref":"#/definitions/classification"},
               "tags": {"$ref":"#/definitions/tags"},
               "info": {"$ref":"#/definitions/info"},
               "origin":{"$ref":"#/definitions/origin"},

raml/schemas/mongo/file.json

@@ -10,10 +10,8 @@
     "size":           { "type": "integer" },
     "hash":           { "type": "string" },
     "modality":       { "type": "string" },
-    "measurements": {
-      "items": { "type": "string"},
-      "type": "array",
-      "uniqueItems": true
+    "classification": {
+      "type": "object"
     },
     "tags": {
       "items": { "type": "string"},