Merged Carol's changes

Author: pintohutch

nipype/interfaces/tests/test_io.py

@@ -94,7 +94,7 @@ def test_selectfiles_valueerror():
     yield assert_raises, ValueError, sf.run
 
 
-@skipif(noboto)
+@skip
 def test_s3datagrabber_communication():
     dg = nio.S3DataGrabber(
         infields=['subj_id', 'run_num'], outfields=['func', 'struct'])

nipype/pipeline/plugins/callback_log.py

@@ -13,27 +13,27 @@ def log_nodes_cb(node, status, result=None):
     if status == 'start':
         message  = '{"name":' + '"' + node.name + '"' + ',"id":' + '"' +\
         node._id + '"' + ',"start":' + '"' +str(datetime.datetime.now()) +\
-        '"' + ',"estimate memory":' + str(node._interface.estimated_memory) + ',"num_threads":' \
+        '"' + ',"estimated_memory":' + str(node._interface.estimated_memory) + ',"num_threads":' \
         + str(node._interface.num_threads) + '}'
 
         logger.debug(message)
 
     elif status == 'end':
         message  = '{"name":' + '"' + node.name + '"' + ',"id":' + '"' + \
-        node._id + '"' + ',"finish":' + '"' + str(datetime.datetime.now()) + \
-        '"' + ',"estimate memory":' + str(node._interface.estimated_memory) + \
-        ',"num_threads":' + str(node._interface.num_threads) + \
-        ',"cmd-level threads":' + str(cmd_threads) + \
-        ',"node-level memory":' + str(node_mem) + \
-        ',"cmd-level memory":' + str(cmd_mem) + \
-        ',"run_seconds":' + str(run_seconds) + '}'
+        node._id + '"' + ',"finish":' + '"' + str(datetime.datetime.now()) +  \
+        '"' + ',"estimated_memory":' + '"'+ str(node._interface.estimated_memory) + '"'+ \
+        ',"num_threads":' + '"'+ str(node._interface.num_threads) + '"'+ \
+        ',"cmd-level_threads":' + '"'+ str(cmd_threads) + '"'+ \
+        ',"node-level_memory":' + '"'+ str(node_mem) + '"'+ \
+        ',"cmd-level_memory":' + '"'+ str(cmd_mem) + '"' + \
+        ',"run_seconds":' + '"'+ str(run_seconds) + '"'+ '}'
 
         logger.debug(message)
 
     else:
         message  = '{"name":' + '"' + node.name + '"' + ',"id":' + '"' + \
         node._id + '"' + ',"finish":' + '"' + str(datetime.datetime.now()) +\
-        '"' + ',"estimate memory":' + str(node._interface.estimated_memory) + ',"num_threads":' \
+        '"' + ',"estimated_memory":' + str(node._interface.estimated_memory) + ',"num_threads":' \
         + str(node._interface.num_threads) + ',"error":"True"}'
 
         logger.debug(message)

nipype/pipeline/plugins/multiproc.py

@@ -9,7 +9,14 @@
 from multiprocessing import Process, Pool, cpu_count, pool
 from traceback import format_exception
 import sys
-
+import numpy as np
+from copy import deepcopy
+from ..engine import MapNode
+from ...utils.misc import str2bool
+import datetime
+import psutil
+from ... import logging
+import semaphore_singleton
 from .base import (DistributedPluginBase, report_crash)
 
 
@@ -76,14 +83,6 @@ class NonDaemonPool(pool.Pool):
     """
     Process = NonDaemonProcess
 
-import numpy as np
-from copy import deepcopy
-from ..engine import MapNode
-from ...utils.misc import str2bool
-import datetime
-import psutil
-from ... import logging
-import semaphore_singleton
 logger = logging.getLogger('workflow')
 
 def release_lock(args):

nipype/pipeline/plugins/tests/test_multiproc.py

@@ -6,7 +6,7 @@
 from nipype.testing import assert_equal, assert_less_equal
 import nipype.pipeline.engine as pe
 
-
+ 
 class InputSpec(nib.TraitedSpec):
     input1 = nib.traits.Int(desc='a random int')
     input2 = nib.traits.Int(desc='a random int')
@@ -182,6 +182,8 @@ def test_do_not_use_more_memory_then_specified():
     os.remove(LOG_FILENAME)
 
 
+
+
 def test_do_not_use_more_threads_then_specified():
     LOG_FILENAME = 'callback.log'
     my_logger = logging.getLogger('callback')
@@ -231,4 +233,4 @@ def test_do_not_use_more_threads_then_specified():
             break
     yield assert_equal, result, True, "using more memory than system has (memory is not specified by user)"
 
-    os.remove(LOG_FILENAME)
+    os.remove(LOG_FILENAME)

nipype/utils/draw_gantt_chart.py

@@ -3,32 +3,39 @@
 import datetime
 import random
 
-
+ 
 def log_to_json(logfile):
     result = []
     with open(logfile, 'r') as content:
 
-            #read file separating each line
-            content = content.read()
-            lines = content.split('\n')
-
-            lines = [ json.loads(x) for x in lines[:-1]]
-
-            last_node = [ x for x in lines if x.has_key('finish')][-1]
-
-            for i, line in enumerate(lines):
-                #get first start it finds
-                if not line.has_key('start'):
-                    continue
-
-                #fint the end node for that start
-                for j in range(i+1, len(lines)):
-                    if lines[j].has_key('finish'):
-                        if lines[j]['id'] == line['id'] and lines[j]['name'] == line['name']:
-                            line['finish'] = lines[j]['finish']
-                            line['duration'] = (parser.parse(line['finish']) - parser.parse(line['start'])).total_seconds()
-                            result.append(line)
-                            break
+        #read file separating each line
+        content = content.read()
+        lines = content.split('\n')
+        l = []
+        for i in lines:
+            try:
+                y = json.loads(i)
+                l.append(y)
+            except Exception, e:
+                pass
+
+        lines = l
+
+        last_node = [ x for x in lines if x.has_key('finish')][-1]
+
+        for i, line in enumerate(lines):
+            #get first start it finds
+            if not line.has_key('start'):
+                continue
+
+            #fint the end node for that start
+            for j in range(i+1, len(lines)):
+                if lines[j].has_key('finish'):
+                    if lines[j]['id'] == line['id'] and lines[j]['name'] == line['name']:
+                        line['finish'] = lines[j]['finish']
+                        line['duration'] = (parser.parse(line['finish']) - parser.parse(line['start'])).total_seconds()
+                        result.append(line)
+                        break
 
     return result, last_node