Scyjava Refactor

cellprofiler_core/analysis/_runner.py

@@ -8,6 +8,7 @@
 import tempfile
 import threading
 from typing import List, Any
+import re
 
 import numpy
 import psutil
@@ -21,7 +22,6 @@
 from . import request as anarequest
 from ..image import ImageSetList
 from ..measurement import Measurements
-from ..utilities.java import JAVA_STARTED
 from ..utilities.measurement import load_measurements_from_buffer
 from ..pipeline import dump
 from ..preferences import get_plugin_directory
@@ -38,6 +38,8 @@
 from ..workspace import Workspace
 
 
+LOGGER = logging.getLogger(__name__)
+
 class Runner:
     """The Runner manages two threads (per instance) and all of the
     workers (per class, i.e., singletons).
@@ -167,22 +169,22 @@ def notify_threads(self):
 
     def cancel(self):
         """cancel the analysis run"""
-        logging.debug("Stopping workers")
+        LOGGER.debug("Stopping workers")
         self.stop_workers()
-        logging.debug("Canceling run")
+        LOGGER.debug("Canceling run")
         self.cancelled = True
         self.paused = False
         self.notify_threads()
-        logging.debug("Waiting on interface thread")
+        LOGGER.debug("Waiting on interface thread")
         self.interface_thread.join()
-        logging.debug("Waiting on jobserver thread")
+        LOGGER.debug("Waiting on jobserver thread")
         self.jobserver_thread.join()
         self.interface_thread = None
         self.jobserver_thread = None
         self.work_queue = queue.Queue()
         self.in_process_queue = queue.Queue()
         self.finished_queue = queue.Queue()
-        logging.debug("Cancel complete")
+        LOGGER.debug("Cancel complete")
 
     def pause(self):
         """pause the analysis run"""
@@ -214,8 +216,6 @@ def interface(
         image_set_end - last image set number to process
         overwrite - whether to recompute imagesets that already have data in initial_measurements.
         """
-        from javabridge import attach, detach
-
         posted_analysis_started = False
         acknowledged_thread_start = False
         measurements = None
@@ -426,9 +426,6 @@ def interface(
         except Exception as e:
             print(e)
         finally:
-            if JAVA_STARTED:
-                import javabridge
-                javabridge.detach()
             # Note - the measurements file is owned by the queue consumer
             #        after this post_event.
             #
@@ -529,21 +526,21 @@ def jobserver(self, start_signal):
                 continue
 
             if isinstance(req, anarequest.PipelinePreferences):
-                logging.debug("Received pipeline preferences request")
+                LOGGER.debug("Received pipeline preferences request")
                 req.reply(
                     Reply(
                         pipeline_blob=numpy.array(self.pipeline_as_string()),
                         preferences=preferences_as_dict(),
                     )
                 )
-                logging.debug("Replied to pipeline preferences request")
+                LOGGER.debug("Replied to pipeline preferences request")
             elif isinstance(req, anarequest.InitialMeasurements):
-                logging.debug("Received initial measurements request")
+                LOGGER.debug("Received initial measurements request")
                 req.reply(Reply(buf=self.initial_measurements_buf))
-                logging.debug("Replied to initial measurements request")
+                LOGGER.debug("Replied to initial measurements request")
             elif isinstance(req, anarequest.Work):
                 if not self.work_queue.empty():
-                    logging.debug("Received work request")
+                    LOGGER.debug("Received work request")
                     (
                         job,
                         worker_runs_post_group,
@@ -557,7 +554,7 @@ def jobserver(self, start_signal):
                         )
                     )
                     self.queue_dispatched_job(job)
-                    logging.debug(
+                    LOGGER.debug(
                         "Dispatched job: image sets=%s"
                         % ",".join([str(i) for i in job])
                     )
@@ -568,21 +565,21 @@ def jobserver(self, start_signal):
             elif isinstance(req, anareply.ImageSetSuccess):
                 # interface() is responsible for replying, to allow it to
                 # request the shared_state dictionary if needed.
-                logging.debug("Received ImageSetSuccess")
+                LOGGER.debug("Received ImageSetSuccess")
                 self.queue_imageset_finished(req)
-                logging.debug("Enqueued ImageSetSuccess")
+                LOGGER.debug("Enqueued ImageSetSuccess")
             elif isinstance(req, anarequest.SharedDictionary):
-                logging.debug("Received shared dictionary request")
+                LOGGER.debug("Received shared dictionary request")
                 req.reply(anareply.SharedDictionary(dictionaries=self.shared_dicts))
-                logging.debug("Sent shared dictionary reply")
+                LOGGER.debug("Sent shared dictionary reply")
             elif isinstance(req, anarequest.MeasurementsReport):
-                logging.debug("Received measurements report")
+                LOGGER.debug("Received measurements report")
                 self.queue_received_measurements(req.image_set_numbers, req.buf)
                 req.reply(anareply.Ack())
-                logging.debug("Acknowledged measurements report")
+                LOGGER.debug("Acknowledged measurements report")
             elif isinstance(req, anarequest.AnalysisCancel):
                 # Signal the interface that we are cancelling
-                logging.debug("Received analysis worker cancel request")
+                LOGGER.debug("Received analysis worker cancel request")
                 with self.interface_work_cv:
                     self.cancelled = True
                     self.interface_work_cv.notify()
@@ -599,13 +596,13 @@ def jobserver(self, start_signal):
                     anarequest.OmeroLogin,
                 ),
             ):
-                logging.debug("Enqueueing interactive request")
+                LOGGER.debug("Enqueueing interactive request")
                 # bump upward
                 self.post_event(req)
-                logging.debug("Interactive request enqueued")
+                LOGGER.debug("Interactive request enqueued")
             else:
                 msg = "Unknown request from worker: %s of type %s" % (req, type(req))
-                logging.error(msg)
+                LOGGER.error(msg)
                 raise ValueError(msg)
 
         # stop the ZMQ-boundary thread - will also deal with any requests waiting on replies
@@ -647,7 +644,7 @@ def start_workers(self, num=None):
 
         boundary = self.boundary
 
-        logging.info("Starting workers on address %s" % boundary.request_address)
+        LOGGER.info("Starting workers on address %s" % boundary.request_address)
 
         close_fds = False
 
@@ -663,7 +660,9 @@ def start_workers(self, num=None):
             "--conserve-memory",
             str(get_conserve_memory()),
             "--always-continue",
-            str(get_always_continue())
+            str(get_always_continue()),
+            "--log-level",
+            str(logging.root.level)
         ]
 
         # start workers
@@ -707,8 +706,18 @@ def run_logger(workR, widx):
                         line = line.decode("utf-8")
                         if not line:
                             break
-                        logging.info("Worker %d: %s", widx, line.rstrip())
-                    except:
+                        log_msg_match = re.match(fr"{workR.pid}\|(10|20|30|40|50)\|(.*)", line)
+                        if log_msg_match:
+                            levelno = int(log_msg_match.group(1))
+                            msg = log_msg_match.group(2)
+                        else:
+                            levelno = 20
+                            msg = line
+
+                        LOGGER.log(levelno, "\n\r  [Worker %d (%d)] %s", widx, workR.pid, msg.rstrip())
+
+                    except Exception as e:
+                        LOGGER.exception(e)
                         break
 
             start_daemon_thread(

cellprofiler_core/bioformats/__init__.py

@@ -0,0 +1 @@
+# NB: No implementation needed.

cellprofiler_core/bioformats/formatreader.py

@@ -0,0 +1,19 @@
+#TODO: unimplemented until CellProfiler/CellProfiler#4684 is resolved
+
+K_OMERO_SERVER = None
+K_OMERO_PORT = None
+K_OMERO_USER = None
+K_OMERO_SESSION_ID = None
+K_OMERO_CONFIG_FILE = None
+
+def clear_image_reader_cache():
+    raise RuntimeError("unimplemented")
+
+def set_omero_login_hook(omero_login):
+    raise RuntimeError("unimplemented")
+
+def get_omero_credentials():
+    raise RuntimeError("unimplemented")
+
+def use_omero_credentials(credentials):
+    raise RuntimeError("unimplemented")

cellprofiler_core/bioformats/formatwriter.py

@@ -0,0 +1,104 @@
+import os
+import scyjava
+import numpy as np
+import logging
+
+from . import omexml
+from ..utilities.java import jimport
+
+
+LOGGER = logging.getLogger(__name__)
+
+p2j = lambda v: scyjava.to_java(v)
+
+def write_image(
+    pathname,
+    pixels,
+    pixel_type,
+    c,
+    z,
+    t,
+    size_c,
+    size_z,
+    size_t,
+    channel_names
+):
+    # https://www.javadoc.io/doc/org.openmicroscopy/ome-common/5.3.2/loci/common/services/ServiceFactory.html
+    OMEXMLServiceFactory = jimport("loci.common.services.ServiceFactory")
+    # https://javadoc.scijava.org/Bio-Formats/loci/formats/services/OMEXMLService.html
+    OMEXMLService = jimport("loci.formats.services.OMEXMLService")
+    # https://javadoc.scijava.org/Bio-Formats/loci/formats/ImageWriter.html
+    ImageWriter = jimport("loci.formats.ImageWriter")
+    # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/IMetadata.html
+    IMetadata = jimport("loci.formats.meta.IMetadata")
+
+    DimensionsOrder = jimport("ome.xml.model.enums.DimensionOrder")
+    PixelType = jimport("ome.xml.model.enums.PixelType")
+    PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger")
+
+    omexml_service = OMEXMLServiceFactory().getInstance(OMEXMLService)
+    # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/OMEXMLMetadata.html
+    # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/MetadataStore.html
+    metadata = omexml_service.createOMEXMLMetadata()
+    metadata.createRoot()
+
+    metadata.setImageName(os.path.split(pathname)[1], 0)
+    metadata.setPixelsSizeX(PositiveInteger(p2j(pixels.shape[1])), 0)
+    metadata.setPixelsSizeY(PositiveInteger(p2j(pixels.shape[0])), 0)
+    metadata.setPixelsSizeC(PositiveInteger(p2j(size_c)), 0)
+    metadata.setPixelsSizeZ(PositiveInteger(p2j(size_z)), 0)
+    metadata.setPixelsSizeT(PositiveInteger(p2j(size_t)), 0)
+    metadata.setPixelsBinDataBigEndian(True, 0, 0)
+    metadata.setPixelsDimensionOrder(DimensionsOrder.XYCTZ, 0)
+    metadata.setPixelsType(PixelType.fromString(pixel_type), 0)
+
+
+    if pixels.ndim == 3:
+        metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0)
+        metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, 0)
+        omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2]))
+        # omexml.structured_annotations.add_original_metadata(
+        #     ome.OM_SAMPLES_PER_PIXEL, str(pixels.shape[2]))
+    elif size_c > 1:
+        # meta.channel_count = size_c <- cant find
+        metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0)
+        omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2]))
+
+    metadata.setImageID("Image:0", 0)
+    metadata.setPixelsID("Pixels:0", 0)
+
+    for i in range(size_c):
+        metadata.setChannelID(f"Channel:0:{i}", 0, i)
+        metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(1)), 0, i)
+
+    index = c + size_c * z + size_c * size_z * t
+    pixel_buffer = convert_pixels_to_buffer(pixels, pixel_type)
+
+
+    writer = ImageWriter()
+    writer.setMetadataRetrieve(metadata)
+    writer.setId(pathname)
+    writer.setInterleaved(True)
+    writer.saveBytes(index, pixel_buffer)
+    writer.close()
+
+def convert_pixels_to_buffer(pixels, pixel_type):
+    '''Convert the pixels in the image into a buffer of the right pixel type
+
+    pixels - a 2d monochrome or color image
+
+    pixel_type - one of the OME pixel types
+
+    returns a 1-d byte array
+    '''
+    if pixel_type == omexml.PT_UINT8:
+        as_dtype = np.uint8
+    elif pixel_type == omexml.PT_UINT16:
+        as_dtype = "<u2"
+    elif pixel_type == omexml.PT_FLOAT:
+        as_dtype = "<f4"
+    else:
+        raise NotImplementedError("Unsupported pixel type: %d" % pixel_type)
+
+    return np.frombuffer(np.ascontiguousarray(pixels, as_dtype).data, np.uint8)
+

cellprofiler_core/bioformats/omexml.py

@@ -0,0 +1,3 @@
+PT_UINT8 = "uint8"
+PT_UINT16 = "uint16"
+PT_FLOAT = "float"

cellprofiler_core/constants/image.py

@@ -1,5 +1,3 @@
-from bioformats import READABLE_FORMATS
-
 UIC1_TAG = 33628
 UIC2_TAG = 33629
 UIC3_TAG = 33630
@@ -86,6 +84,32 @@
     ".zvi",
 }
 
+# See https://docs.openmicroscopy.org/bio-formats/latest/supported-formats.html
+ALL_BIOFORMATS_EXTENSIONS = (
+    ".1sc", ".2fl", ".acff", ".afi", ".afm", ".aim", ".al3d", ".ali",
+    ".am", ".amiramesh", ".apl", ".arf", ".avi", ".bif", ".bin", ".bip",
+    ".bmp", ".btf", ".c01", ".cfg", ".ch5", ".cif", ".cr2", ".crw",
+    ".cxd", ".czi", ".dat", ".dcm", ".dib", ".dicom", ".dm2", ".dm3",
+    ".dm4", ".dti", ".dv", ".eps", ".epsi", ".exp", ".fdf", ".fff",
+    ".ffr", ".fits", ".flex", ".fli", ".frm", ".gel", ".gif", ".grey",
+    ".h5", ".hdf", ".hdr", ".hed", ".his", ".htd", ".html", ".hx", ".i2i",
+    ".ics", ".ids", ".im3", ".img", ".ims", ".inr", ".ipl", ".ipm", ".ipw",
+    ".j2k", ".jp2", ".jpeg", ".jpf", ".jpg", ".jpk", ".jpx", ".klb",
+    ".l2d", ".labels", ".lei", ".lif", ".liff", ".lim", ".lms", ".lsm",
+    ".map", ".mdb", ".mea", ".mnc", ".mng", ".mod", ".mov", ".mrc", ".mrcs",
+    ".mrw", ".msr", ".mtb", ".mvd2", ".naf", ".nd", ".nd2", ".ndpi", ".ndpis",
+    ".nef", ".nhdr", ".nii", ".nii.gz", ".nrrd", ".obf", ".obsep", ".oib",
+    ".oif", ".oir", ".ome", ".ome.btf", ".ome.tf2", ".ome.tf8", ".ome.tif",
+    ".ome.tiff", ".ome.xml", ".par", ".pbm", ".pcoraw", ".pcx", ".pds",
+    ".pgm", ".pic", ".pict", ".png", ".pnl", ".ppm", ".pr3", ".ps", ".psd",
+    ".qptiff", ".r3d", ".raw", ".rcpnl", ".rec", ".res", ".scn", ".sdt",
+    ".seq", ".sif", ".sld", ".sm2", ".sm3", ".spc", ".spe", ".spi", ".st",
+    ".stk", ".stp", ".svs", ".sxm", ".tc.", ".tf2", ".tf8", ".tfr", ".tga",
+    ".tif", ".tiff", ".tnb", ".top", ".txt", ".v", ".vff", ".vms", ".vsi",
+    ".vws", ".wat", ".wlz", ".wpi", ".xdce", ".xml", ".xqd", ".xqf", ".xv",
+    ".xys", ".zfp", ".zfr", ".zvi", ".ets"
+)
+
 DISALLOWED_BIOFORMATS_EXTENSIONS = {
     ".cfg",
     ".csv",
@@ -102,7 +126,7 @@
     ".zip"
 }
 
-BIOFORMATS_IMAGE_EXTENSIONS = set([f".{ext}" for ext in READABLE_FORMATS]) - DISALLOWED_BIOFORMATS_EXTENSIONS
+BIOFORMATS_IMAGE_EXTENSIONS = set(ALL_BIOFORMATS_EXTENSIONS) - DISALLOWED_BIOFORMATS_EXTENSIONS
 
 ALL_IMAGE_EXTENSIONS = SUPPORTED_IMAGE_EXTENSIONS.union(BIOFORMATS_IMAGE_EXTENSIONS)
 
@@ -128,7 +152,9 @@
 SUB_ALL = "All"
 SUB_SOME = "Some"
 FILE_SCHEME = "file:"
-PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "omero", "s3","gs")
+#TODO: disabled until CellProfiler/CellProfiler#4684 is resolved
+# PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "omero", "s3","gs")
+PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "s3","gs")
 
 CT_GRAYSCALE = "Grayscale"
 CT_COLOR = "Color"

cellprofiler_core/constants/measurement.py

@@ -69,7 +69,7 @@
 C_OBJECTS_Z = "ObjectsZPlane"
 C_OBJECTS_T = "ObjectsTimepoint"
 C_CHANNEL_TYPE = "ChannelType"
-C_FILE_LOCATION = "File_Location"
+C_FILE_LOCATION = "FileLocation"
 M_METADATA_TAGS = "_".join((C_METADATA, "Tags"))
 M_GROUPING_TAGS = "_".join((C_METADATA, "GroupingTags"))
 RESERVED_METADATA_KEYS = (C_URL, C_SERIES, C_SERIES_NAME, C_FRAME, C_FILE_LOCATION,