added by_name loader with LRU cache

added lru_caching to by-name lookups

global accessor works, but namespaces still a problem

cache works for different namespaces, all tests pass

Author: gegnew

README.md

@@ -17,4 +17,12 @@ experiments = client.experiments
 exp = cellengine.Experiment(name="160311-96plex-4dye")
 ```
 
+##Developer Notes
+- `id` is a python builtin, which causes some confusion. We use `_id` to indicate
+the ID of an API object, but the `attrs` package does not accept leading
+underscores (i.e. an `_id = attr.ib()` in a class is treated by `attrs` as the
+string "id"). Practically, this means:
+    - pass `_id` to functions that take an ID as an argument.
+    - pass `id` to `attrs` classes when instantiating them.
+    - pass `properties` to `attrs` classes when instantiating them.
 

cellengine/Gates/gate_util.py

@@ -1,12 +1,14 @@
+from cellengine import loader
 import re
-
-from ..fcsfile import FcsFile
 from .. import gate  # circular import here
 from .. import _helpers
 
+cellengine = __import__(__name__.split(".")[0])
+
 
-def common_gate_create(experiment_id, body, tailored_per_file, fcs_file_id,
-                       fcs_file, create_population):
+def common_gate_create(
+    experiment_id, body, tailored_per_file, fcs_file_id, fcs_file, create_population
+):
     """
     <Description>
 
@@ -46,40 +48,37 @@ def common_gate_create(experiment_id, body, tailored_per_file, fcs_file_id,
         Example:
             <Example>
     """
-    body = parse_fcs_file_args(experiment_id, body, tailored_per_file,
-                               fcs_file_id, fcs_file)
-
-    body = _helpers.convert_dict(body, 'snake_to_camel')
-    res = _helpers.base_create(gate.Gate,
-                               url="experiments/{0}/gates".format(experiment_id),
-                               expected_status=201,
-                               json=body, params={'createPopulation':
-                                                  create_population}
-                               )
+    body = parse_fcs_file_args(
+        experiment_id, body, tailored_per_file, fcs_file_id, fcs_file
+    )
+
+    body = _helpers.convert_dict(body, "snake_to_camel")
+    res = _helpers.base_create(
+        gate.Gate,
+        url="experiments/{0}/gates".format(experiment_id),
+        expected_status=201,
+        json=body,
+        params={"createPopulation": create_population},
+    )
     return res
 
 
-def parse_fcs_file_args(experiment_id, body, tailored_per_file, fcs_file_id,
-                        fcs_file):
+def parse_fcs_file_args(experiment_id, body, tailored_per_file, fcs_file_id, fcs_file):
     """Find the fcs file ID if 'tailored_per_file' and either 'fcs_file' or
     'fcs_file_id' are specified."""
     if fcs_file is not None and fcs_file_id is not None:
         raise ValueError("Please specify only 'fcs_file' or 'fcs_file_id'.")
     if fcs_file is not None and tailored_per_file is True:  # lookup by name
         _file = get_fcsfile(experiment_id, name=fcs_file)
         fcs_file_id = _file._id
-    body['tailoredPerFile'] = tailored_per_file
-    body['fcsFileId'] = fcs_file_id
+    body["tailoredPerFile"] = tailored_per_file
+    body["fcsFileId"] = fcs_file_id
     return body
 
 
 def get_fcsfile(experiment_id, _id=None, name=None):
-    if _id:
-        content = _helpers.base_get("experiments/{0}/fcsfiles/{1}".format(experiment_id, _id))
-        content = FcsFile(properties=content)
-    else:
-        content = _helpers.load_fcsfile_by_name(experiment_id, name)
-    return content
+    fcs_loader = loader.FcsFileLoader(experiment_id=experiment_id, id=_id, name=name)
+    return fcs_loader.load()
 
 
 def create_gates(experiment_id=None, gates=None, create_all_populations=True):
@@ -108,8 +107,9 @@ def create_gates(experiment_id=None, gates=None, create_all_populations=True):
                     ``help(cellengine.Gate.<Gate Type>)``.
             fcs_file_id (optional): ``str``: ID of FCS file, if tailored per file. Use
                 ``None`` for the global gate in the tailored gate group.
-            tailored_per_file (optional): ``bool``: Whether this gate is tailored per FCS file.
-            names (optional): ``list(str)``: For compound gates, a list of gate names.
+            tailored_per_file (optional): ``bool``: Whether this gate is
+                tailored per FCS file.  names (optional): ``list(str)``: For
+                compound gates, a list of gate names.
             create_population (optional): Whether to create populations for each gate.
         create_populations: Whether to create populations for all gates. If set
         to False, ``create_population`` may be specified for each gate.
@@ -119,23 +119,25 @@ def create_gates(experiment_id=None, gates=None, create_all_populations=True):
     """
     prepared_gates = []
     for g in gates:
-        g.update({'_id': _helpers.generate_id()})
-        if 'gid' not in g.keys():
-            g.update({'gid': _helpers.generate_id()})
+        g.update({"_id": _helpers.generate_id()})
+        if "gid" not in g.keys():
+            g.update({"gid": _helpers.generate_id()})
         prepared_gates.append(g)
 
     new_gates = []
     for each_gate in prepared_gates:
         if create_all_populations is False:
-            create_populations = each_gate.get('create_population', False)
+            create_populations = each_gate.get("create_population", False)
         else:
             create_populations = create_all_populations
-        new_gate = common_gate_create(experiment_id=each_gate.get('experiment_id', experiment_id),
-                                 body=each_gate,
-                                 tailored_per_file=each_gate.get('tailored_per_file', False),
-                                 fcs_file_id=each_gate.get('fcs_file_id', None),
-                                 fcs_file=each_gate.get('fcs_file', None),
-                                 create_population=create_populations)
+        new_gate = common_gate_create(
+            experiment_id=each_gate.get("experiment_id", experiment_id),
+            body=each_gate,
+            tailored_per_file=each_gate.get("tailored_per_file", False),
+            fcs_file_id=each_gate.get("fcs_file_id", None),
+            fcs_file=each_gate.get("fcs_file", None),
+            create_population=create_populations,
+        )
         new_gates.append(new_gate)
 
     return new_gates
@@ -175,12 +177,14 @@ def delete_gates(experiment_id, _id=None, gid=None, exclude=None):
 
 
 def gate_style(prnt_doc, child_doc):
-    desc = child_doc[:child_doc.index('Args')].strip()
-    args = child_doc[child_doc.index('Args')+5:child_doc.index('Returns')].strip()
-    args = re.sub('\n', '\n    ', args)
-    returns = child_doc[child_doc.index('Returns')+10:child_doc.index('Example')].strip()
-    example = child_doc[child_doc.index('Example')+10:].strip()
-    keys = ['<Description>', '<Gate Args>', '<Returns>', '<Example>']
+    desc = child_doc[: child_doc.index("Args")].strip()
+    args = child_doc[child_doc.index("Args") + 5 : child_doc.index("Returns")].strip()
+    args = re.sub("\n", "\n    ", args)
+    returns = child_doc[
+        child_doc.index("Returns") + 10 : child_doc.index("Example")
+    ].strip()
+    example = child_doc[child_doc.index("Example") + 10 :].strip()
+    keys = ["<Description>", "<Gate Args>", "<Returns>", "<Example>"]
     sections = [desc, args, returns, example]
     docs = prnt_doc
     for key, section in zip(keys, sections):

cellengine/__init__.py

@@ -2,15 +2,26 @@
 import requests
 from requests_toolbelt import sessions
 
-BASE_URL = os.environ.get('CELLENGINE_DEVELOPMENT', 'https://cellengine.com/api/v1/')
+BASE_URL = os.environ.get("CELLENGINE_DEVELOPMENT", "https://cellengine.com/api/v1/")
 ID_INDEX = 0
 
 session = sessions.BaseUrlSession(base_url=BASE_URL)
-session.headers.update({'User-Agent': "CellEngine Python API Toolkit/0.1.1 requests/{0}".format(requests.__version__)})
+session.headers.update(
+    {
+        "User-Agent": "CellEngine Python API Toolkit/0.1.1 requests/{0}".format(
+            requests.__version__
+        )
+    }
+)
 
 from .client import Client
 from .experiment import Experiment
 from .population import Population
 from .compensation import Compensation
 from .fcsfile import FcsFile
 from .gate import Gate
+
+from .loader import by_name
+
+cache_info = by_name.cache_info
+clear_cache = by_name.cache_clear

cellengine/_helpers.py

@@ -2,6 +2,7 @@
 import re
 import time
 import binascii
+from functools import lru_cache
 from .client import session
 from datetime import datetime
 from cellengine import ID_INDEX
@@ -95,57 +96,6 @@ def today_timestamp():
 def created(self):
     return timestamp_to_datetime(self._properties.get("created"))
 
-def load(self, path, query='name'):
-    if self._id is None:
-        load_by_name(self, query)
-    else:
-        content = base_get(path)
-        if len(content) == 0:
-            ValueError("Failed to load object from {0}".format(self.path))
-        else:
-            self._properties = content
-            self.__dict__.update(self._properties)
-
-
-def load_by_id(_id):
-    content = base_get("experiments/{0}".format(_id))
-    return content
-
-
-def load_by_name(self, query):
-    # TODO does requests encode URI components for us?
-    url = "{0}?query=eq({1},\"{2}\")&limit=2".format(self.path, query, self.name)
-    content = base_get(url)
-    if len(content) == 0:
-        raise RuntimeError("No objects found with the name {0}.".format(self.name))
-    elif len(content) > 1:
-        raise RuntimeError("Multiple objects found with the name {0}, use _id to query instead.".format(self.name))
-    else:
-        self._properties = content[0]
-        self._id = self._properties.get("_id")
-        self.__dict__.update(self._properties)
-
-
-def load_experiment_by_name(name):
-    content = base_get("experiments?query=eq(name, \"{0}\")&limit=2".format(name))
-    return load_object_by_name('__import__("cellengine").Experiment', content)
-
-
-def load_fcsfile_by_name(experiment_id, name=None):
-    content = base_get("experiments/{0}/fcsfiles?query=eq(filename, \"{1}\")&limit=2".format(experiment_id, name))
-    return load_object_by_name('__import__("cellengine").FcsFile', content)
-
-
-def load_object_by_name(classname, content):
-    if len(content) == 0:
-        raise RuntimeError("No objects found.")
-    elif len(content) > 1:
-        raise RuntimeError("Multiple objects found; use _id to query instead.")
-    elif type(content) is list:
-        return make_class(classname, content[0])
-    else:
-        return make_class(classname, content)
-
 
 def generate_id():
     """Generates a hexadecimal ID based on a mongoDB ObjectId"""

cellengine/client.py

@@ -2,6 +2,7 @@
 from getpass import getpass
 from . import session
 from . import _helpers
+from .loader import ExperimentLoader
 from .experiment import Experiment
 
 
@@ -28,6 +29,7 @@ class Client(object):
     Returns:
         client: Authenticated client object
     """
+
     username = attr.ib(default=None)
     password = attr.ib(default=None, repr=False)
     token = attr.ib(default=None, repr=False)
@@ -39,14 +41,13 @@ def __attrs_post_init__(self):
             if self.password is None:
                 self.password = getpass()
 
-            req = session.post("signin", {
-                "username": self.username,
-                "password": self.password
-            })
+            req = session.post(
+                "signin", {"username": self.username, "password": self.password}
+            )
             req.raise_for_status()
 
             if req.status_code == 200:
-                print('Authentication successful.')
+                print("Authentication successful.")
 
         elif self.token is not None:
             session.cookies.update({"token": "{0}".format(self.token)})
@@ -55,14 +56,16 @@ def __attrs_post_init__(self):
             raise RuntimeError("Username or token must be provided")
 
     def get_experiment(self, _id=None, name=None):
-        if _id:
-            content = _helpers.base_get("experiments/{0}".format(_id))
-            content = Experiment(properties=content)
-        else:
-            content = _helpers.load_experiment_by_name(name)
-        return content
+        loader = ExperimentLoader(id=_id, name=name)
+        return loader.load()
+        # if _id:
+        #     content = _helpers.base_get("experiments/{0}".format(_id))
+        #     content = Experiment(properties=content)
+        # else:
+        #     content = _helpers.load_experiment_by_name(name)
+        # return content
 
     @property
     def experiments(self):
         """Return a list of Experiment objects for all experiments on client"""
-        return _helpers.base_list('experiments', Experiment)
+        return _helpers.base_list("experiments", Experiment)