diff --git a/cellengine/resources/experiment.py b/cellengine/resources/experiment.py index 123eb539..594c476a 100644 --- a/cellengine/resources/experiment.py +++ b/cellengine/resources/experiment.py @@ -189,8 +189,7 @@ def clone(self, props: Optional[Dict[str, Any]] = None) -> Experiment: """ return ce.APIClient().clone_experiment(self._id, props) - @property - def delete(self): + def delete(self) -> None: """Marks the experiment as deleted. Deleted experiments are permanently deleted after approximately @@ -198,8 +197,7 @@ def delete(self): """ self.deleted = datetime.today() - @property - def undelete(self): + def undelete(self) -> None: """Clear a scheduled deletion.""" if self.deleted: del self.deleted diff --git a/cellengine/resources/fcs_file.py b/cellengine/resources/fcs_file.py index c7107378..3cf5b2c5 100644 --- a/cellengine/resources/fcs_file.py +++ b/cellengine/resources/fcs_file.py @@ -76,7 +76,7 @@ def annotations(self, val): self._annotations = val @property - def channels(self) -> List: + def channels(self) -> List[str]: """Return all channels in the file""" return [f["channel"] for f in self.panel] # type: ignore diff --git a/cellengine/resources/gate.py b/cellengine/resources/gate.py index 4b9fc4cb..dfc26d0f 100644 --- a/cellengine/resources/gate.py +++ b/cellengine/resources/gate.py @@ -1,5 +1,9 @@ from __future__ import annotations import importlib +from cellengine.utils.types import ( + ApplyTailoringInsert, + ApplyTailoringUpdate, +) from math import pi from operator import itemgetter from typing import Any, Dict, List, Optional, Union, Tuple, overload @@ -13,7 +17,6 @@ from numpy import array, mean, stack import cellengine as ce -from cellengine.resources.fcs_file import FcsFile from cellengine.resources.population import Population from cellengine.utils import parse_fcs_file_args from cellengine.utils import converter, generate_id, readonly @@ -57,6 +60,13 @@ def deep_update(source, overrides): return source +class ApplyTailoringResult: + def __init__(self): + self.inserted: List[Gate] = [] + self.updated: List[Gate] = [] + self.deleted: List[str] = [] + + @define(repr=False) class Gate: _id: str = field(on_setattr=readonly) @@ -138,10 +148,22 @@ def update_gate_family(experiment_id: str, gid: str, body: Dict) -> None: if res["nModified"] < 1: raise Warning("No gates updated.") - def tailor_to(self, fcs_file: FcsFile): - self.tailored_per_file = True - self.fcs_file_id = fcs_file._id - self.update() + def apply_tailoring(self, fcs_file_ids: List[str]) -> ApplyTailoringResult: + """Apply this gate's tailoring (copy its geometry) to other FCS files.""" + payload = ce.APIClient().apply_tailoring(self.experiment_id, self, fcs_file_ids) + ret = ApplyTailoringResult() + [ret.inserted.append(self._synthesize_gate(i)) for i in payload["inserted"]] + [ret.updated.append(self._synthesize_gate(i)) for i in payload["updated"]] + [ret.deleted.append(i["_id"]) for i in payload["deleted"]] + return ret + + def _synthesize_gate( + self, + payload: Union[ApplyTailoringInsert, ApplyTailoringUpdate], + ): + gate = self.to_dict() + gate.update(payload) + return Gate.from_dict(gate) class RectangleGate(Gate): diff --git a/cellengine/utils/api_client/APIClient.py b/cellengine/utils/api_client/APIClient.py index ed1d9b27..f6379607 100644 --- a/cellengine/utils/api_client/APIClient.py +++ b/cellengine/utils/api_client/APIClient.py @@ -1,4 +1,5 @@ from __future__ import annotations +from cellengine.utils.types import ApplyTailoringRes from functools import lru_cache from getpass import getpass import importlib @@ -603,12 +604,15 @@ def update_gate_family(self, experiment_id, gid, body: dict = None) -> dict: json=body, ) - def tailor_to(self, experiment_id, gate_id, fcs_file_id): - """Tailor a gate to a file.""" - gate = self.get_gate(experiment_id, gate_id, as_dict=True) - gate["tailoredPerFile"] = True - gate["fcsFileId"] = fcs_file_id - return self.update_entity(experiment_id, gate_id, "gates", gate) + def apply_tailoring( + self, experiment_id: str, gate: Gate, fcs_file_ids: List[str] + ) -> ApplyTailoringRes: + """Tailor a gate to a file or files.""" + return self._post( + f"{self.base_url}/experiments/{experiment_id}/gates/applyTailored", + params={"gid": gate.gid}, + json={"gate": gate.to_dict(), "fcsFileIds": fcs_file_ids}, + ) def get_plot( self, diff --git a/cellengine/utils/types.py b/cellengine/utils/types.py new file mode 100644 index 00000000..1a7f78c8 --- /dev/null +++ b/cellengine/utils/types.py @@ -0,0 +1,20 @@ +from __future__ import annotations +import sys +from typing import List + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +ApplyTailoringInsert = TypedDict("ApplyTailoringInsert", {"_id": str, "fcsFileId": str}) +ApplyTailoringUpdate = TypedDict("ApplyTailoringUpdate", {"_id": str, "fcsFileId": str}) +ApplyTailoringDelete = TypedDict("ApplyTailoringDelete", {"_id": str, "fcsFileId": str}) +ApplyTailoringRes = TypedDict( + "ApplyTailoringRes", + { + "inserted": List[ApplyTailoringInsert], + "updated": List[ApplyTailoringUpdate], + "deleted": List[ApplyTailoringDelete], + }, +) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0e6a779f..87b9968e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -pytest~=5.3 +pytest~=7.1.3 pytest-recording~=0.12 pyright~=0.0.9 responses~=0.10 diff --git a/tests/data/Specimen_001_A1_A01_MeOHperm(DL350neg).fcs b/tests/data/Specimen_001_A1_A01_MeOHperm(DL350neg).fcs new file mode 100644 index 00000000..6a8d2f1c Binary files /dev/null and b/tests/data/Specimen_001_A1_A01_MeOHperm(DL350neg).fcs differ diff --git a/tests/data/Specimen_001_A2_A02_MeOHperm(DL350neg).fcs b/tests/data/Specimen_001_A2_A02_MeOHperm(DL350neg).fcs new file mode 100644 index 00000000..3fe5e669 Binary files /dev/null and b/tests/data/Specimen_001_A2_A02_MeOHperm(DL350neg).fcs differ diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 21c91638..a09ef402 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -3,6 +3,8 @@ import pandas import uuid +from typing import Iterator + import cellengine from cellengine.utils.api_client.APIClient import APIClient from cellengine.utils.api_client.APIError import APIError @@ -20,52 +22,77 @@ @pytest.fixture(scope="module") -def client(): +def run_id() -> str: + return uuid.uuid4().hex[:5] + + +@pytest.fixture(scope="module") +def client() -> APIClient: username = os.environ.get("CELLENGINE_USERNAME", "gegnew") password = os.environ.get("CELLENGINE_PASSWORD", "testpass1") - return cellengine.APIClient(username=username, password=password) + return APIClient(username=username, password=password) -@pytest.fixture(scope="module") -def setup_experiment(request, client: APIClient): - print("Setting up CellEngine experiment for {}".format(__name__)) - exp = cellengine.Experiment.create("new_experiment") - exp.upload_fcs_file("tests/data/Acea - Novocyte.fcs") +@pytest.fixture() +def blank_experiment(run_id: str, client: APIClient) -> Iterator[Experiment]: + exp_name = f"Ligands {run_id}" + print(f"Setting up CellEngine experiment {exp_name}") + exp = Experiment.create(exp_name) yield exp - print("Starting teardown of: {}".format(__name__)) + print(f"Starting teardown of {exp_name}") client.delete_experiment(exp._id) -def test_experiment_attachments(setup_experiment, client: APIClient): - experiment = client.get_experiment(name="new_experiment") +@pytest.fixture() +def setup_experiment(blank_experiment: Experiment) -> Iterator[Experiment]: + blank_experiment.upload_fcs_file("tests/data/Acea - Novocyte.fcs") + + yield blank_experiment + + +@pytest.fixture() +def ligands_experiment( + blank_experiment: Experiment, client: APIClient +) -> Iterator[Experiment]: + blank_experiment.upload_fcs_file( + "tests/data/Specimen_001_A1_A01_MeOHperm(DL350neg).fcs" + ) + blank_experiment.upload_fcs_file( + "tests/data/Specimen_001_A2_A02_MeOHperm(DL350neg).fcs" + ) + + yield blank_experiment + +def test_experiment_attachments(blank_experiment: Experiment, client: APIClient): # POST - experiment.upload_attachment("tests/data/text.txt") - experiment.upload_attachment("tests/data/text.txt", filename="text2.txt") + blank_experiment.upload_attachment("tests/data/text.txt") + blank_experiment.upload_attachment("tests/data/text.txt", filename="text2.txt") # GET - attachments = experiment.attachments + attachments = blank_experiment.attachments assert all([type(a) is Attachment for a in attachments]) assert len(attachments) == 2 - assert experiment.download_attachment(name="text2.txt") == b"hello world\n\n" - att1 = experiment.get_attachment(name="text.txt") - experiment.get_attachment(_id=experiment.attachments[1]._id) + assert blank_experiment.download_attachment(name="text2.txt") == b"hello world\n\n" + att1 = blank_experiment.get_attachment(name="text.txt") + blank_experiment.get_attachment(_id=blank_experiment.attachments[1]._id) # UPDATE att1.filename = "newname.txt" att1.update() - assert experiment.download_attachment(name="newname.txt") == b"hello world\n\n" + assert ( + blank_experiment.download_attachment(name="newname.txt") == b"hello world\n\n" + ) # DELETE - [a.delete() for a in experiment.attachments] - assert len(experiment.attachments) == 0 + [a.delete() for a in blank_experiment.attachments] + assert len(blank_experiment.attachments) == 0 -def test_fcs_file_events(setup_experiment, client: APIClient): - experiment = client.get_experiment(name="new_experiment") - file = experiment.fcs_files[0] +def test_fcs_file_events(ligands_experiment: Experiment): + file = ligands_experiment.fcs_files[0] all_events = file.events limited_events = file.get_events(preSubsampleN=10) @@ -78,15 +105,13 @@ def test_fcs_file_events(setup_experiment, client: APIClient): assert len(limited_events) == len(file.events) -def test_apply_compensations(setup_experiment, client: APIClient): - experiment = client.get_experiment(name="new_experiment") - +def test_apply_compensations(setup_experiment): # POST - file1 = experiment.fcs_files[0] - experiment.create_compensation("test_comp", file1.channels[0:2], [1, 2, 3, 4]) + file1 = setup_experiment.fcs_files[0] + setup_experiment.create_compensation("test_comp", file1.channels[0:2], [1, 2, 3, 4]) # GET - compensations = experiment.compensations + compensations = setup_experiment.compensations assert all([type(c) is Compensation for c in compensations]) comp = compensations[0] assert comp.name == "test_comp" @@ -94,7 +119,7 @@ def test_apply_compensations(setup_experiment, client: APIClient): # UPDATE comp.name = "new_name" comp.update() - comp = experiment.get_compensation(name="new_name") + comp = setup_experiment.get_compensation(name="new_name") # Additional functionality events_df = comp.apply(file1, preSubsampleN=100) @@ -106,14 +131,12 @@ def test_apply_compensations(setup_experiment, client: APIClient): # TODO assert # DELETE - experiment.compensations[0].delete() - assert experiment.compensations == [] - + setup_experiment.compensations[0].delete() + assert setup_experiment.compensations == [] -def test_apply_file_internal_compensation(setup_experiment, client: APIClient): - experiment = client.get_experiment(name="new_experiment") - file = experiment.fcs_files[0] +def test_apply_file_internal_compensation(setup_experiment: Experiment): + file = setup_experiment.fcs_files[0] comp = file.get_file_internal_compensation() events_df = comp.apply(file, preSubsampleN=100) @@ -124,7 +147,7 @@ def test_apply_file_internal_compensation(setup_experiment, client: APIClient): assert all([c in events_df.columns for c in comp.channels]) -def test_experiment(setup_experiment, client: APIClient): +def test_experiment(client: APIClient): experiment_name = uuid.uuid4().hex # POST @@ -149,16 +172,14 @@ def test_experiment(setup_experiment, client: APIClient): client.get_experiment(exp._id) -def test_experiment_fcs_files(setup_experiment, client): - experiment = client.get_experiment(name="new_experiment") - +def test_experiment_fcs_files(setup_experiment: Experiment, client: APIClient): # GET - files = experiment.fcs_files + files = setup_experiment.fcs_files assert all([type(f) is FcsFile for f in files]) # CREATE file2 = FcsFile.create( - experiment._id, + setup_experiment._id, fcs_files=files[0]._id, filename="new_fcs_file.fcs", pre_subsample_n=10, @@ -167,7 +188,7 @@ def test_experiment_fcs_files(setup_experiment, client): # UPDATE file2.filename = "renamed.fcs" file2.update() - file = experiment.get_fcs_file(name="renamed.fcs") + file = setup_experiment.get_fcs_file(name="renamed.fcs") assert file._id == file2._id # events @@ -177,16 +198,15 @@ def test_experiment_fcs_files(setup_experiment, client): # DELETE file.delete() with pytest.raises(APIError): - client.get_fcs_file(experiment._id, file._id) + client.get_fcs_file(setup_experiment._id, file._id) -def test_experiment_gates(setup_experiment, client: APIClient): - experiment = client.get_experiment(name="new_experiment") - fcs_file = experiment.fcs_files[0] +def test_experiment_gates(setup_experiment: Experiment): + fcs_file = setup_experiment.fcs_files[0] # CREATE split_gate = SplitGate.create( - experiment._id, + setup_experiment._id, fcs_file.channels[0], "split_gate", 2300000, @@ -194,7 +214,7 @@ def test_experiment_gates(setup_experiment, client: APIClient): create_population=False, ) range_gate = RangeGate.create( - experiment._id, + setup_experiment._id, fcs_file.channels[0], "range_gate", 2100000, @@ -203,32 +223,77 @@ def test_experiment_gates(setup_experiment, client: APIClient): ) # UPDATE - range_gate.tailor_to(fcs_file) - assert range_gate.tailored_per_file is True - assert range_gate.fcs_file_id == fcs_file._id - Gate.update_gate_family( - experiment._id, + setup_experiment._id, split_gate.gid, body={"name": "new split gate name"}, ) - assert experiment.gates[0].name == "new split gate name" + assert setup_experiment.gates[0].name == "new split gate name" # DELETE range_gate.delete() - assert len(experiment.gates) == 1 + assert len(setup_experiment.gates) == 1 + + setup_experiment.delete_gate(gid=split_gate.gid) + assert setup_experiment.gates == [] + + +def test_apply_tailoring(ligands_experiment: Experiment): + fcs_files = ligands_experiment.fcs_files + + # Global + global_gate, pop = ligands_experiment.create_rectangle_gate( + x_channel=fcs_files[0].channels[0], + y_channel=fcs_files[0].channels[0], + name="gate 1", + x1=10, + y1=10, + x2=20, + y2=20, + create_population=True, + tailored_per_file=True, + fcs_file_id=None, + ) + print(global_gate.model) + + # Tailored to f1 + f1_gate = ligands_experiment.create_rectangle_gate( + x_channel=fcs_files[0].channels[0], + y_channel=fcs_files[0].channels[0], + name="gate 1", + x1=50, + y1=50, + x2=70, + y2=70, + tailored_per_file=True, + fcs_file_id=fcs_files[0]._id, + gid=global_gate.gid, + create_population=False, + ) + + # Apply to f2 should insert a gate for that file + tailored1 = f1_gate.apply_tailoring([fcs_files[1]._id]) + assert len(tailored1.inserted) == 1 + assert len(tailored1.updated) == 0 + assert len(tailored1.deleted) == 0 + assert tailored1.inserted[0].fcs_file_id == fcs_files[1]._id + print(tailored1.inserted[0].model) - experiment.delete_gate(gid=split_gate.gid) - assert experiment.gates == [] + # Apply global to f2 should delete f2's tailored gate + print(global_gate.model) + tailored2 = global_gate.apply_tailoring([fcs_files[1]._id]) + assert len(tailored2.inserted) == 0 + assert len(tailored2.updated) == 0 + assert len(tailored2.deleted) == 1 + assert tailored2.deleted[0] == tailored1.inserted[0]._id -def test_experiment_populations(setup_experiment, client: APIClient): - experiment = client.get_experiment(name="new_experiment") - fcs_file = experiment.fcs_files[0] +def test_experiment_populations(setup_experiment: Experiment, client: APIClient): + fcs_file = setup_experiment.fcs_files[0] # GET quad_gate, quad_pops = QuadrantGate.create( - experiment._id, + setup_experiment._id, fcs_file.channels[0], fcs_file.channels[1], "quadrant_gate", @@ -237,7 +302,7 @@ def test_experiment_populations(setup_experiment, client: APIClient): ) split_gate, split_pops = SplitGate.create( - experiment._id, fcs_file.channels[0], "split gate", 2300000, 250000 + setup_experiment._id, fcs_file.channels[0], "split gate", 2300000, 250000 ) assert ["split gate (L)", "split gate (R)"] == [p.name for p in split_pops] @@ -247,8 +312,10 @@ def test_experiment_populations(setup_experiment, client: APIClient): .Or([quad_gate.model["gids"][0], quad_gate.model["gids"][2]]) .build() ) - client.post_population(experiment._id, complex_payload) - complex_pop = [p for p in experiment.populations if "complex pop" in p.name][0] + client.post_population(setup_experiment._id, complex_payload) + complex_pop = [p for p in setup_experiment.populations if "complex pop" in p.name][ + 0 + ] # UPDATE complex_pop.name = "my new name" @@ -257,16 +324,15 @@ def test_experiment_populations(setup_experiment, client: APIClient): # DELETE complex_pop.delete() - assert "complex pop" not in [p.name for p in experiment.populations] + assert "complex pop" not in [p.name for p in setup_experiment.populations] -def test_create_new_fcsfile_from_s3(setup_experiment, client: APIClient): +def test_create_new_fcsfile_from_s3(blank_experiment: Experiment): if not "S3_ACCESS_KEY" in os.environ: pytest.skip( "Skipping S3 tests. Set S3_ACCESS_KEY and S3_SECRET_KEY to run them." ) - experiment = client.get_experiment(name="new_experiment") s3_dict = { "host": "ce-test-s3-a.s3.us-east-2.amazonaws.com", "path": "/Specimen_001_A6_A06.fcs", @@ -275,7 +341,7 @@ def test_create_new_fcsfile_from_s3(setup_experiment, client: APIClient): } file = FcsFile.create( - experiment._id, + blank_experiment._id, s3_dict, "new name", )