Merge pull request #1116 from ethho/dev-tests

PLAT-138  Follow Up to #1114

Author: A-Baji

tests/__init__.py

@@ -3,10 +3,24 @@
 import pytest
 import os
 
-PREFIX = "djtest"
+PREFIX = os.environ.get("DJ_TEST_DB_PREFIX", "djtest")
+
+# Connection for testing
+CONN_INFO = dict(
+    host=os.environ.get("DJ_TEST_HOST", "fakeservices.datajoint.io"),
+    user=os.environ.get("DJ_TEST_USER", "datajoint"),
+    password=os.environ.get("DJ_TEST_PASSWORD", "datajoint"),
+)
 
 CONN_INFO_ROOT = dict(
-    host=os.getenv("DJ_HOST"),
-    user=os.getenv("DJ_USER"),
-    password=os.getenv("DJ_PASS"),
+    host=os.environ.get("DJ_HOST", "fakeservices.datajoint.io"),
+    user=os.environ.get("DJ_USER", "root"),
+    password=os.environ.get("DJ_PASS", "simple"),
+)
+
+S3_CONN_INFO = dict(
+    endpoint=os.environ.get("S3_ENDPOINT", "fakeservices.datajoint.io"),
+    access_key=os.environ.get("S3_ACCESS_KEY", "datajoint"),
+    secret_key=os.environ.get("S3_SECRET_KEY", "datajoint"),
+    bucket=os.environ.get("S3_BUCKET", "datajoint.test"),
 )

tests/conftest.py

@@ -1,10 +1,38 @@
 import datajoint as dj
 from packaging import version
 import os
+import minio
+import urllib3
+import certifi
+import shutil
 import pytest
-from . import PREFIX, schema, schema_simple, schema_advanced
+import networkx as nx
+import json
+from pathlib import Path
+import tempfile
+from datajoint import errors
+from datajoint.errors import ADAPTED_TYPE_SWITCH, FILEPATH_FEATURE_SWITCH
+from . import (
+    PREFIX,
+    CONN_INFO,
+    S3_CONN_INFO,
+    schema,
+    schema_simple,
+    schema_advanced,
+    schema_adapted,
+)
 
-namespace = locals()
+
+@pytest.fixture(scope="session")
+def monkeysession():
+    with pytest.MonkeyPatch.context() as mp:
+        yield mp
+
+
+@pytest.fixture(scope="module")
+def monkeymodule():
+    with pytest.MonkeyPatch.context() as mp:
+        yield mp
 
 
 @pytest.fixture(scope="session")
@@ -64,11 +92,12 @@ def connection_test(connection_root):
     connection.close()
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def schema_any(connection_test):
     schema_any = dj.Schema(
-        PREFIX + "_test1", schema.__dict__, connection=connection_test
+        PREFIX + "_test1", schema.LOCALS_ANY, connection=connection_test
     )
+    assert schema.LOCALS_ANY, "LOCALS_ANY is empty"
     schema_any(schema.TTest)
     schema_any(schema.TTest2)
     schema_any(schema.TTest3)
@@ -109,10 +138,10 @@ def schema_any(connection_test):
     schema_any.drop()
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def schema_simp(connection_test):
     schema = dj.Schema(
-        PREFIX + "_relational", schema_simple.__dict__, connection=connection_test
+        PREFIX + "_relational", schema_simple.LOCALS_SIMPLE, connection=connection_test
     )
     schema(schema_simple.IJ)
     schema(schema_simple.JI)
@@ -136,10 +165,12 @@ def schema_simp(connection_test):
     schema.drop()
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def schema_adv(connection_test):
     schema = dj.Schema(
-        PREFIX + "_advanced", schema_advanced.__dict__, connection=connection_test
+        PREFIX + "_advanced",
+        schema_advanced.LOCALS_ADVANCED,
+        connection=connection_test,
     )
     schema(schema_advanced.Person)
     schema(schema_advanced.Parent)
@@ -152,3 +183,30 @@ def schema_adv(connection_test):
     schema(schema_advanced.GlobalSynapse)
     yield schema
     schema.drop()
+
+
+@pytest.fixture
+def httpClient():
+    # Initialize httpClient with relevant timeout.
+    httpClient = urllib3.PoolManager(
+        timeout=30,
+        cert_reqs="CERT_REQUIRED",
+        ca_certs=certifi.where(),
+        retries=urllib3.Retry(
+            total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504]
+        ),
+    )
+    yield httpClient
+
+
+@pytest.fixture
+def minioClient():
+    # Initialize minioClient with an endpoint and access/secret keys.
+    minioClient = minio.Minio(
+        S3_CONN_INFO["endpoint"],
+        access_key=S3_CONN_INFO["access_key"],
+        secret_key=S3_CONN_INFO["secret_key"],
+        secure=True,
+        http_client=httpClient,
+    )
+    yield minioClient

tests/schema.py

@@ -7,8 +7,6 @@
 import datajoint as dj
 import inspect
 
-LOCALS_ANY = locals()
-
 
 class TTest(dj.Lookup):
     """
@@ -33,15 +31,15 @@ class TTest2(dj.Manual):
 
 class TTest3(dj.Manual):
     definition = """
-    key : int 
+    key : int
     ---
     value : varchar(300)
     """
 
 
 class NullableNumbers(dj.Manual):
     definition = """
-    key : int 
+    key : int
     ---
     fvalue = null : float
     dvalue = null : double
@@ -450,3 +448,7 @@ class Longblob(dj.Manual):
     ---
     data: longblob
     """
+
+
+LOCALS_ANY = {k: v for k, v in locals().items() if inspect.isclass(v)}
+__all__ = list(LOCALS_ANY)

tests/schema_adapted.py

@@ -0,0 +1,62 @@
+import datajoint as dj
+import inspect
+import networkx as nx
+import json
+from pathlib import Path
+import tempfile
+
+
+class GraphAdapter(dj.AttributeAdapter):
+    attribute_type = "longblob"  # this is how the attribute will be declared
+
+    @staticmethod
+    def get(obj):
+        # convert edge list into a graph
+        return nx.Graph(obj)
+
+    @staticmethod
+    def put(obj):
+        # convert graph object into an edge list
+        assert isinstance(obj, nx.Graph)
+        return list(obj.edges)
+
+
+class LayoutToFilepath(dj.AttributeAdapter):
+    """
+    An adapted data type that saves a graph layout into fixed filepath
+    """
+
+    attribute_type = "filepath@repo-s3"
+
+    @staticmethod
+    def get(path):
+        with open(path, "r") as f:
+            return json.load(f)
+
+    @staticmethod
+    def put(layout):
+        path = Path(dj.config["stores"]["repo-s3"]["stage"], "layout.json")
+        with open(str(path), "w") as f:
+            json.dump(layout, f)
+        return path
+
+
+class Connectivity(dj.Manual):
+    definition = """
+    connid : int
+    ---
+    conn_graph = null : <graph>
+    """
+
+
+class Layout(dj.Manual):
+    definition = """
+    # stores graph layout
+    -> Connectivity
+    ---
+    layout: <layout_to_filepath>
+    """
+
+
+LOCALS_ADAPTED = {k: v for k, v in locals().items() if inspect.isclass(v)}
+__all__ = list(LOCALS_ADAPTED)

tests/schema_advanced.py

@@ -1,6 +1,5 @@
 import datajoint as dj
-
-LOCALS_ADVANCED = locals()
+import inspect
 
 
 class Person(dj.Manual):
@@ -135,3 +134,7 @@ class GlobalSynapse(dj.Manual):
     -> Cell.proj(pre_slice="slice", pre_cell="cell")
     -> Cell.proj(post_slice="slice", post_cell="cell")
     """
+
+
+LOCALS_ADVANCED = {k: v for k, v in locals().items() if inspect.isclass(v)}
+__all__ = list(LOCALS_ADVANCED)

tests/schema_simple.py

@@ -9,8 +9,7 @@
 import faker
 import numpy as np
 from datetime import date, timedelta
-
-LOCALS_SIMPLE = locals()
+import inspect
 
 
 class IJ(dj.Lookup):
@@ -237,8 +236,8 @@ class ReservedWord(dj.Manual):
     # Test of SQL reserved words
     key : int
     ---
-    in    :  varchar(25)    
-    from  :  varchar(25)   
+    in    :  varchar(25)
+    from  :  varchar(25)
     int   :  int
     select : varchar(25)
     """
@@ -260,3 +259,7 @@ class OutfitPiece(dj.Part, dj.Lookup):
         piece: varchar(20)
         """
         contents = [(0, "jeans"), (0, "sneakers"), (0, "polo")]
+
+
+LOCALS_SIMPLE = {k: v for k, v in locals().items() if inspect.isclass(v)}
+__all__ = list(LOCALS_SIMPLE)